def test_run_process_more_than_1000_entities(self):
"""Tests we can process more entities than the old limit of 1k."""
counter = counters.PerfCounter(
'test-run-process-more-than-1000-entities-counter',
'counter for testing increment by QueryMapper')
db.put([Model() for _ in xrange(1001)])
# Also pass custom args to QueryMapper ctor.
num_processed = utils.QueryMapper(Model.all(),
batch_size=50,
counter=counter,
report_every=0).run(process,
1,
string='foo')
last_written = Model.all().order('-create_date').get()
self.assertEqual(1001, counter.value)
self.assertEqual(1001, num_processed)
self.assertEqual(1, last_written.number)
self.assertEqual('foo', last_written.string)
from google.appengine.ext import db
MODULE_NAME = 'Full Text Search'
DEPRECATED = config.ConfigProperty(
'gcb_can_index_automatically',
bool,
safe_dom.Text(
'This property has been deprecated; it is retained so that we '
'will not generate no-such-variable error messages for existing '
'installations that have this property set.'),
default_value=False,
label='Automatically index search',
deprecated=True)
SEARCH_QUERIES_MADE = counters.PerfCounter(
'gcb-search-queries-made',
'The number of student queries made to the search module.')
SEARCH_RESULTS_RETURNED = counters.PerfCounter(
'gcb-search-results-returned',
'The number of search results returned across all student queries.')
SEARCH_FAILURES = counters.PerfCounter(
'gcb-search-failures',
'The number of search failure messages returned across all student '
'queries.')
INDEX_NAME = 'gcb_search_index_loc_%s'
RESULTS_LIMIT = 10
GCB_SEARCH_FOLDER_NAME = os.path.normpath('/modules/search/')
MAX_RETRIES = 5
from models import entities
from models import entity_transforms
from models import student_work
from models import transforms
from models import utils
import models.review
from modules.dashboard import dashboard
from modules.review import dashboard as review_dashboard
from modules.review import domain
from modules.review import peer
from modules.review import stats
from google.appengine.ext import db
# In-process increment-only performance counters.
COUNTER_ADD_REVIEWER_BAD_SUMMARY_KEY = counters.PerfCounter(
'gcb-pr-add-reviewer-bad-summary-key',
'number of times add_reviewer() failed due to a bad review summary key')
COUNTER_ADD_REVIEWER_SET_ASSIGNER_KIND_HUMAN = counters.PerfCounter(
'gcb-pr-add-reviewer-set-assigner-kind-human',
("number of times add_reviewer() changed an existing step's assigner_kind "
'to ASSIGNER_KIND_HUMAN'))
COUNTER_ADD_REVIEWER_CREATE_REVIEW_STEP = counters.PerfCounter(
'gcb-pr-add-reviewer-create-review-step',
'number of times add_reviewer() created a new review step')
COUNTER_ADD_REVIEWER_EXPIRED_STEP_REASSIGNED = counters.PerfCounter(
'gcb-pr-add-reviewer-expired-step-reassigned',
'number of times add_reviewer() reassigned an expired step')
COUNTER_ADD_REVIEWER_FAILED = counters.PerfCounter(
'gcb-pr-add-reviewer-failed',
'number of times add_reviewer() had a fatal error')
COUNTER_ADD_REVIEWER_REMOVED_STEP_UNREMOVED = counters.PerfCounter(
from models import custom_modules
from models import models
from models import review as models_review
from models import roles
from models import student_work
from models import transforms
from modules.assessments import assessments
from modules.courses import unit_outline
from modules.review import domain
from tools import verify
from google.appengine.ext import db
COURSE_EVENTS_RECEIVED = counters.PerfCounter(
'gcb-course-events-received',
'A number of activity/assessment events received by the server.')
COURSE_EVENTS_RECORDED = counters.PerfCounter(
'gcb-course-events-recorded',
'A number of activity/assessment events recorded in a datastore.')
UNIT_PAGE_TYPE = 'unit'
ACTIVITY_PAGE_TYPE = 'activity'
ASSESSMENT_PAGE_TYPE = 'assessment'
ASSESSMENT_CONFIRMATION_PAGE_TYPE = 'test_confirmation'
TAGS_THAT_TRIGGER_BLOCK_COMPLETION = ['attempt-activity']
TAGS_THAT_TRIGGER_COMPONENT_COMPLETION = ['tag-assessment']
TAGS_THAT_TRIGGER_HTML_COMPLETION = ['attempt-lesson']
models/student_work.py. Entities declared here should not be used by external
clients.
"""
__author__ = [
'[email protected] (John Cox)',
]
from models import counters
from models import models
from models import student_work
from modules.review import domain
from google.appengine.ext import db
COUNTER_INCREMENT_COUNT_COUNT_AGGREGATE_EXCEEDED_MAX = counters.PerfCounter(
'gcb-pr-increment-count-count-aggregate-exceeded-max',
('number of times increment_count() failed because the new aggregate of '
'the counts would have exceeded domain.MAX_UNREMOVED_REVIEW_STEPS'))
class ReviewSummary(student_work.BaseEntity):
"""Object that tracks the aggregate state of reviews for a submission."""
# UTC last modification timestamp.
change_date = db.DateTimeProperty(auto_now=True, required=True)
# UTC create date.
create_date = db.DateTimeProperty(auto_now_add=True, required=True)
# Strong counters. Callers should never manipulate these directly. Instead,
# use decrement|increment_count.
# Count of ReviewStep entities for this submission currently in state
# STATE_ASSIGNED.
def build(cls, name, label, desc, max_size_bytes, ttl_sec, dao_class):
"""Build the family of classes for a process-scoped Entity cache.
Args:
name: Name under which cache is registered. This should be in the
lower_case_and_underscores naming style
label: Label for the course-level setting enabling/disabling
process-level caching for this entity type.
desc: Description to add to the course-level setting
enabling/disabling process level caching for this entity type.
max_size_bytes: Largest size the cache may take on. If adding
an item to the cache would make it exceed this size, items
are LRU'd out until the item fits.
ttl_sec: Number of seconds after which cached entries are
considered stale and a (lazy) refresh is performed.
dao_class: The class of an DAO in the Entity/DTO/DAO scheme
common for Course Builder data access. Used for itself
and also for its references to its matching DTO, Entity
classes.
Returns:
A ResourceCacheFactory entry containing the constellation of
objects that interoperate to form a cche.
"""
if name in cls._CACHES:
return cls._CACHES[name]
config_property = config.ConfigProperty(
'gcb_can_use_%s_in_process_cache' % name,
bool,
desc,
label=label,
default_value=True)
class EntityCache(caching.ProcessScopedSingleton):
"""This class holds in-process global cache of objects."""
@classmethod
def get_cache_len(cls):
# pylint: disable=protected-access
return len(cls.instance()._cache.items.keys())
@classmethod
def get_cache_size(cls):
# pylint: disable=protected-access
return cls.instance()._cache.total_size
def __init__(self):
self._cache = caching.LRUCache(max_size_bytes=max_size_bytes)
self._cache.get_entry_size = self._get_entry_size
def _get_entry_size(self, key, value):
if not value:
return 0
return sys.getsizeof(key) + sys.getsizeof(value)
@property
def cache(self):
return self._cache
class CacheEntry(caching.AbstractCacheEntry):
"""Cache entry containing an entity."""
def __init__(self, entity):
self.entity = entity
self.created_on = datetime.datetime.utcnow()
def getsizeof(self):
return (dao_class.ENTITY.getsizeof(self.entity) +
sys.getsizeof(self.created_on))
def has_expired(self):
age = (datetime.datetime.utcnow() -
self.created_on).total_seconds()
return age > ttl_sec
def is_up_to_date(self, key, update):
if update and self.entity:
return update.updated_on == self.entity.updated_on
return not update and not self.entity
def updated_on(self):
if self.entity:
return self.entity.updated_on
return None
@classmethod
def externalize(cls, key, entry):
entity = entry.entity
if not entity:
return None
return dao_class.DTO(entity.key().id_or_name(),
transforms.loads(entity.data))
@classmethod
def internalize(cls, key, entity):
return cls(entity)
class CacheConnection(caching.AbstractCacheConnection):
PERSISTENT_ENTITY = dao_class.ENTITY
CACHE_ENTRY = CacheEntry
@classmethod
def init_counters(cls):
caching.AbstractCacheConnection.init_counters()
@classmethod
def is_enabled(cls):
return config_property.value
def __init__(self, namespace):
caching.AbstractCacheConnection.__init__(self, namespace)
self.cache = EntityCache.instance().cache
def get_updates_when_empty(self):
"""Load in all ResourceBundles when cache is empty."""
q = self.PERSISTENT_ENTITY.all()
for entity in caching.iter_all(q):
self.put(entity.key().name(), entity)
self.CACHE_UPDATE_COUNT.inc()
# we don't have any updates to apply; all items are new
return {}
class ConnectionManager(caching.RequestScopedSingleton):
"""Class that provides access to in-process Entity cache.
This class only supports get() and does not intercept
put() or delete() and is unaware of changes to
Entities made in this very process. When
entites change, the changes will be picked up
when new instance of this class is created. If you are
watching perfomance counters, you will see EVICT and
EXPIRE being incremented, but not DELETE or PUT.
"""
def __init__(self):
# Keep a separate CacheConnection for each namespace that
# makes a get() request.
self._conns = {}
def _conn(self, ns):
connected = self._conns.get(ns)
if not connected:
logging.debug(
'CONNECTING a CacheConnection for namespace "%s",', ns)
connected = CacheConnection.new_connection(ns)
self._conns[ns] = connected
return connected
@classmethod
def _ns(cls, app_context):
if app_context:
return app_context.get_namespace_name()
return namespace_manager.get_namespace()
def _get(self, key, namespace):
found, stream = self._conn(namespace).get(key)
if found and stream:
return stream
with utils.Namespace(namespace):
entity = dao_class.ENTITY_KEY_TYPE.get_entity_by_key(
dao_class.ENTITY, str(key))
if entity:
self._conn(namespace).put(key, entity)
return dao_class.DTO(entity.key().id_or_name(),
transforms.loads(entity.data))
self._conn(namespace).CACHE_NOT_FOUND.inc()
self._conn(namespace).put(key, None)
return None
def _get_multi(self, keys, namespace):
return [self._get(key, namespace) for key in keys]
@classmethod
def get(cls, key, app_context=None):
# pylint: disable=protected-access
return cls.instance()._get(key, cls._ns(app_context))
@classmethod
def get_multi(cls, keys, app_context=None):
# pylint: disable=protected-access
return cls.instance()._get_multi(keys, cls._ns(app_context))
cache_len = counters.PerfCounter(
'gcb-models-%sCacheConnection-cache-len' %
dao_class.ENTITY.__name__, 'Total number of items in the cache')
cache_len.poll_value = EntityCache.get_cache_len
cache_size = counters.PerfCounter(
'gcb-models-%sCacheConnection-cache-bytes' %
dao_class.ENTITY.__name__, 'Total number of bytes in the cache.')
cache_size.poll_value = EntityCache.get_cache_size
CacheConnection.init_counters()
entry = CacheFactoryEntry(EntityCache, CacheEntry, CacheConnection,
ConnectionManager, config_property,
cache_len, cache_size)
cls._CACHES[name] = entry
return entry
mail_errors.InvalidSenderError,
])
_ENQUEUED_BUFFER_MULTIPLIER = 1.5
_KEY_DELIMITER = ':'
_MAX_ENQUEUED_HOURS = 3
_MAX_RETRY_DAYS = 3
# Number of times past which recoverable failure of send_mail() calls becomes
# hard failure. Used as a brake on runaway queues. Should be larger than the
# expected cap on the number of retries imposed by taskqueue.
_RECOVERABLE_FAILURE_CAP = 20
_SECONDS_PER_HOUR = 60 * 60
_SECONDS_PER_DAY = 24 * _SECONDS_PER_HOUR
_USECS_PER_SECOND = 10**6
COUNTER_RETENTION_POLICY_RUN = counters.PerfCounter(
'gcb-notifications-retention-policy-run',
'number of times a retention policy was run')
COUNTER_SEND_ASYNC_FAILED_BAD_ARGUMENTS = counters.PerfCounter(
'gcb-notifications-send-async-failed-bad-arguments',
'number of times send_async failed because arguments were bad')
COUNTER_SEND_ASYNC_FAILED_DATASTORE_ERROR = counters.PerfCounter(
'gcb-notifications-send-async-failed-datastore-error',
'number of times send_async failed because of datastore error')
COUNTER_SEND_ASYNC_START = counters.PerfCounter(
'gcb-notifications-send-async-called',
'number of times send_async has been called')
COUNTER_SEND_ASYNC_SUCCESS = counters.PerfCounter(
'gcb-notifications-send-async-success',
'number of times send_async succeeded')
COUNTER_SEND_MAIL_TASK_FAILED = counters.PerfCounter(
'gcb-notifications-send-mail-task-failed',
