def process_activity_loggers():
"""Processes the pending activities from the local request environ."""
if not ACTIVITIES_ENVIRON_KEY in os.environ:
return
processors = {}
deferred_activity_loggers = []
for logger in os.environ[ACTIVITIES_ENVIRON_KEY]:
# Discover all the processors.
processors.update(getattr(logger, 'processors', {}))
for logger in os.environ[ACTIVITIES_ENVIRON_KEY]:
if logger.defer_finalize:
deferred_activity_loggers.append(logger)
# Perform processing.
if hasattr(logger, 'processors'):
logger.process(processors)
if deferred_activity_loggers:
deferred.defer(_finalize_activity_loggers,
deferred_activity_loggers,
_queue=ACTIVITY_QUEUE)
for processor in processors.itervalues():
pipelines.push(processor.batch_name,
processor.batch_processor,
data=processor.serialize())
def testPushBadFinalize(self):
# Weakly test that it executes completely.
processor = BadProcessorFinalize()
data = {'test': True}
with self.assertRaises(pipelines.PipelineProcessError):
pipelines.push('mock_pipeline', processor=processor, data=data)
self.RunDeferredTasks(queue_name=pipelines.PIPELINES_QUEUE)
def process_activity_loggers():
"""Processes the pending activities from the local request environ."""
if not ACTIVITIES_ENVIRON_KEY in os.environ:
return
processors = {}
deferred_activity_loggers = []
for logger in os.environ[ACTIVITIES_ENVIRON_KEY]:
# Discover all the processors.
processors.update(getattr(logger, 'processors', {}))
for logger in os.environ[ACTIVITIES_ENVIRON_KEY]:
if logger.defer_finalize:
deferred_activity_loggers.append(logger)
# Perform processing.
if hasattr(logger, 'processors'):
logger.process(processors)
if deferred_activity_loggers:
deferred.defer(_finalize_activity_loggers, deferred_activity_loggers,
_queue=ACTIVITY_QUEUE)
for processor in processors.itervalues():
pipelines.push(
processor.batch_name, processor.batch_processor,
data=processor.serialize())
def testPipelineCleanup(self, internal_cleanup):
processor = GoodProcessor()
data = {'test': True}
# Push twice to test that it cleans up multiple work items.
pipelines.push('mock_pipeline', processor=processor, data=data)
pipelines.push('mock_pipeline', processor=processor, data=data)
self.RunDeferredTasks(queue_name=pipelines.PIPELINES_QUEUE)
# Check that the cleanup was called with one argument.
internal_cleanup.assert_called_with(
[ndb.Key('WorkItem', 1),
ndb.Key('WorkItem', 2)])
def testPushGood(self):
# Weakly test that it executes completely.
processor = GoodProcessor()
data = {'test': True}
pipelines.push('mock_pipeline', processor=processor, data=data)
self.RunDeferredTasks(queue_name=pipelines.PIPELINES_QUEUE)