def test_get_registers_appropriate_tasks(self):
moxer = Mox()
request, response = new_mock_request_response(moxer)
_stub_taskqueue(moxer)
moxer.StubOutWithMock(Feed, "find_active", use_mock_anything=True)
def create_call(i):
source_name = "source-%i" % i
source = MockEntity(key_name=source_name, name=source_name)
return MockEntity(key_name="feed-%i" % i,
artifact_source=source,
url="hi")
q_range = xrange(0, 5)
Feed.find_active().AndReturn(
MockQuery(q_range, create_call=create_call))
# expects queued tasks for each feed
for i in q_range:
taskqueue.add(name=IgnoreArg(), url=IgnoreArg())
moxer.ReplayAll()
handler = CronIngestDriverHandler()
handler.initialize(request, response)
handler.get()
moxer.VerifyAll()
def test_get_registers_appropriate_tasks(self):
moxer = Mox()
request, response = new_mock_request_response(moxer)
_stub_taskqueue(moxer)
moxer.StubOutWithMock(Feed, "find_active", use_mock_anything=True)
def create_call(i):
source_name = "source-%i" % i
source = MockEntity(key_name=source_name, name=source_name)
return MockEntity(key_name="feed-%i" % i, artifact_source=source, url="hi")
q_range = xrange(0,5)
Feed.find_active().AndReturn(MockQuery(q_range, create_call=create_call))
# expects queued tasks for each feed
for i in q_range:
taskqueue.add(name=IgnoreArg(), url=IgnoreArg())
moxer.ReplayAll()
handler = CronIngestDriverHandler()
handler.initialize(request, response)
handler.get()
moxer.VerifyAll()
def get(self):
""" enqueues all active Feeds for ingest """
# find all active feeds
q = Feed.find_active()
if not q.count():
logging.warn("no active feeds found")
return
# generates unique is for task names
ingest_id = dates.timestamp(separator="")
for f in q.fetch(1000):
try:
# schedule tasks to ingest by source name
source_name = f.artifact_source.name
# replace invalid chars from source name
normalized_source_name = self._source_sanitize_regex.sub("-", source_name)
task_name = "ingest-%s-%s" % (normalized_source_name, ingest_id)
taskqueue.add(name=task_name, url="/cron/ingest/%s" % source_name)
logging.debug("queued ingest task %s for source '%s' (feed %s)" % (task_name, source_name, f.url))
except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError):
logging.warn(traceback.format_exc())
