def test_get_registers_appropriate_tasks(self): moxer = Mox() request, response = new_mock_request_response(moxer) _stub_taskqueue(moxer) moxer.StubOutWithMock(Feed, "find_active", use_mock_anything=True) def create_call(i): source_name = "source-%i" % i source = MockEntity(key_name=source_name, name=source_name) return MockEntity(key_name="feed-%i" % i, artifact_source=source, url="hi") q_range = xrange(0, 5) Feed.find_active().AndReturn( MockQuery(q_range, create_call=create_call)) # expects queued tasks for each feed for i in q_range: taskqueue.add(name=IgnoreArg(), url=IgnoreArg()) moxer.ReplayAll() handler = CronIngestDriverHandler() handler.initialize(request, response) handler.get() moxer.VerifyAll()
def test_get_registers_appropriate_tasks(self): moxer = Mox() request, response = new_mock_request_response(moxer) _stub_taskqueue(moxer) moxer.StubOutWithMock(Feed, "find_active", use_mock_anything=True) def create_call(i): source_name = "source-%i" % i source = MockEntity(key_name=source_name, name=source_name) return MockEntity(key_name="feed-%i" % i, artifact_source=source, url="hi") q_range = xrange(0,5) Feed.find_active().AndReturn(MockQuery(q_range, create_call=create_call)) # expects queued tasks for each feed for i in q_range: taskqueue.add(name=IgnoreArg(), url=IgnoreArg()) moxer.ReplayAll() handler = CronIngestDriverHandler() handler.initialize(request, response) handler.get() moxer.VerifyAll()
def get(self): """ enqueues all active Feeds for ingest """ # find all active feeds q = Feed.find_active() if not q.count(): logging.warn("no active feeds found") return # generates unique is for task names ingest_id = dates.timestamp(separator="") for f in q.fetch(1000): try: # schedule tasks to ingest by source name source_name = f.artifact_source.name # replace invalid chars from source name normalized_source_name = self._source_sanitize_regex.sub("-", source_name) task_name = "ingest-%s-%s" % (normalized_source_name, ingest_id) taskqueue.add(name=task_name, url="/cron/ingest/%s" % source_name) logging.debug("queued ingest task %s for source '%s' (feed %s)" % (task_name, source_name, f.url)) except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError): logging.warn(traceback.format_exc())