def run(self, job_keys):
project = get_shub_project()
jobs = [project.job(x) for x in job_keys]
unfinished = [x for x in jobs if x.info['state'] != 'finished']
logging.info("Waiting for %s unfinished spiders", len(unfinished))
if unfinished:
# Try again in 30 seconds
with pipeline.InOrder():
yield common.Delay(seconds=30)
yield WaitForJobs(job_keys)
else:
yield common.Return(True)
def run(self, email_address, entity_kind, property_name, *value_list):
yield common.Log.info('UselessPipeline.coolness = %s',
(yield UselessPipeline()).coolness)
split_counts = yield SplitCount(entity_kind, property_name,
*value_list)
yield common.Log.info('SplitCount result = %s', split_counts)
with pipeline.After(split_counts):
with pipeline.InOrder():
yield common.Delay(seconds=1)
yield common.Log.info('Done waiting')
yield EmailCountReport(email_address, split_counts)
def testDelay(self):
self.assertRaises(TypeError, common.Delay, 1234)
self.assertRaises(TypeError, common.Delay, stuff=1234)
self.assertEquals(
5,
self.run_pipeline(common.Delay(seconds=5)).default.value)