def handle_noargs(self, **options):
eight_hours_ago = datetime.now() - timedelta(hours=8)
failed_tasks = TaskState.objects.filter(state__in=['FAILURE', 'STARTED', 'RETRY'],
tstamp__gt=eight_hours_ago,
name__in=['tasks.etl_to_solr',
'tasks.priority_etl_to_solr'])
for task in failed_tasks:
task_etl_to_solr.delay(*ast.literal_eval(task.args))
print "Requeuing task with args %s" % task.args
def confirm_load_jobs_from_etl(response):
"""
Called when a job source is ready to be imported. Calls celery update
tasks.
"""
blocked_jsids=('4051c882-fa2c-4c93-9db5-91c9add39def',
'10f89212-654e-4ee5-8655-2bbb4770252f',
'e6fa8adb-07c9-4b15-87ad-5816c3968d43',
'8667bd35-c6d3-4008-8b62-36d6b6e3bb62',
'c94ddf73-23c8-4b6c-ad5a-a98b5f2a04b0',
'5de582a0-cab5-45f8-88a8-a31f7fe03025',
'72690d11-b3fc-403c-8726-c80883e27774',
'e0fe5671-c591-40d4-bba9-f0d8542882e2',
'dd5fd646-655b-4867-8784-700bb5c6315b',
'c4d56d17-7b35-436a-8871-80d8c2e37bf9',
'ff794484-8f24-4bdc-96e3-227c9dec2c26',
'cd7fc92a-9bff-4bbd-a10a-ffd6a57a93a1',
'3388e1e9-8292-4d6f-a5ee-c4ed2bae59a4',
'868672a0-c22b-4337-9dcc-2fa5b6671592',
'09ae740c-36d8-432d-bc84-de538dbac8fd',
'03516574-c452-45ab-b217-8ea0357be747',
'1b0e4f3b-a9e1-40b9-8c8b-85a65882c2a3',
'de0762b3-698b-4a3b-92d2-388144edb15a',
'77e1b0eb-4017-44b7-ab9d-898d35390b81',
'905ad700-0a73-4da1-8bf5-b12bf6ba89a7',
'f39aaaf4-e126-4d53-bdf3-98831f45d731',
'1f78a1c6-1ced-4d80-b338-1c3bd8ac57a7',
'4d9330b3-8ca8-41ef-8ca2-305da6ddc5f0',
'aeb6cdb4-1b02-4bab-b398-4d3980097659',
'c20f2c86-bd08-4cce-af94-b3339944676e',
'249308c5-623b-41b9-9364-2589e49b5e02',
'27f0d51d-2882-4168-bfca-cb415f666fb3',
'8d506b65-f911-449e-bad8-c308a196e1c0',
'c6203550-2435-4137-9c11-b0710f3ef4cf',
'682deefb-fde9-4de2-8985-13371d04a8ff',
'c8bfb1b1-398a-46a4-a1f4-fbdb5354ee78',
'769ca60a-f4e7-446d-b2ba-b66a5a3e9313',
'7ecfcde8-b7e1-4bb1-a671-b52d729903b5',
'817d2b90-9299-4ef6-b484-46208ce69e19',
'00152da3-1abe-458e-8895-121ca9008cf7',
'94d95b97-24e4-4d98-8ecf-1dce6202c523',
'ccc31e40-a65f-46b0-a194-b0517c33a7f6',
'be4dcd74-ff51-4f99-8057-55a876b3ce56',
'15079de2-7de2-4191-b8cf-7924036b4b97',
'c8d8da8c-542f-4620-b90b-4a37d55d659f',
'66cbd5e6-c86b-4659-b80c-11aa5a5fa6a7',
'265357bd-a619-40b7-b9bc-9674d6e96400',
'7d6ea31f-e36d-43e7-b68e-d9dbd45446f7',
'b3c58f53-144a-4de7-807b-8fe140259d7f',
'23322abe-6faf-4303-b08b-713e5127e019',
'bedee5c5-a9ca-459f-899b-29482712d7c9',
'69f485d0-40d5-430f-a5eb-6221ee14092d',
'b0c01590-0085-4ab5-b0b0-27149fa0fb4a',
'536dcdbb-2a88-40d8-bf68-8630306c2818',
'ad875783-c49e-49ff-b82a-b0538026e089',
'0ab41358-8323-4863-9f19-fdb344a75a35',)
logger.info("sns received for ETL")
if response:
if response.get('Subject', None)!='END':
msg=json.loads(response['Message'])
jsid=msg['jsid']
buid=msg['buid']
name=msg['name']
prio=msg["priority"]
if jsid.lower() in blocked_jsids:
logger.info("Ignoring sns for %s", jsid)
return None
logger.info("Creating ETL Task (%s, %s, %s)"%(jsid, buid, name))
if int(prio)==1:
task_priority_etl_to_solr.delay(jsid, buid, name)
else:
task_etl_to_solr.delay(jsid, buid, name)
task_jobsfs_to_mongo.delay(jsid, buid, name)
def confirm_load_jobs_from_etl(response):
"""
Called when a job source is ready to be imported. Calls celery update
tasks.
"""
blocked_jsids = (
'4051c882-fa2c-4c93-9db5-91c9add39def',
'10f89212-654e-4ee5-8655-2bbb4770252f',
'e6fa8adb-07c9-4b15-87ad-5816c3968d43',
'8667bd35-c6d3-4008-8b62-36d6b6e3bb62',
'c94ddf73-23c8-4b6c-ad5a-a98b5f2a04b0',
'5de582a0-cab5-45f8-88a8-a31f7fe03025',
'72690d11-b3fc-403c-8726-c80883e27774',
'e0fe5671-c591-40d4-bba9-f0d8542882e2',
'dd5fd646-655b-4867-8784-700bb5c6315b',
'c4d56d17-7b35-436a-8871-80d8c2e37bf9',
'ff794484-8f24-4bdc-96e3-227c9dec2c26',
'cd7fc92a-9bff-4bbd-a10a-ffd6a57a93a1',
'3388e1e9-8292-4d6f-a5ee-c4ed2bae59a4',
'868672a0-c22b-4337-9dcc-2fa5b6671592',
'09ae740c-36d8-432d-bc84-de538dbac8fd',
'03516574-c452-45ab-b217-8ea0357be747',
'1b0e4f3b-a9e1-40b9-8c8b-85a65882c2a3',
'de0762b3-698b-4a3b-92d2-388144edb15a',
'77e1b0eb-4017-44b7-ab9d-898d35390b81',
'905ad700-0a73-4da1-8bf5-b12bf6ba89a7',
'f39aaaf4-e126-4d53-bdf3-98831f45d731',
'1f78a1c6-1ced-4d80-b338-1c3bd8ac57a7',
'4d9330b3-8ca8-41ef-8ca2-305da6ddc5f0',
'aeb6cdb4-1b02-4bab-b398-4d3980097659',
'c20f2c86-bd08-4cce-af94-b3339944676e',
'249308c5-623b-41b9-9364-2589e49b5e02',
'27f0d51d-2882-4168-bfca-cb415f666fb3',
'8d506b65-f911-449e-bad8-c308a196e1c0',
'c6203550-2435-4137-9c11-b0710f3ef4cf',
'682deefb-fde9-4de2-8985-13371d04a8ff',
'c8bfb1b1-398a-46a4-a1f4-fbdb5354ee78',
'769ca60a-f4e7-446d-b2ba-b66a5a3e9313',
'7ecfcde8-b7e1-4bb1-a671-b52d729903b5',
'817d2b90-9299-4ef6-b484-46208ce69e19',
'00152da3-1abe-458e-8895-121ca9008cf7',
'94d95b97-24e4-4d98-8ecf-1dce6202c523',
'ccc31e40-a65f-46b0-a194-b0517c33a7f6',
'be4dcd74-ff51-4f99-8057-55a876b3ce56',
'15079de2-7de2-4191-b8cf-7924036b4b97',
'c8d8da8c-542f-4620-b90b-4a37d55d659f',
'66cbd5e6-c86b-4659-b80c-11aa5a5fa6a7',
'265357bd-a619-40b7-b9bc-9674d6e96400',
'7d6ea31f-e36d-43e7-b68e-d9dbd45446f7',
'b3c58f53-144a-4de7-807b-8fe140259d7f',
'23322abe-6faf-4303-b08b-713e5127e019',
'bedee5c5-a9ca-459f-899b-29482712d7c9',
'69f485d0-40d5-430f-a5eb-6221ee14092d',
'b0c01590-0085-4ab5-b0b0-27149fa0fb4a',
'536dcdbb-2a88-40d8-bf68-8630306c2818',
'ad875783-c49e-49ff-b82a-b0538026e089',
'0ab41358-8323-4863-9f19-fdb344a75a35',
)
logger.info("sns received for ETL")
if response:
if response.get('Subject', None) != 'END':
msg = json.loads(response['Message'])
jsid = msg['jsid']
buid = msg['buid']
name = msg['name']
prio = msg["priority"]
if jsid.lower() in blocked_jsids:
logger.info("Ignoring sns for %s", jsid)
return None
logger.info("Creating ETL Task (%s, %s, %s)" % (jsid, buid, name))
if int(prio) == 1:
task_priority_etl_to_solr.delay(jsid, buid, name)
else:
task_etl_to_solr.delay(jsid, buid, name)
task_jobsfs_to_mongo.delay(jsid, buid, name)
