def schedule_aggregation_to_fargate(param, param1):
assert 'body' in param, "please ensure you have a body set!"
assert 'job' in param[
'body'], "please ensure your body contains the job name"
body = param['body']
try:
job = json.loads(param['body'])['job']
import boto3
overrides = {
"containerOverrides": [{
"name":
"carrot-aggregator",
"environment": [{
"name": "CARROT_JOB",
"value": "{}".format(job)
}]
}]
}
task_name = "{}-{}".format(os.getenv("current_stage"),
SECURE_CARROT_AGGREGATOR)
if 'key' in body and body['key'] is not None:
overrides['containerOverrides'][0]['environment'].append({
"name":
"STASIS_API_TOKEN",
"value":
body['key']
})
if 'url' in body and body['url'] is not None:
overrides['containerOverrides'][0]['environment'].append({
"name":
"STASIS_URL",
"value":
body['url']
})
logger.info('utilizing taskDefinition: {}'.format(task_name))
logger.info(overrides)
logger.info("")
response = send_to_fargate(overrides, task_name)
logger.info(f'Response: {response}')
return {
'statusCode': 200,
'isBase64Encoded': False,
'headers': __HTTP_HEADERS__
}
except Exception as e:
update_job_state(job=body['job'],
state=FAILED,
reason="job failed to aggregate due to %".format(
str(e)))
raise e
def monitor_jobs(event, context):
"""
monitors the current jobs in the system. It asks the job table for all unfinished jobs
if they are ready for processing
TODO looks like it's only used in tests and nowhere else
"""
logger.info("job monitor triggered from event {}".format(event))
# 1. query JOB state table in state running
tm = TableManager()
table = tm.get_job_state_table()
query_params = {
'IndexName': 'state-index',
'Select': 'ALL_ATTRIBUTES',
'KeyConditionExpression': Key('state').eq(SCHEDULED)
}
result = table.query(**query_params)
if 'Items' in result:
if len(result['Items']) == 0:
logger.info("no jobs in state scheduled!")
query_params = {
'IndexName': 'state-index',
'Select': 'ALL_ATTRIBUTES',
'FilterExpression': Attr('state').ne(SCHEDULED)
}
logger.info(
"WARNING: never good todo a able scan!!! find a better solution"
)
result = table.scan(**query_params)
for x in result['Items']:
try:
if x['state'] in [
FAILED, AGGREGATED_AND_UPLOADED, AGGREGATING_SCHEDULED,
AGGREGATING_SCHEDULING
]:
continue
sync_job(x)
except Exception as e:
traceback.print_exc()
error_diagnostics = traceback.format_exc()
update_job_state(job=x['id'],
state=FAILED,
reason=f"{str(e)} = {error_diagnostics}")
def bucket_zip(event, context):
"""
triggers the job api on uploaded files
:param event:
:param context:
:return:
"""
logger.info("received upload request")
for record in event['Records']:
o = record['s3']['object']
k = str(o['key'])
logger.info("received key {}".format(k))
if k.endswith(".zip"):
job = k.replace(".zip", "")
logger.info(f"belongs to job {job}")
result = update_job_state(job=job,
state=AGGREGATED_AND_UPLOADED,
reason=f"client uploaded file {k}")
if result is None:
logger.info("we were not able to update the job")
else:
logger.info("job state was set to: {}".format(result))
else:
logger.info("received wrong key type, ignored!")
def sync_job(job: dict):
logger.info("synchronizing job: {}".format(job))
if job is None:
result = "warning a none job was provided, we ignore this one!"
else:
state = calculate_job_state(job['id'])
if 'resource' in job:
resource = Backend(job['resource'])
else:
resource = DEFAULT_PROCESSING_BACKEND
if state == EXPORTED:
result = "schedule aggregation for job {}, due to state being {}".format(job['id'], state)
update_job_state(job=job['id'], state=AGGREGATING_SCHEDULING, reason="synchronization triggered")
schedule_to_queue({"job": job['id'], "profile": job['profile']},
service=SECURE_CARROT_AGGREGATOR,
resource=resource)
update_job_state(job=job['id'], state=AGGREGATING_SCHEDULED,
reason="synchronization was triggered and completed")
else:
result = f"state {state} for job {job['id']} did not justify triggering an aggregation."
return result
def send_success_notifications(config, event, object_id):
"""
sends success notifications to all users who are supposed to receive status updates on this particular job changes
"""
details = event.get('detail', {})
object_id = details.get('id', 'unknown')
destinations = load_notifications(object_id)
for destination in destinations:
try:
send_email(
"LC BinBase", config.email_sender(),
f"LC BinBase your job {object_id} is ready for download now!",
json.dumps(event, indent=4), destination)
except Exception as e:
logging.error(e)
logging.error(
f"Could not send email to {destination}! This not fatal")
update_job_state(
job=object_id,
state=NOTIFICATIONS,
reason=
f"all {len(destinations)} subscribers for this job {object_id} have been notified"
)
def status(event, context):
"""
returns the status of the current job, as well as some meta information
TODO due to expense it might be better to store the whole calculation result
in an addiitonal table, since it can take a LONG time to execute and so not perfect
as solution for http requests
"""
if 'pathParameters' in event:
parameters = event['pathParameters']
if 'job' in parameters:
job = parameters['job']
tm = TableManager()
table_overall_state = tm.get_job_state_table()
if 'body' in event and event.get("httpMethod", "") != 'GET':
content = json.loads(event['body'])
result = update_job_state(job, content['job_state'], content.get("reason", ""))
return {
"statusCode": 200,
"headers": __HTTP_HEADERS__,
"body": json.dumps({
"job_state": result,
"job_info": result
}
)
}
else:
job_state = table_overall_state.query(
**{
'IndexName': 'job-id-state-index',
'Select': 'ALL_ATTRIBUTES',
'KeyConditionExpression': Key('job').eq(job)
}
)
# this queries the state of all the samples
if "Items" in job_state and len(job_state['Items']) > 0:
job_state = job_state["Items"]
if len(job_state) > 0:
job_state = job_state[0]
return {
"statusCode": 200,
"headers": __HTTP_HEADERS__,
"body": json.dumps({
"job_state": job_state['state'],
"job_info": job_state
}
)
}
else:
return {
"statusCode": 503,
"headers": __HTTP_HEADERS__,
"body": json.dumps({
"job_state": "no associated state found!",
}
)
}
else:
return {
"statusCode": 404,
"headers": __HTTP_HEADERS__,
"body": json.dumps({"error": "no job found with this identifier : {}".format(
event['pathParameters']['job'])})
}
# invalid!
return {
'statusCode': 503
}
def calculate_job_state(job: str) -> Optional[str]:
"""
this method keeps the stasis tracking table and the job tracking in sync.
"""
# 1. evaluate existing job state
# to avoid expensive synchronization
state = get_job_state(job=job)
s = States()
logger.info("current job state for {} is {}".format(job, state))
if state is None:
logger.info(f"no job state found -> forcing scheduled state for {job}")
update_job_state(job=job, state=SCHEDULED,
reason="job was forced to state scheduled due to no state being available!")
elif s.priority(state) >= s.priority(AGGREGATING_SCHEDULING):
logger.info(f"job was already in a finished state {job}, state {state} and so needs no further analysis")
return state
else:
logger.info(f"job {job} was in state {state}, which requires it to get it's final state analyzed")
# 2. load job definition
# loading all the samples here still causes timeouts or excessive CPU cost todo find a solution
job_definition = load_job_samples_with_states(job=job)
job_config = get_job_config(job=job)
if job_definition is not None and job_config is not None and job_config['state'] != REGISTERING:
states = []
try:
# 3. go over all samples
for sample, tracking_state in job_definition.items():
states.append(tracking_state)
logger.info("received sample states for job are: {}".format(states))
if len(states) == 0:
# bigger issue nothing found to synchronize
logger.info("no states found!")
return None
# ALL ARE FAILED
elif states.count(FAILED) == len(states):
update_job_state(job=job_config['id'], state=FAILED,
reason="job is in state failed, due to all samples being in state failed")
logger.info("job is failed, no sample was successful")
return FAILED
# ALL ARE EXPORTED OR FAILED
elif states.count(EXPORTED) + states.count(UPLOADED) + states.count(EXPORTING) + states.count(FAILED) == len(states):
update_job_state(job=job_config['id'], state=EXPORTED,
reason="job state was set to exported due to all samples having been exported or failed")
logger.info("job should now be exported")
return EXPORTED
# ANY ARE SCHEDULED
elif states.count(SCHEDULED) == len(states):
update_job_state(job=job_config['id'], state=SCHEDULED,
reason="job is in state scheduled, due to all samples being in state scheduled")
logger.info("job still in state scheduled")
return SCHEDULED
# otherwise we must be processing
else:
update_job_state(job=job_config['id'], state=PROCESSING, reason="job is in state processing")
logger.info("job is in state processing right now")
from collections import Counter
logger.info(Counter(states))
return PROCESSING
finally:
from collections import Counter
logger.info("state distribution for job '{}' with {} samples is: {}".format(job, len(states), Counter(states)))
counter = 0
for sample, tracking_state in job_definition.items():
logger.info(f"{counter} - sample {sample} is in state {tracking_state} ")
counter = counter + 1
logger.info("done")
else:
raise Exception("we did not find a job definition for {}, Please investigate".format(job))