def list_all_jobs(event, context):
""" list all jobs """
tm = TableManager()
table = tm.get_job_state_table()
try:
result = table.scan()
items = result['Items']
while 'LastEvaluatedKey' in result:
# logger.info(result['LastEvaluatedKey'])
result = table.scan(ExclusiveStartKey=result['LastEvaluatedKey'])
items.extend(result['Items'])
return {
'statusCode': 200,
'headers': __HTTP_HEADERS__,
'body': json.dumps({'jobs': items, 'error': ''})
}
except Exception as ex:
return {
'statusCode': 500,
'headers': __HTTP_HEADERS__,
'body': json.dumps({'jobs': [], 'error': str(ex)})
}
def monitor_jobs(event, context):
"""
monitors the current jobs in the system. It asks the job table for all unfinished jobs
if they are ready for processing
TODO looks like it's only used in tests and nowhere else
"""
logger.info("job monitor triggered from event {}".format(event))
# 1. query JOB state table in state running
tm = TableManager()
table = tm.get_job_state_table()
query_params = {
'IndexName': 'state-index',
'Select': 'ALL_ATTRIBUTES',
'KeyConditionExpression': Key('state').eq(SCHEDULED)
}
result = table.query(**query_params)
if 'Items' in result:
if len(result['Items']) == 0:
logger.info("no jobs in state scheduled!")
query_params = {
'IndexName': 'state-index',
'Select': 'ALL_ATTRIBUTES',
'FilterExpression': Attr('state').ne(SCHEDULED)
}
logger.info(
"WARNING: never good todo a able scan!!! find a better solution"
)
result = table.scan(**query_params)
for x in result['Items']:
try:
if x['state'] in [
FAILED, AGGREGATED_AND_UPLOADED, AGGREGATING_SCHEDULED,
AGGREGATING_SCHEDULING
]:
continue
sync_job(x)
except Exception as e:
traceback.print_exc()
error_diagnostics = traceback.format_exc()
update_job_state(job=x['id'],
state=FAILED,
reason=f"{str(e)} = {error_diagnostics}")
def status(event, context):
"""
returns the status of the current job, as well as some meta information
TODO due to expense it might be better to store the whole calculation result
in an addiitonal table, since it can take a LONG time to execute and so not perfect
as solution for http requests
"""
if 'pathParameters' in event:
parameters = event['pathParameters']
if 'job' in parameters:
job = parameters['job']
tm = TableManager()
table_overall_state = tm.get_job_state_table()
if 'body' in event and event.get("httpMethod", "") != 'GET':
content = json.loads(event['body'])
result = update_job_state(job, content['job_state'], content.get("reason", ""))
return {
"statusCode": 200,
"headers": __HTTP_HEADERS__,
"body": json.dumps({
"job_state": result,
"job_info": result
}
)
}
else:
job_state = table_overall_state.query(
**{
'IndexName': 'job-id-state-index',
'Select': 'ALL_ATTRIBUTES',
'KeyConditionExpression': Key('job').eq(job)
}
)
# this queries the state of all the samples
if "Items" in job_state and len(job_state['Items']) > 0:
job_state = job_state["Items"]
if len(job_state) > 0:
job_state = job_state[0]
return {
"statusCode": 200,
"headers": __HTTP_HEADERS__,
"body": json.dumps({
"job_state": job_state['state'],
"job_info": job_state
}
)
}
else:
return {
"statusCode": 503,
"headers": __HTTP_HEADERS__,
"body": json.dumps({
"job_state": "no associated state found!",
}
)
}
else:
return {
"statusCode": 404,
"headers": __HTTP_HEADERS__,
"body": json.dumps({"error": "no job found with this identifier : {}".format(
event['pathParameters']['job'])})
}
# invalid!
return {
'statusCode': 503
}