def invoke(data_str,
callset_id,
call_id,
s3_func_key,
host_job_meta,
s3_agg_data_key=None,
data_byte_range=None):
s3_data_key, s3_output_key, s3_status_key \
= s3util.create_keys(self.s3_bucket,
self.s3_prefix,
callset_id, call_id)
host_job_meta['job_invoke_timestamp'] = time.time()
if s3_agg_data_key is None:
data_upload_time = time.time()
self.put_data(s3_data_key, data_str, callset_id, call_id)
data_upload_time = time.time() - data_upload_time
host_job_meta['data_upload_time'] = data_upload_time
host_job_meta['data_upload_timestamp'] = time.time()
data_key = s3_data_key
else:
data_key = s3_agg_data_key
return self.invoke_with_keys(s3_func_key, data_key, s3_output_key,
s3_status_key, callset_id, call_id,
extra_env, extra_meta,
data_byte_range, use_cached_runtime,
host_job_meta.copy())
def get_call_output(callset_id,
call_id,
AWS_S3_BUCKET=wrenconfig.AWS_S3_BUCKET,
AWS_S3_PREFIX=wrenconfig.AWS_S3_PREFIX,
AWS_REGION=wrenconfig.AWS_REGION,
s3=None):
s3_data_key, s3_output_key, s3_status_key = s3util.create_keys(
AWS_S3_BUCKET, AWS_S3_PREFIX, callset_id, call_id)
if s3 is None:
s3 = global_s3_client # boto3.client('s3', region_name = AWS_REGION)
r = s3.get_object(Bucket=s3_output_key[0], Key=s3_output_key[1])
return pickle.loads(r['Body'].read())
def get_call_status(callset_id,
call_id,
AWS_S3_BUCKET=wrenconfig.AWS_S3_BUCKET,
AWS_S3_PREFIX=wrenconfig.AWS_S3_PREFIX,
AWS_REGION=wrenconfig.AWS_REGION,
s3=None):
s3_data_key, s3_output_key, s3_status_key = s3util.create_keys(
AWS_S3_BUCKET, AWS_S3_PREFIX, callset_id, call_id)
if s3 is None:
s3 = global_s3_client
try:
r = s3.get_object(Bucket=s3_status_key[0], Key=s3_status_key[1])
result_json = r['Body'].read()
return json.loads(result_json)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "NoSuchKey":
return None
else:
raise e
def invoke(data_str,
callset_id,
call_id,
s3_func_key,
s3_agg_data_key=None,
data_byte_range=None):
s3_data_key, s3_output_key, s3_status_key \
= s3util.create_keys(self.s3_bucket,
self.s3_prefix,
callset_id, call_id)
if s3_agg_data_key is None:
self.put_data(s3_data_key, data_str, callset_id, call_id)
data_key = s3_data_key
else:
data_key = s3_agg_data_key
return self.invoke_with_keys(s3_func_key, data_key, s3_output_key,
s3_status_key, callset_id, call_id,
extra_env, extra_meta,
data_byte_range, use_cached_runtime)