def invoke(data_str, callset_id, call_id, s3_func_key, host_job_meta, s3_agg_data_key=None, data_byte_range=None): s3_data_key, s3_output_key, s3_status_key \ = s3util.create_keys(self.s3_bucket, self.s3_prefix, callset_id, call_id) host_job_meta['job_invoke_timestamp'] = time.time() if s3_agg_data_key is None: data_upload_time = time.time() self.put_data(s3_data_key, data_str, callset_id, call_id) data_upload_time = time.time() - data_upload_time host_job_meta['data_upload_time'] = data_upload_time host_job_meta['data_upload_timestamp'] = time.time() data_key = s3_data_key else: data_key = s3_agg_data_key return self.invoke_with_keys(s3_func_key, data_key, s3_output_key, s3_status_key, callset_id, call_id, extra_env, extra_meta, data_byte_range, use_cached_runtime, host_job_meta.copy())
def get_call_output(callset_id, call_id, AWS_S3_BUCKET=wrenconfig.AWS_S3_BUCKET, AWS_S3_PREFIX=wrenconfig.AWS_S3_PREFIX, AWS_REGION=wrenconfig.AWS_REGION, s3=None): s3_data_key, s3_output_key, s3_status_key = s3util.create_keys( AWS_S3_BUCKET, AWS_S3_PREFIX, callset_id, call_id) if s3 is None: s3 = global_s3_client # boto3.client('s3', region_name = AWS_REGION) r = s3.get_object(Bucket=s3_output_key[0], Key=s3_output_key[1]) return pickle.loads(r['Body'].read())
def get_call_status(callset_id, call_id, AWS_S3_BUCKET=wrenconfig.AWS_S3_BUCKET, AWS_S3_PREFIX=wrenconfig.AWS_S3_PREFIX, AWS_REGION=wrenconfig.AWS_REGION, s3=None): s3_data_key, s3_output_key, s3_status_key = s3util.create_keys( AWS_S3_BUCKET, AWS_S3_PREFIX, callset_id, call_id) if s3 is None: s3 = global_s3_client try: r = s3.get_object(Bucket=s3_status_key[0], Key=s3_status_key[1]) result_json = r['Body'].read() return json.loads(result_json) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "NoSuchKey": return None else: raise e
def invoke(data_str, callset_id, call_id, s3_func_key, s3_agg_data_key=None, data_byte_range=None): s3_data_key, s3_output_key, s3_status_key \ = s3util.create_keys(self.s3_bucket, self.s3_prefix, callset_id, call_id) if s3_agg_data_key is None: self.put_data(s3_data_key, data_str, callset_id, call_id) data_key = s3_data_key else: data_key = s3_agg_data_key return self.invoke_with_keys(s3_func_key, data_key, s3_output_key, s3_status_key, callset_id, call_id, extra_env, extra_meta, data_byte_range, use_cached_runtime)