Пример #1
0
        def invoke(data_str,
                   callset_id,
                   call_id,
                   s3_func_key,
                   host_job_meta,
                   s3_agg_data_key=None,
                   data_byte_range=None):
            s3_data_key, s3_output_key, s3_status_key \
                = s3util.create_keys(self.s3_bucket,
                                     self.s3_prefix,
                                     callset_id, call_id)

            host_job_meta['job_invoke_timestamp'] = time.time()

            if s3_agg_data_key is None:
                data_upload_time = time.time()
                self.put_data(s3_data_key, data_str, callset_id, call_id)
                data_upload_time = time.time() - data_upload_time
                host_job_meta['data_upload_time'] = data_upload_time
                host_job_meta['data_upload_timestamp'] = time.time()

                data_key = s3_data_key
            else:
                data_key = s3_agg_data_key

            return self.invoke_with_keys(s3_func_key, data_key, s3_output_key,
                                         s3_status_key, callset_id, call_id,
                                         extra_env, extra_meta,
                                         data_byte_range, use_cached_runtime,
                                         host_job_meta.copy())
Пример #2
0
def get_call_output(callset_id,
                    call_id,
                    AWS_S3_BUCKET=wrenconfig.AWS_S3_BUCKET,
                    AWS_S3_PREFIX=wrenconfig.AWS_S3_PREFIX,
                    AWS_REGION=wrenconfig.AWS_REGION,
                    s3=None):
    s3_data_key, s3_output_key, s3_status_key = s3util.create_keys(
        AWS_S3_BUCKET, AWS_S3_PREFIX, callset_id, call_id)

    if s3 is None:
        s3 = global_s3_client  # boto3.client('s3', region_name = AWS_REGION)

    r = s3.get_object(Bucket=s3_output_key[0], Key=s3_output_key[1])
    return pickle.loads(r['Body'].read())
Пример #3
0
def get_call_status(callset_id,
                    call_id,
                    AWS_S3_BUCKET=wrenconfig.AWS_S3_BUCKET,
                    AWS_S3_PREFIX=wrenconfig.AWS_S3_PREFIX,
                    AWS_REGION=wrenconfig.AWS_REGION,
                    s3=None):
    s3_data_key, s3_output_key, s3_status_key = s3util.create_keys(
        AWS_S3_BUCKET, AWS_S3_PREFIX, callset_id, call_id)
    if s3 is None:
        s3 = global_s3_client

    try:
        r = s3.get_object(Bucket=s3_status_key[0], Key=s3_status_key[1])
        result_json = r['Body'].read()
        return json.loads(result_json)

    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == "NoSuchKey":
            return None
        else:
            raise e
Пример #4
0
        def invoke(data_str,
                   callset_id,
                   call_id,
                   s3_func_key,
                   s3_agg_data_key=None,
                   data_byte_range=None):
            s3_data_key, s3_output_key, s3_status_key \
                = s3util.create_keys(self.s3_bucket,
                                     self.s3_prefix,
                                     callset_id, call_id)

            if s3_agg_data_key is None:
                self.put_data(s3_data_key, data_str, callset_id, call_id)
                data_key = s3_data_key
            else:
                data_key = s3_agg_data_key

            return self.invoke_with_keys(s3_func_key, data_key, s3_output_key,
                                         s3_status_key, callset_id, call_id,
                                         extra_env, extra_meta,
                                         data_byte_range, use_cached_runtime)