def __init__(self, group, stream, pos=0, sleep_on_no_data=0): self._client = get_authenticated_boto3_client('logs') self._group = group self._stream = stream self._pos = pos self._sleep_on_no_data = sleep_on_no_data self._buf = deque() self._token = None
def get_state_machine_arn(self, name): if AWS_SANDBOX_ENABLED: # We can't execute list_state_machines within the sandbox, # but we can construct the statemachine arn since we have # explicit access to the region. account_id = get_authenticated_boto3_client('sts') \ .get_caller_identity().get('Account') region = AWS_SANDBOX_REGION return 'arn:aws:states:%s:%s:stateMachine:%s' \ % (region, account_id, name) else: state_machine = self.search(name) if state_machine: return state_machine['stateMachineArn'] return None
def worker(queue, mode): try: from metaflow.metaflow_config import get_authenticated_boto3_client s3 = get_authenticated_boto3_client('s3') while True: url = queue.get() if url is None: break if mode == 'download': tmp = NamedTemporaryFile(dir='.', delete=False) try: s3.download_file(url.bucket, url.path, tmp.name) os.rename(tmp.name, url.local) except: # TODO specific error message for out of disk space os.unlink(tmp.name) raise else: s3.upload_file(url.local, url.bucket, url.path) except: traceback.print_exc() sys.exit(ERROR_WORKER_EXCEPTION)
def __init__(self): self._client = get_authenticated_boto3_client('batch')
def __init__(self): self._client = get_authenticated_boto3_client('stepfunctions')
def get_s3_client(): return get_authenticated_boto3_client('s3', { 'endpoint_url': S3_ENDPOINT_URL, 'verify': S3_VERIFY_CERTIFICATE }), ClientError
def get_s3_client(): return get_authenticated_boto3_client('s3'), ClientError
def __init__(self): self._client = get_authenticated_boto3_client( 'dynamodb', params={'region_name': self._get_instance_region()}) self.name = SFN_DYNAMO_DB_TABLE
def __init__(self, name): self._client = get_authenticated_boto3_client('events') self.name = name
def reset_client(self, hard_reset=False): from metaflow.metaflow_config import get_authenticated_boto3_client if hard_reset or self.s3 is None: self.s3 = get_authenticated_boto3_client('s3')