def __init__(self, group, stream, pos=0, sleep_on_no_data=0):
self._client = get_authenticated_boto3_client('logs')
self._group = group
self._stream = stream
self._pos = pos
self._sleep_on_no_data = sleep_on_no_data
self._buf = deque()
self._token = None
def get_state_machine_arn(self, name):
if AWS_SANDBOX_ENABLED:
# We can't execute list_state_machines within the sandbox,
# but we can construct the statemachine arn since we have
# explicit access to the region.
account_id = get_authenticated_boto3_client('sts') \
.get_caller_identity().get('Account')
region = AWS_SANDBOX_REGION
return 'arn:aws:states:%s:%s:stateMachine:%s' \
% (region, account_id, name)
else:
state_machine = self.search(name)
if state_machine:
return state_machine['stateMachineArn']
return None
def worker(queue, mode):
try:
from metaflow.metaflow_config import get_authenticated_boto3_client
s3 = get_authenticated_boto3_client('s3')
while True:
url = queue.get()
if url is None:
break
if mode == 'download':
tmp = NamedTemporaryFile(dir='.', delete=False)
try:
s3.download_file(url.bucket, url.path, tmp.name)
os.rename(tmp.name, url.local)
except:
# TODO specific error message for out of disk space
os.unlink(tmp.name)
raise
else:
s3.upload_file(url.local, url.bucket, url.path)
except:
traceback.print_exc()
sys.exit(ERROR_WORKER_EXCEPTION)
def __init__(self):
self._client = get_authenticated_boto3_client('batch')
def __init__(self):
self._client = get_authenticated_boto3_client('stepfunctions')
def get_s3_client():
return get_authenticated_boto3_client('s3', {
'endpoint_url': S3_ENDPOINT_URL,
'verify': S3_VERIFY_CERTIFICATE
}), ClientError
def get_s3_client():
return get_authenticated_boto3_client('s3'), ClientError
def __init__(self):
self._client = get_authenticated_boto3_client(
'dynamodb', params={'region_name': self._get_instance_region()})
self.name = SFN_DYNAMO_DB_TABLE
def __init__(self, name):
self._client = get_authenticated_boto3_client('events')
self.name = name
def reset_client(self, hard_reset=False):
from metaflow.metaflow_config import get_authenticated_boto3_client
if hard_reset or self.s3 is None:
self.s3 = get_authenticated_boto3_client('s3')
