def put_records(stream_name, records):
stream = get_stream(stream_name)
for dest in stream['Destinations']:
if 'ESDestinationDescription' in dest:
es_dest = dest['ESDestinationDescription']
es_index = es_dest['IndexName']
es_type = es_dest['TypeName']
es = connect_elasticsearch()
for record in records:
obj_id = uuid.uuid4()
data = base64.b64decode(record['Data'])
body = json.loads(data)
try:
es.create(index=es_index, doc_type=es_type, id=obj_id, body=body)
except Exception as e:
LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc()))
raise e
if 'S3DestinationDescription' in dest:
s3_dest = dest['S3DestinationDescription']
bucket = bucket_name(s3_dest['BucketARN'])
prefix = s3_dest['Prefix']
s3 = get_s3_client()
for record in records:
data = base64.b64decode(record['Data'])
obj_name = str(uuid.uuid4())
obj_path = '%s%s' % (prefix, obj_name)
try:
s3.Object(bucket, obj_path).put(Body=data)
except Exception as e:
LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc()))
raise e
def map_all_s3_objects(to_json=True):
s3_client = aws_stack.get_s3_client()
result = {}
for bucket in s3_client.buckets.all():
for key in bucket.objects.all():
value = download_s3_object(s3_client, key.bucket_name, key.key)
if to_json:
value = json.loads(value)
result['%s/%s' % (key.bucket_name, key.key)] = value
return result
def map_all_s3_objects(to_json=True):
s3_client = aws_stack.get_s3_client()
result = {}
for bucket in s3_client.buckets.all():
for key in bucket.objects.all():
value = download_s3_object(s3_client, key.bucket_name, key.key)
if to_json:
value = json.loads(value)
result['%s/%s' % (key.bucket_name, key.key)] = value
return result
def put_records(stream_name, records):
stream = get_stream(stream_name)
for dest in stream['Destinations']:
if 'ESDestinationDescription' in dest:
es_dest = dest['ESDestinationDescription']
es_index = es_dest['IndexName']
es_type = es_dest['TypeName']
es = connect_elasticsearch()
for record in records:
obj_id = uuid.uuid4()
# DirectPut
if 'Data' in record:
data = base64.b64decode(record['Data'])
# KinesisAsSource
elif 'data' in record:
data = base64.b64decode(record['data'])
body = json.loads(data)
try:
es.create(index=es_index,
doc_type=es_type,
id=obj_id,
body=body)
except Exception as e:
LOG.error('Unable to put record to stream: %s %s' %
(e, traceback.format_exc()))
raise e
if 'S3DestinationDescription' in dest:
s3_dest = dest['S3DestinationDescription']
bucket = bucket_name(s3_dest['BucketARN'])
prefix = s3_dest.get('Prefix', '')
s3 = get_s3_client()
for record in records:
# DirectPut
if 'Data' in record:
data = base64.b64decode(record['Data'])
# KinesisAsSource
elif 'data' in record:
data = base64.b64decode(record['data'])
obj_name = str(uuid.uuid4())
obj_path = '%s%s%s' % (prefix, '' if prefix.endswith('/') else
'/', obj_name)
try:
s3.Object(bucket, obj_path).put(Body=data)
except Exception as e:
LOG.error('Unable to put record to stream: %s %s' %
(e, traceback.format_exc()))
raise e
def map_all_s3_objects(to_json=True):
s3_client = aws_stack.get_s3_client()
result = {}
for bucket in s3_client.buckets.all():
for key in bucket.objects.all():
value = download_s3_object(s3_client, key.bucket_name, key.key)
try:
if to_json:
value = json.loads(value)
result['%s/%s' % (key.bucket_name, key.key)] = value
except Exception:
# skip non-JSON or binary objects
pass
return result
def map_all_s3_objects(to_json=True, buckets=None):
s3_client = aws_stack.get_s3_client()
result = {}
buckets = [s3_client.Bucket(b) for b in buckets] if buckets else s3_client.buckets.all()
for bucket in buckets:
for key in bucket.objects.all():
value = download_s3_object(s3_client, key.bucket_name, key.key)
try:
if to_json:
value = json.loads(value)
key = '%s%s%s' % (key.bucket_name, '' if key.key.startswith('/') else '/', key.key)
result[key] = value
except Exception:
# skip non-JSON or binary objects
pass
return result
def put_records(stream_name, records):
stream = get_stream(stream_name)
for dest in stream['Destinations']:
if 'S3DestinationDescription' in dest:
s3_dest = dest['S3DestinationDescription']
bucket = bucket_name(s3_dest['BucketARN'])
prefix = s3_dest['Prefix']
s3 = get_s3_client()
for record in records:
data = base64.b64decode(record['Data'])
obj_name = str(uuid.uuid4())
obj_path = '%s%s' % (prefix, obj_name)
try:
s3.Object(bucket, obj_path).put(Body=data)
except Exception as e:
LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc()))
raise e
