def put_records(stream_name, records): stream = get_stream(stream_name) for dest in stream['Destinations']: if 'ESDestinationDescription' in dest: es_dest = dest['ESDestinationDescription'] es_index = es_dest['IndexName'] es_type = es_dest['TypeName'] es = connect_elasticsearch() for record in records: obj_id = uuid.uuid4() data = base64.b64decode(record['Data']) body = json.loads(data) try: es.create(index=es_index, doc_type=es_type, id=obj_id, body=body) except Exception as e: LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc())) raise e if 'S3DestinationDescription' in dest: s3_dest = dest['S3DestinationDescription'] bucket = bucket_name(s3_dest['BucketARN']) prefix = s3_dest['Prefix'] s3 = get_s3_client() for record in records: data = base64.b64decode(record['Data']) obj_name = str(uuid.uuid4()) obj_path = '%s%s' % (prefix, obj_name) try: s3.Object(bucket, obj_path).put(Body=data) except Exception as e: LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc())) raise e
def map_all_s3_objects(to_json=True): s3_client = aws_stack.get_s3_client() result = {} for bucket in s3_client.buckets.all(): for key in bucket.objects.all(): value = download_s3_object(s3_client, key.bucket_name, key.key) if to_json: value = json.loads(value) result['%s/%s' % (key.bucket_name, key.key)] = value return result
def put_records(stream_name, records): stream = get_stream(stream_name) for dest in stream['Destinations']: if 'ESDestinationDescription' in dest: es_dest = dest['ESDestinationDescription'] es_index = es_dest['IndexName'] es_type = es_dest['TypeName'] es = connect_elasticsearch() for record in records: obj_id = uuid.uuid4() # DirectPut if 'Data' in record: data = base64.b64decode(record['Data']) # KinesisAsSource elif 'data' in record: data = base64.b64decode(record['data']) body = json.loads(data) try: es.create(index=es_index, doc_type=es_type, id=obj_id, body=body) except Exception as e: LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc())) raise e if 'S3DestinationDescription' in dest: s3_dest = dest['S3DestinationDescription'] bucket = bucket_name(s3_dest['BucketARN']) prefix = s3_dest.get('Prefix', '') s3 = get_s3_client() for record in records: # DirectPut if 'Data' in record: data = base64.b64decode(record['Data']) # KinesisAsSource elif 'data' in record: data = base64.b64decode(record['data']) obj_name = str(uuid.uuid4()) obj_path = '%s%s%s' % (prefix, '' if prefix.endswith('/') else '/', obj_name) try: s3.Object(bucket, obj_path).put(Body=data) except Exception as e: LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc())) raise e
def map_all_s3_objects(to_json=True): s3_client = aws_stack.get_s3_client() result = {} for bucket in s3_client.buckets.all(): for key in bucket.objects.all(): value = download_s3_object(s3_client, key.bucket_name, key.key) try: if to_json: value = json.loads(value) result['%s/%s' % (key.bucket_name, key.key)] = value except Exception: # skip non-JSON or binary objects pass return result
def map_all_s3_objects(to_json=True, buckets=None): s3_client = aws_stack.get_s3_client() result = {} buckets = [s3_client.Bucket(b) for b in buckets] if buckets else s3_client.buckets.all() for bucket in buckets: for key in bucket.objects.all(): value = download_s3_object(s3_client, key.bucket_name, key.key) try: if to_json: value = json.loads(value) key = '%s%s%s' % (key.bucket_name, '' if key.key.startswith('/') else '/', key.key) result[key] = value except Exception: # skip non-JSON or binary objects pass return result
def put_records(stream_name, records): stream = get_stream(stream_name) for dest in stream['Destinations']: if 'S3DestinationDescription' in dest: s3_dest = dest['S3DestinationDescription'] bucket = bucket_name(s3_dest['BucketARN']) prefix = s3_dest['Prefix'] s3 = get_s3_client() for record in records: data = base64.b64decode(record['Data']) obj_name = str(uuid.uuid4()) obj_path = '%s%s' % (prefix, obj_name) try: s3.Object(bucket, obj_path).put(Body=data) except Exception as e: LOG.error('Unable to put record to stream: %s %s' % (e, traceback.format_exc())) raise e