def getcli(): ''' Quick and dirty utility to all quick command line query to ES Usage: python -m esutils getcli <an_id> ''' import sys Configs = configutils.load_configs() objectId=sys.argv[2] result = getById(objectId, Configs['esEndpoint']) print json.dumps(result, indent=4) exit(0)
def setUp(self): import configutils self.configs = configutils.load_configs() self.bucket = self.configs['bucket'] self.key = 'data/unittest-a1234-15-imager_1234567890.tif' self.body_expected = {'bodykey1': 'value1', 'bodykey2': 'value2'} self.head_expected = { 'LastModified': 'DUMMY LASTMODIFIED', 'ContentLength': 115 } self.s3meta_expected = { 's3meta1': 'metaValue1', 's3meta2': 'metaValue2' }
def getcli(): ''' Quick and dirty utility to all quick command line query to ES Usage: python -m esutils getcli <an_id> ''' import sys Configs = configutils.load_configs() objectId = sys.argv[2] result = getById(objectId, Configs['esEndpoint']) print json.dumps(result, indent=4) exit(0)
def setUp(self): import configutils self.configs = configutils.load_configs() self.bucket = self.configs['bucket'] self.key = 'data/unittest-a1234-15-imager_1234567890.tif' self.body_expected = { 'bodykey1': 'value1', 'bodykey2': 'value2' } self.head_expected = { 'LastModified': 'DUMMY LASTMODIFIED', 'ContentLength': 115 } self.s3meta_expected = { 's3meta1': 'metaValue1', 's3meta2': 'metaValue2' }
def event_handler(event, context): logging.info('Received s3 object event... ') logging.debug('Event = ' + json.dumps(event, indent=4)) configs = configutils.load_configs() attributes = metadata.get_attributes(event, configs) if attributes is None: logging.error('get_attributes returned None. Nothing will be indexed') return False objectId = metadata.save_attributes(attributes, configs['esEndpoint']) if objectId is not None: logging.info('Successfully handled s3 object created/updated event. objectID=' + objectId) return objectId else: logging.error('save_attributes returned an error. Indexing may not be complete.') return None
def setUp(self): import configutils self.configs = configutils.load_configs() self.bucket = self.configs['bucket'] self.keyBase = 'meta/unittest-a1234-15-imager_1234567890.' self.json_expected = { 'json1': 'jsonValue1', 'json2': 'jsonValue2' } self.csv_expected = { 'csv1': 'csvValue1', 'csv2': 'csvValue2', 'csv3': 'csvValue3' } self.custom_expected = { 'custom1': 'customValue1', 'custom2': 'customValue2' } self.plugin = 'defaultMetafileParser'
limitations under the License. ''' import unittest import logging import boto3 import time import json import configutils import secret from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth # TODO KLR: Review all es calls and surround with try/except Configs = configutils.load_configs() HabitatIndex = Configs['esHabitatIndex'] DocType = Configs['esDocType'] awsauth = AWS4Auth(secret.AWS_ACCESS_KEY_ID, secret.AWS_SECRET_ACCESS_KEY, secret.AWS_DEFAULT_REGION, 'es') def putEndpointInConfigFile(): ''' Utility function to get the endpoint from an existing domain and write it to the config file Usage: python -m esutils putEndpointInConfigFile ''' # Get the es endpoint given the domain name esclient = boto3.client('es') response = esclient.describe_elasticsearch_domain(DomainName=Configs['esDomain'])
limitations under the License. ''' import unittest import logging import boto3 import time import json import configutils import secret from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth # TODO KLR: Review all es calls and surround with try/except Configs = configutils.load_configs() HabitatIndex = Configs['esHabitatIndex'] DocType = Configs['esDocType'] awsauth = AWS4Auth(secret.AWS_ACCESS_KEY_ID, secret.AWS_SECRET_ACCESS_KEY, secret.AWS_DEFAULT_REGION, 'es') def putEndpointInConfigFile(): ''' Utility function to get the endpoint from an existing domain and write it to the config file Usage: python -m esutils putEndpointInConfigFile ''' # Get the es endpoint given the domain name
def setUp(self): import configutils self.configs = configutils.load_configs() ''' Assumes that a test file has been uploaded and matches the expected name and content ''' self.region = self.configs['region'] self.bucket = self.configs['bucket'] self.assayId = 'a1234' self.runId = '15' self.key = 'data/unittest-{}-{}-imager_1234567890.tif'.format( self.assayId, self.runId) self.user = '******' self.contentLength = 115 self.size = 1024 self.event = { "Records": [{ "eventVersion": "2.0", "eventTime": "1970-01-01T00:00:00.000Z", "requestParameters": { "sourceIPAddress": "127.0.0.1" }, "s3": { "configurationId": "testConfigRule", "object": { "eTag": "0123456789abcdef0123456789abcdef", "sequencer": "0A1B2C3D4E5F678901", "key": self.key, "size": self.size }, "bucket": { "arn": "arn:aws:s3:::" + self.bucket, "name": self.bucket, "ownerIdentity": { "principalId": "EXAMPLE" } }, "s3SchemaVersion": "1.0" }, "responseElements": { "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH", "x-amz-request-id": "EXAMPLE123456789" }, "awsRegion": self.region, "eventName": "ObjectCreated:Put", "userIdentity": { "principalId": self.user }, "eventSource": "aws:s3" }] } self.expected_head_attributes = { 'LastModified': '2016-03-26T16:14:13+00:00', 'ContentLength': self.contentLength } self.expected_event_attributes = { 'region': self.region, 'bucket': self.bucket, 'user': self.user, 'key': self.key, 'size': self.size } self.expected_body_attributes = { 'bodykey1': 'value1', 'bodykey2': 'value2' } self.expected_filename_attributes = { 'assayId': self.assayId, 'runId': self.runId } self.expected_s3meta_attributes = { 's3meta1': 'metaValue1', 's3meta2': 'metaValue2' } self.expected_attributes = {} self.expected_attributes.update(self.expected_event_attributes) self.expected_attributes.update(self.expected_head_attributes) self.expected_attributes.update(self.expected_s3meta_attributes) self.expected_attributes.update(self.expected_body_attributes) self.expected_attributes.update(self.expected_filename_attributes)
def setUp(self): self.configs = configutils.load_configs() self.bucket = self.configs['bucket']