def getcli():
'''
Quick and dirty utility to all quick command line query to ES
Usage:
python -m esutils getcli <an_id>
'''
import sys
Configs = configutils.load_configs()
objectId=sys.argv[2]
result = getById(objectId, Configs['esEndpoint'])
print json.dumps(result, indent=4)
exit(0)
def setUp(self):
import configutils
self.configs = configutils.load_configs()
self.bucket = self.configs['bucket']
self.key = 'data/unittest-a1234-15-imager_1234567890.tif'
self.body_expected = {'bodykey1': 'value1', 'bodykey2': 'value2'}
self.head_expected = {
'LastModified': 'DUMMY LASTMODIFIED',
'ContentLength': 115
}
self.s3meta_expected = {
's3meta1': 'metaValue1',
's3meta2': 'metaValue2'
}
def getcli():
'''
Quick and dirty utility to all quick command line query to ES
Usage:
python -m esutils getcli <an_id>
'''
import sys
Configs = configutils.load_configs()
objectId = sys.argv[2]
result = getById(objectId, Configs['esEndpoint'])
print json.dumps(result, indent=4)
exit(0)
def setUp(self):
import configutils
self.configs = configutils.load_configs()
self.bucket = self.configs['bucket']
self.key = 'data/unittest-a1234-15-imager_1234567890.tif'
self.body_expected = {
'bodykey1': 'value1',
'bodykey2': 'value2'
}
self.head_expected = {
'LastModified': 'DUMMY LASTMODIFIED',
'ContentLength': 115
}
self.s3meta_expected = {
's3meta1': 'metaValue1',
's3meta2': 'metaValue2'
}
def event_handler(event, context):
logging.info('Received s3 object event... ')
logging.debug('Event = ' + json.dumps(event, indent=4))
configs = configutils.load_configs()
attributes = metadata.get_attributes(event, configs)
if attributes is None:
logging.error('get_attributes returned None. Nothing will be indexed')
return False
objectId = metadata.save_attributes(attributes, configs['esEndpoint'])
if objectId is not None:
logging.info('Successfully handled s3 object created/updated event. objectID=' + objectId)
return objectId
else:
logging.error('save_attributes returned an error. Indexing may not be complete.')
return None
def setUp(self):
import configutils
self.configs = configutils.load_configs()
self.bucket = self.configs['bucket']
self.keyBase = 'meta/unittest-a1234-15-imager_1234567890.'
self.json_expected = {
'json1': 'jsonValue1',
'json2': 'jsonValue2'
}
self.csv_expected = {
'csv1': 'csvValue1',
'csv2': 'csvValue2',
'csv3': 'csvValue3'
}
self.custom_expected = {
'custom1': 'customValue1',
'custom2': 'customValue2'
}
self.plugin = 'defaultMetafileParser'
limitations under the License.
'''
import unittest
import logging
import boto3
import time
import json
import configutils
import secret
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
# TODO KLR: Review all es calls and surround with try/except
Configs = configutils.load_configs()
HabitatIndex = Configs['esHabitatIndex']
DocType = Configs['esDocType']
awsauth = AWS4Auth(secret.AWS_ACCESS_KEY_ID, secret.AWS_SECRET_ACCESS_KEY, secret.AWS_DEFAULT_REGION, 'es')
def putEndpointInConfigFile():
'''
Utility function to get the endpoint from an existing domain and write it to the config file
Usage:
python -m esutils putEndpointInConfigFile
'''
# Get the es endpoint given the domain name
esclient = boto3.client('es')
response = esclient.describe_elasticsearch_domain(DomainName=Configs['esDomain'])
limitations under the License.
'''
import unittest
import logging
import boto3
import time
import json
import configutils
import secret
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
# TODO KLR: Review all es calls and surround with try/except
Configs = configutils.load_configs()
HabitatIndex = Configs['esHabitatIndex']
DocType = Configs['esDocType']
awsauth = AWS4Auth(secret.AWS_ACCESS_KEY_ID, secret.AWS_SECRET_ACCESS_KEY,
secret.AWS_DEFAULT_REGION, 'es')
def putEndpointInConfigFile():
'''
Utility function to get the endpoint from an existing domain and write it to the config file
Usage:
python -m esutils putEndpointInConfigFile
'''
# Get the es endpoint given the domain name
def setUp(self):
import configutils
self.configs = configutils.load_configs()
''' Assumes that a test file has been uploaded and matches the expected name and content '''
self.region = self.configs['region']
self.bucket = self.configs['bucket']
self.assayId = 'a1234'
self.runId = '15'
self.key = 'data/unittest-{}-{}-imager_1234567890.tif'.format(
self.assayId, self.runId)
self.user = '******'
self.contentLength = 115
self.size = 1024
self.event = {
"Records": [{
"eventVersion": "2.0",
"eventTime": "1970-01-01T00:00:00.000Z",
"requestParameters": {
"sourceIPAddress": "127.0.0.1"
},
"s3": {
"configurationId": "testConfigRule",
"object": {
"eTag": "0123456789abcdef0123456789abcdef",
"sequencer": "0A1B2C3D4E5F678901",
"key": self.key,
"size": self.size
},
"bucket": {
"arn": "arn:aws:s3:::" + self.bucket,
"name": self.bucket,
"ownerIdentity": {
"principalId": "EXAMPLE"
}
},
"s3SchemaVersion": "1.0"
},
"responseElements": {
"x-amz-id-2":
"EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH",
"x-amz-request-id": "EXAMPLE123456789"
},
"awsRegion": self.region,
"eventName": "ObjectCreated:Put",
"userIdentity": {
"principalId": self.user
},
"eventSource": "aws:s3"
}]
}
self.expected_head_attributes = {
'LastModified': '2016-03-26T16:14:13+00:00',
'ContentLength': self.contentLength
}
self.expected_event_attributes = {
'region': self.region,
'bucket': self.bucket,
'user': self.user,
'key': self.key,
'size': self.size
}
self.expected_body_attributes = {
'bodykey1': 'value1',
'bodykey2': 'value2'
}
self.expected_filename_attributes = {
'assayId': self.assayId,
'runId': self.runId
}
self.expected_s3meta_attributes = {
's3meta1': 'metaValue1',
's3meta2': 'metaValue2'
}
self.expected_attributes = {}
self.expected_attributes.update(self.expected_event_attributes)
self.expected_attributes.update(self.expected_head_attributes)
self.expected_attributes.update(self.expected_s3meta_attributes)
self.expected_attributes.update(self.expected_body_attributes)
self.expected_attributes.update(self.expected_filename_attributes)
def setUp(self):
self.configs = configutils.load_configs()
self.bucket = self.configs['bucket']