class NewRelicQueryExecutor:
def __init__(self,
account_name,
http=HttpRequestor(),
secrets_manager=SecretManager()):
self.http = http
self.secrets_manager = secrets_manager
self.account_name = account_name
self.account_id = get_newrelic_account_id(self.account_name)
self.api_key = get_newrelic_api_key(self.account_name,
self.secrets_manager)
self.logger = Logger(Settings.SERVICE_NAME)
def execute_faceted_query(self, query):
result = self._execute_query(query)
result_json = result.json()
if not result.ok:
return self._get_failure_result(result_json)
return QueryResult(result_json, ResultTypes.SUCCESS)
def execute_single_value_query(self, query):
try:
result = self._execute_query(query)
result_json = result.json()
if not result.ok:
return self._get_failure_result(result_json)
results = result_json.get(Constants.RESULTS, None)
if results:
first_result = results[0]
if first_result:
filtered_result = next(iter(first_result.values()))
return QueryResult(filtered_result, ResultTypes.SUCCESS)
return QueryResult(None, ResultTypes.UNKNOWN,
Constants.NR_EMPTY_RESULTS_MESSAGE)
return QueryResult(None, ResultTypes.UNKNOWN,
Constants.NR_NO_RESULT_ITEM_MESSAGE)
except (Exception, ValueError) as ex: # pylint: disable=broad-except
self.logger.error(
f'Newrelic query execution for {query} failed: {ex}')
return QueryResult(None, ResultTypes.ERROR, ex)
def _get_failure_result(self, result_json):
self.logger.info(f'Newrelic result: {result_json}')
return QueryResult(None, ResultTypes.ERROR, result_json)
def build_url(self, account_id, query):
try:
url = Settings.NEWRELIC_QUERY_URL
encoded_query = quote(query)
return url.format(account_id=account_id, query=encoded_query)
except TypeError as ex:
self.logger.error(
f'Encoding query failed due to non-string parameter: {ex}')
return None
def _execute_query(self, query):
self.logger.info(f'Executing newrelic query: {query}')
headers = {Constants.X_QUERY_KEY_HEADER: self.api_key}
url = self.build_url(self.account_id, query)
return self.http.get(url, headers=headers)
class SumologicQueryExecutor:
def __init__(self, http=HttpRequestor()):
self.http = http
self.logger = Logger(Settings.SERVICE_NAME)
self.headers = {Constants.CONTENT_TYPE_HEADER: Constants.CONTENT_TYPE_JSON,
Constants.ACCEPT_HEADER: Constants.CONTENT_TYPE_JSON}
def execute_sumologic_query(self, query, from_time, to_time, api_id, api_key):
try:
payload = {Constants.QUERY: query, Constants.FROM_DATE_PARAMETER: from_time,
Constants.TO_DATE_PARAMETER: to_time}
self.logger.info(f'Running sumologic query... {payload}')
result = self.http.post(
Constants.SUMOLOGIC_QUERY_URL, auth=self.http.get_auth_value(api_id, api_key),
data=json.dumps(payload, default=str), headers=self.headers)
self.logger.info(f'Result from sumologic post is... {result.text}')
if result.status_code == Constants.RECEIVED_SUCCESS_CODE:
return self.get_sumologic_query_results(result, api_id, api_key)
return QueryResult(None, ResultTypes.UNKNOWN, Constants.SUMO_NO_RESULT_FOUND_MESSAGE)
except (Exception, ValueError) as ex: # pylint: disable=broad-except
self.logger.error(f'Sumologic query execution failed: {ex}', exc_info=True)
return QueryResult(None, ResultTypes.ERROR, ex)
def get_sumologic_query_results(self, query_request_json, api_id, api_key, retry_count=5, wait_time_seconds=5):
try:
retries = retry_count
result_json = query_request_json.json()
url_get = result_json[Constants.LINK][Constants.HREF]
self.logger.info(f'Making sumologic get request for query results. {url_get}')
while retries >= 0:
retries -= 1
query_result = self.http.get(
url_get, auth=self.http.get_auth_value(api_id, api_key), headers=self.headers)
if query_result.status_code == Constants.SUCCESS_RESPONSE_CODE:
result = query_result.json()
message_count = result.get(Constants.SUMOLOGIC_MESSAGE_COUNT, None)
if message_count:
return QueryResult(message_count, ResultTypes.SUCCESS)
return QueryResult(None, ResultTypes.UNKNOWN, Constants.SUMO_MISSING_COUNT_MESSAGE)
sleep(wait_time_seconds)
return QueryResult(0, ResultTypes.UNKNOWN, Constants.SUMO_RETRY_EXCEEDED_MESSAGE)
except (Exception, ValueError) as ex: # pylint: disable=broad-except
self.logger.error(f'Error getting sumologic query results. {ex}')
return QueryResult(None, ResultTypes.ERROR, ex)
class CollectionController:
name = Settings.SERVICE_NAME
monitor = MonitoringProvider()
def __init__(self, **kwargs):
self.logger = Logger()
self.collector = kwargs.get(Constants.COLLECTOR, None)
self.controller = kwargs.get(Constants.CONTROLLER,
DataController(**kwargs))
self.client = kwargs.get(Constants.CLIENT, None)
@http(Constants.GET_REQUEST, get_route(Constants.PING_ROUTE))
def ping(self, request):
return self.controller.ping(request)
@http(Constants.OPTIONS_REQUEST, get_route())
@http(Constants.OPTIONS_REQUEST, get_route(Constants.RUN_ROUTE))
def options(self, request):
return self.controller.options(request)
@http(Constants.OPTIONS_REQUEST, get_item_route())
def options_item(self, request, item_id):
return self.controller.options_item(request, item_id)
@http(Constants.GET_REQUEST, get_route())
def get(self, request):
return self.controller.get(request)
@http(Constants.POST_REQUEST, get_route(Constants.PURGE_ROUTE))
def purge(self, request):
return self.controller.purge(request)
@http(Constants.GET_REQUEST, get_item_route())
def get_one(self, request, item_id):
return self.controller.get_one(request, item_id)
@http(Constants.GET_REQUEST, get_route(Constants.COUNT_ROUTE))
def count(self, request):
return self.controller.count(request)
@http(Constants.GET_REQUEST, get_route(Constants.HEALTH_ROUTE))
def health(self, request):
return self.controller.health(request)
@http(Constants.GET_REQUEST, get_route(Constants.VERSION_ROUTE))
def version(self, request):
return self.controller.version(request)
@http(Constants.PUT_REQUEST, get_route())
@http(Constants.DELETE_REQUEST, get_route())
def bad_verbs(self, request):
return self.controller.bad_verbs(request)
@http(Constants.POST_REQUEST, get_route())
def post(self, request):
self.logger.info(
get_log_string(request, Constants.POST_REQUEST, self.name))
return Constants.BAD_REQUEST_CODE, Constants.BAD_REQUEST_MESSAGE
@http(Constants.PUT_REQUEST, get_item_route())
def put(self, request, item_id):
self.logger.info(
get_log_string(request, Constants.PUT_REQUEST, self.name, item_id))
return Constants.BAD_REQUEST_CODE, Constants.BAD_REQUEST_MESSAGE
@http(Constants.POST_REQUEST, get_route(Constants.COLLECTION_ROUTE))
def collect(self, request):
return self.collect_post(request, self.collector, self.logger)
@http(Constants.GET_REQUEST, get_route(Constants.COLLECTION_ROUTE))
def collect_result(self, request):
return self.collect_get(request, self.collector, self.logger)
@http(Constants.DELETE_REQUEST, get_route(Constants.COLLECTION_ROUTE))
def collect_purge(self, request):
return self.collect_delete(request, self.collector, self.logger)
@http(Constants.DELETE_REQUEST, get_item_route())
def delete(self, request, item_id):
self.logger.info(
get_log_string(request, Constants.DELETE_REQUEST, self.name,
item_id))
return Constants.BAD_REQUEST_CODE, Constants.BAD_REQUEST_MESSAGE
@staticmethod
def collect_post(request, collector, logger):
logger.info(get_log_string(request, Constants.POST_REQUEST, __name__))
thread = Thread(target=perform_collection, args=(collector, ))
thread.start()
return Constants.RECEIVED_SUCCESS_CODE, Constants.COLLECTION_RECEIVED_MESSAGE
@staticmethod
def collect_get(request, collector, logger):
logger.info(get_log_string(request, Constants.GET_REQUEST, __name__))
result = collector.manager.get_collection_status(collector.name)
if not result:
return Constants.SUCCESS_RESPONSE_CODE, Constants.NO_COLLECTION_FOUND_MESSAGE
return Constants.SUCCESS_RESPONSE_CODE, json.dumps(result, default=str)
@staticmethod
def collect_delete(request, collector, logger):
logger.info(get_log_string(request, Constants.DELETE_REQUEST,
__name__))
collector.manager.purge_prior_collections()
return Constants.SUCCESS_RESPONSE_CODE, Constants.PURGE_REQUEST_MESSAGE
class DynamoDataClient(DataClient):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.logger = Logger()
self.table_name = kwargs.get(Constants.TABLE, Settings.SERVICE_NAME)
self.client = kwargs.get(Constants.CLIENT, get_default_dynamo_client())
self.resource = kwargs.get(Constants.RESOURCE,
get_default_dynamo_resource())
self._initialize_tables()
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_create'))
def create(self, model):
model = self.get_converted_model(model)
result = self.resource.Table(self.table_name).put_item(Item=model)
return self.check_status(result)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_update'))
def update(self, key, model):
model.id = key
return self.create(model)
def get_converted_model(self, model, nest=False):
empty_keys = list()
if not isinstance(model, dict):
model = model.__dict__
for key, value in model.items():
if isinstance(value, float):
model[key] = Decimal(str(value))
continue
if not value:
if key not in empty_keys:
empty_keys.append(key)
continue
if isinstance(value, dict):
model[key] = self.get_converted_model(value, True)
if not model[key]:
if key not in empty_keys:
empty_keys.append(key)
for key in empty_keys:
model.pop(key)
if not nest:
if Constants.ID not in model.keys():
model[Constants.ID] = str(uuid4())
return model
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_get'))
def get(self, key):
self.logger.debug(f'Get item on dynamo for id: {key}')
self.create_table(self.table_name)
return self.resource.Table(self.table_name).get_item(Key={
Constants.ID: key
}).get(Constants.DYNAMO_ITEM_KEY, None)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_get_specific'))
def get_specific(self, id_value, table_name):
self.logger.debug(
f'Get specific item on dynamo for id: {id_value} from {table_name}'
)
self.create_table(self.table_name)
return self.resource.Table(
self.table_name).get_item(Key={
Constants.ID: id_value
}).get(Constants.DYNAMO_ITEM_KEY, None)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_find'))
def find(self, key, value):
self.logger.debug(f'Find called on dynamo key: {key} val: {value}')
all_items = self.get_all()
return self.find_item_in_list(all_items, **{key: value})
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_get_all'))
def get_all(self):
self.logger.debug(f'Get all called on dynamo')
self.create_table(self.table_name)
return self.resource.Table(self.table_name).scan().get(
Constants.DYNAMO_ITEMS_KEY, [])
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_count'))
def count(self):
return self.resource.Table(self.table_name).scan().get(
Constants.DYNAMO_COUNT_KEY, 0)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_purge'))
def purge(self):
self.logger.debug('Purge called on dynamo...')
success = True
items = self.get_all()
for item in items:
success = success and self.delete(item.get(Constants.ID))
return success
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_delete'))
def delete(self, key):
self.logger.debug(f'Delete on dynamo called id: {key}')
result = self.resource.Table(
self.table_name).delete_item(Key={Constants.ID: key})
return self.check_status(result)
@staticmethod
def check_status(result):
return result.get(Constants.DYNAMO_RESPONSE_METADATA, {}).get(
Constants.DYNAMO_SUCCESS_STATUS,
Constants.BAD_REQUEST_CODE) == Constants.SUCCESS_RESPONSE_CODE
@log_exception
def table_exists(self, table_name):
try:
return self.resource.Table(table_name).table_status == 'ACTIVE'
except Exception: # pylint: disable=broad-except
return False
@log_exception
def delete_table(self, table_name):
try:
return self.check_status(
self.client.delete_table(TableName=table_name))
except Exception: # pylint: disable=broad-except
return False
@log_exception
def filter(self, **kwargs):
if not kwargs:
items = self.get_all()
return items, 0, len(items), len(items)
offset, limit, query = get_query(**kwargs)
all_items = self.get_all()
return self.find_items_in_list(all_items, offset, limit, **query)
@log_exception
def delete_db(self):
self.logger.debug('Drop db on dynamo called.')
return self.delete_table(self.table_name)
def create_table(self, table_name):
try:
if not self.table_exists(table_name):
self.resource.create_table(
AttributeDefinitions=[Constants.DYNAMO_ID_TABLE_ATTR],
KeySchema=[Constants.DYNAMO_ID_TABLE_KEY],
ProvisionedThroughput=Constants.DEFAULT_DYNAMO_THROUGHPUT,
TableName=table_name)
return True
except Exception as ex: # pylint: disable=broad-except
self.logger.info(f'Error creating dynamo table. {ex}',
exc_info=True)
return False
@staticmethod
def _get_dictionary_set(items_dictionary):
converted_list = list()
for key, value in items_dictionary.items():
converted_list.append(f'{key}|{value}')
return set(converted_list)
def find_item_in_list(self, all_items, **kwargs):
args_set = self._get_dictionary_set(kwargs)
match_list = list()
for item in all_items:
item_set = self._get_dictionary_set(item)
if len(item_set.intersection(args_set)) == len(args_set):
match_list.append(item)
return match_list
def find_items_in_list(self, all_items, offset, limit, **kwargs):
self.logger.info(f'Find called on dynamo with {offset}, {limit}')
match_list = self.find_item_in_list(all_items, **kwargs)
return match_list, offset, limit, len(match_list)
def _tag_table(self, table_arn):
try:
self.client.tag_resource(ResourceArn=table_arn,
Tags=Constants.DYNAMO_TAGS)
except Exception: # pylint: disable=broad-except
pass
def _initialize_tables(self):
if self.create_table(self.table_name):
self._tag_table(self.resource.Table(self.table_name).table_arn)
def get_status(self):
return self.table_exists(self.table_name)