def test_monitor_ids(self):
service = 'test'
method = 'get'
monitor_prefix = f'{Constants.MONITOR_PREFIX}{service}.{method}'
monitor_client = MonitoringProvider()
self.assertEqual(
monitor_client.get_monitor_id(service, method, False, False,
False),
f'{monitor_prefix}{Constants.MONITOR_CALLS_SUFFIX}')
self.assertEqual(
monitor_client.get_monitor_id(service, method, False, True, False),
f'{monitor_prefix}{Constants.MONITOR_LARGE_KEY}{Constants.MONITOR_CALLS_SUFFIX}'
)
self.assertEqual(
monitor_client.get_monitor_id(service, method, False, True, True),
f'{monitor_prefix}{Constants.MONITOR_LARGE_KEY}'
f'{Constants.MONITOR_ERROR_KEY}{Constants.MONITOR_CALLS_SUFFIX}')
self.assertEqual(
monitor_client.get_monitor_id(service, method, True, False, False),
f'{monitor_prefix}{Constants.MONITOR_ELAPSED_SUFFIX}')
self.assertEqual(
monitor_client.get_monitor_id(service, method, True, True, False),
f'{monitor_prefix}{Constants.MONITOR_LARGE_KEY}{Constants.MONITOR_ELAPSED_SUFFIX}'
)
self.assertEqual(
monitor_client.get_monitor_id(service, method, True, True, True),
f'{monitor_prefix}{Constants.MONITOR_LARGE_KEY}'
f'{Constants.MONITOR_ERROR_KEY}{Constants.MONITOR_ELAPSED_SUFFIX}')
def create(component_type, **kwargs):
if isinstance(component_type, Components.DataClients):
return Factory._create_data_client(component_type, **kwargs)
if isinstance(component_type, Components.ServiceClients):
return Factory._create_service_client(component_type, **kwargs)
if isinstance(component_type, Components.MonitorClients):
return MonitoringProvider()
if isinstance(component_type, Components.LogClients):
return Logger()
if isinstance(component_type, Components.SecretsClients):
return SecretManager()
if isinstance(component_type, Components.Controllers):
return Factory._create_controller(component_type, **kwargs)
return None
def test_utility_methods(self):
mock = MagicMock()
monitor_client = MonitoringProvider()
mock.args = "<Request 'http://localhost:8000/test/?f=j&limit=1000' [GET]>"
result = monitor_client.check_large_page(mock)
self.assertTrue(result)
mock.args = "<Request 'http://localhost:8000/test/' [GET]>"
result = monitor_client.check_large_page(mock)
self.assertFalse(result)
mock.args = "<Request 'http://localhost:8000/test/?f=j&limit=10' [GET]>"
result = monitor_client.check_large_page(mock)
self.assertFalse(result)
mock.args = "<Request 'http://localhost:8000/test/?limit=1000' [GET]>"
result = monitor_client.check_large_page(mock)
self.assertTrue(result)
mock.args = ''
result = monitor_client.check_large_page(mock)
self.assertFalse(result)
class CollectionController:
name = Settings.SERVICE_NAME
monitor = MonitoringProvider()
def __init__(self, **kwargs):
self.logger = Logger()
self.collector = kwargs.get(Constants.COLLECTOR, None)
self.controller = kwargs.get(Constants.CONTROLLER,
DataController(**kwargs))
self.client = kwargs.get(Constants.CLIENT, None)
@http(Constants.GET_REQUEST, get_route(Constants.PING_ROUTE))
def ping(self, request):
return self.controller.ping(request)
@http(Constants.OPTIONS_REQUEST, get_route())
@http(Constants.OPTIONS_REQUEST, get_route(Constants.RUN_ROUTE))
def options(self, request):
return self.controller.options(request)
@http(Constants.OPTIONS_REQUEST, get_item_route())
def options_item(self, request, item_id):
return self.controller.options_item(request, item_id)
@http(Constants.GET_REQUEST, get_route())
def get(self, request):
return self.controller.get(request)
@http(Constants.POST_REQUEST, get_route(Constants.PURGE_ROUTE))
def purge(self, request):
return self.controller.purge(request)
@http(Constants.GET_REQUEST, get_item_route())
def get_one(self, request, item_id):
return self.controller.get_one(request, item_id)
@http(Constants.GET_REQUEST, get_route(Constants.COUNT_ROUTE))
def count(self, request):
return self.controller.count(request)
@http(Constants.GET_REQUEST, get_route(Constants.HEALTH_ROUTE))
def health(self, request):
return self.controller.health(request)
@http(Constants.GET_REQUEST, get_route(Constants.VERSION_ROUTE))
def version(self, request):
return self.controller.version(request)
@http(Constants.PUT_REQUEST, get_route())
@http(Constants.DELETE_REQUEST, get_route())
def bad_verbs(self, request):
return self.controller.bad_verbs(request)
@http(Constants.POST_REQUEST, get_route())
def post(self, request):
self.logger.info(
get_log_string(request, Constants.POST_REQUEST, self.name))
return Constants.BAD_REQUEST_CODE, Constants.BAD_REQUEST_MESSAGE
@http(Constants.PUT_REQUEST, get_item_route())
def put(self, request, item_id):
self.logger.info(
get_log_string(request, Constants.PUT_REQUEST, self.name, item_id))
return Constants.BAD_REQUEST_CODE, Constants.BAD_REQUEST_MESSAGE
@http(Constants.POST_REQUEST, get_route(Constants.COLLECTION_ROUTE))
def collect(self, request):
return self.collect_post(request, self.collector, self.logger)
@http(Constants.GET_REQUEST, get_route(Constants.COLLECTION_ROUTE))
def collect_result(self, request):
return self.collect_get(request, self.collector, self.logger)
@http(Constants.DELETE_REQUEST, get_route(Constants.COLLECTION_ROUTE))
def collect_purge(self, request):
return self.collect_delete(request, self.collector, self.logger)
@http(Constants.DELETE_REQUEST, get_item_route())
def delete(self, request, item_id):
self.logger.info(
get_log_string(request, Constants.DELETE_REQUEST, self.name,
item_id))
return Constants.BAD_REQUEST_CODE, Constants.BAD_REQUEST_MESSAGE
@staticmethod
def collect_post(request, collector, logger):
logger.info(get_log_string(request, Constants.POST_REQUEST, __name__))
thread = Thread(target=perform_collection, args=(collector, ))
thread.start()
return Constants.RECEIVED_SUCCESS_CODE, Constants.COLLECTION_RECEIVED_MESSAGE
@staticmethod
def collect_get(request, collector, logger):
logger.info(get_log_string(request, Constants.GET_REQUEST, __name__))
result = collector.manager.get_collection_status(collector.name)
if not result:
return Constants.SUCCESS_RESPONSE_CODE, Constants.NO_COLLECTION_FOUND_MESSAGE
return Constants.SUCCESS_RESPONSE_CODE, json.dumps(result, default=str)
@staticmethod
def collect_delete(request, collector, logger):
logger.info(get_log_string(request, Constants.DELETE_REQUEST,
__name__))
collector.manager.purge_prior_collections()
return Constants.SUCCESS_RESPONSE_CODE, Constants.PURGE_REQUEST_MESSAGE
class DataController(metaclass=Meta):
name = Settings.SERVICE_NAME
model = BaseModel()
monitor = MonitoringProvider()
logger = LoggingProvider(name)
def __init__(self, **kwargs):
self.client = kwargs.get(Constants.CLIENT,
MongoDataClient(table=self.name))
self.model = kwargs.get(Constants.MODEL, BaseModel())
self.data_handler = DataHandler(self.client)
self.healthcheck = HealthCheckProvider(mongo=self.client)
# pylint: disable=no-self-use
@http(Constants.OPTIONS_REQUEST, get_route())
def options(self, _):
return Constants.SUCCESS_RESPONSE_CODE, version()
# pylint: disable=no-self-use
@http(Constants.OPTIONS_REQUEST, get_item_route())
def options_item(self, _, item_id):
return Constants.SUCCESS_RESPONSE_CODE, item_id
@http(Constants.GET_REQUEST, get_route())
def get(self, request):
self.logger.info(
get_log_string(request, Constants.GET_REQUEST, self.name))
result = self.data_handler.handle_get_request(request)
return result
@http(Constants.GET_REQUEST, get_item_route())
def get_one(self, request, item_id):
self.logger.info(
get_log_string(request, Constants.GET_REQUEST, self.name))
return self.data_handler.handle_get_request_single(item_id)
# pylint: disable=no-self-use
@http(Constants.GET_REQUEST, get_route(Constants.PING_ROUTE))
def ping(self, _):
return Constants.SUCCESS_RESPONSE_CODE, Constants.PING_RESPONSE
@http(Constants.GET_REQUEST, get_route(Constants.COUNT_ROUTE))
def count(self, request):
self.logger.info(
get_log_string(request, Constants.GET_REQUEST, self.name))
return self.data_handler.handle_count_request()
@http(Constants.GET_REQUEST, get_route(Constants.HEALTH_ROUTE))
def health(self, request):
self.logger.info(
get_log_string(request, Constants.GET_REQUEST, self.name,
Constants.HEALTH))
return Constants.SUCCESS_RESPONSE_CODE, self.healthcheck.check_health()
@http(Constants.GET_REQUEST, get_route(Constants.VERSION_ROUTE))
def version(self, request):
self.logger.info(
get_log_string(request, Constants.GET_REQUEST, self.name))
return Constants.SUCCESS_RESPONSE_CODE, version()
@http(Constants.POST_REQUEST, get_route())
def post(self, request):
self.logger.info(
get_log_string(request, Constants.POST_REQUEST, self.name))
payload = json.loads(request.get_data(as_text=True))
if self.model.validate(payload):
return self.data_handler.handle_post_request(payload)
return Constants.UNPROCESSABLE_ENTITY_CODE, Constants.UNPROCESSABLE_ENTITY_MESSAGE
@http(Constants.PUT_REQUEST, get_item_route())
def put(self, request, item_id):
self.logger.info(
get_log_string(request, Constants.PUT_REQUEST, self.name))
payload = json.loads(request.get_data(as_text=True))
return self.data_handler.handle_update_request(item_id, payload)
@http(Constants.POST_REQUEST, get_route(Constants.RESTORE_ROUTE))
def restore(self, request):
self.logger.info(
get_log_string(request, Constants.POST_REQUEST, self.name))
payload = json.loads(request.get_data(as_text=True))
return self.data_handler.handle_restore_request(payload)
@http(Constants.PUT_REQUEST, get_route())
@http(Constants.DELETE_REQUEST, get_route())
def bad_verbs(self, request):
self.logger.info(
get_log_string(request, Constants.PUT_REQUEST, self.name))
return Constants.BAD_REQUEST_CODE, Constants.BAD_REQUEST_MESSAGE
@http(Constants.DELETE_REQUEST, get_item_route())
@http(Constants.DELETE_REQUEST, get_route())
def delete(self, request, item_id):
self.logger.info(
get_log_string(request, Constants.DELETE_REQUEST, self.name,
item_id))
self.logger.info(f'{self.name} DELETE {item_id} called...')
return self.data_handler.handle_delete_request(item_id)
@http(Constants.POST_REQUEST, get_route(Constants.PURGE_ROUTE))
def purge(self, request):
self.logger.info(
get_log_string(request, Constants.POST_REQUEST, self.name))
return self.data_handler.handle_purge_request()
def test_provider():
mock = MagicMock()
mock.service_name = Constants.DUMMY
provider = MonitoringProvider()
provider.worker_setup(mock)
provider.worker_result(mock, True)
class MongoDataClient(DataClient):
""" Data client for the mongodb nosql document data store. """
def __init__(self, **kwargs):
""" Initialization for mongodb with support for TDD. """
super().__init__(**kwargs)
self.db_name = Settings.MONGO_DB_NAME
collection_name = kwargs.get(Constants.TABLE, Settings.SERVICE_NAME)
self.host = kwargs.get(Constants.HOST, Settings.MONGO_URL)
self.port = kwargs.get(Constants.PORT, Settings.MONGO_PORT)
self.client = kwargs.get(Constants.CLIENT, MongoClient(self.host, self.port, serverSelectionTimeoutMS=20))
self.collection = None
self.logger = Logger()
self.database = self.client[self.db_name]
self._select_collection(collection_name)
def get_health(self):
try:
return self.database.command(Constants.MONGO_STATS_COMMAND)
except ServerSelectionTimeoutError:
return {Constants.HEALTH: Constants.MONGO_CONNECTION_HEALTH_ERROR.format(host=self.host, port=self.port)}
@log_exception
def _select_collection(self, collection_name):
collection = self.get_collection_name(collection_name)
self.collection = self.client[self.db_name][collection]
@log_exception
def rename_table(self, old_name, new_name):
self.logger.debug(f'Rename table called on data client. old: {old_name} new: {new_name}')
self.database[old_name].rename(new_name, dropTarget=True)
@log_exception
def table_exists(self, table_name):
self.logger.debug(f'Existence check on table: {table_name}')
return table_name in self.database.collection_names()
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_create'))
def create(self, model):
if isinstance(model, BaseModel):
model = model.__dict__
result = self.collection.insert_one(model)
return result.inserted_id is not None
@log_exception
def restore(self, data):
self.logger.debug(f'Restoring data...')
result = self.collection.insert_many(data)
return result.inserted_ids is not None
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_get'))
def get(self, key):
self.logger.debug(f'Get item for id: {key}')
return self.collection.find_one({Constants.DB_ID_KEY: ObjectId(key)})
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_update'))
def update(self, key, model):
self.logger.debug(f'Update item for id: {key}')
if isinstance(model, BaseModel):
model = model.__dict__
model.pop(Constants.ID)
return self.collection.find_one_and_replace({Constants.DB_ID_KEY: ObjectId(key)}, model, upsert=True)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_get_specific'))
def get_specific(self, id_value, collection):
self.logger.debug(f'Get specific item for id: {id_value} from {collection}')
self.collection = self.client[self.db_name][collection]
return self.collection.find_one({Constants.DB_ID_KEY: id_value})
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_get_all'))
def get_all(self):
self.logger.debug(f'Get all called')
return self.convert_cursor_to_array(self.collection.find())
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_find'))
def find(self, key, value):
self.logger.debug(f'Find called key: {key} val: {value}')
return self.collection.find_one({key: value})
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_count'))
def count(self):
self.logger.debug('Count called')
return self.collection.find().count(True)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_purge'))
def purge(self):
self.logger.debug('Purge called')
return self.collection.drop()
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(Settings.SERVICE_NAME, 'mongo_read'))
def read(self, offset, limit, query):
self.logger.debug(f'Read called offset: {offset} limit: {limit} query: {query}')
count = self.collection.find(query).count()
cursor = self.collection.find(query).skip(offset).limit(limit)
return self.convert_cursor_to_array(cursor), offset, limit, count
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'mongo_delete'))
def delete(self, key):
self.logger.debug(f'Delete called id: {key}')
result = self.collection.delete_one({Constants.DB_ID_KEY: ObjectId(key)})
return result.deleted_count == 1
@log_exception
def delete_db(self):
self.logger.debug('Drop db called.')
self.client.drop_database(self.db_name)
@log_exception
def get_server_info(self):
self.logger.debug('Queries DB for server info')
return self.client.server_info()
@staticmethod
def convert_cursor_to_array(cursor):
documents = []
if cursor is not None:
for document in cursor:
documents.append(document)
return documents
@staticmethod
def get_collection_name(model):
if model:
if isinstance(model, str):
return model
return model.__name__
return 'not_set'
def get_status(self):
pass
def filter(self, **kwargs):
offset, limit, query = get_query(**kwargs)
return self.read(offset, limit, query)
class DynamoDataClient(DataClient):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.logger = Logger()
self.table_name = kwargs.get(Constants.TABLE, Settings.SERVICE_NAME)
self.client = kwargs.get(Constants.CLIENT, get_default_dynamo_client())
self.resource = kwargs.get(Constants.RESOURCE,
get_default_dynamo_resource())
self._initialize_tables()
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_create'))
def create(self, model):
model = self.get_converted_model(model)
result = self.resource.Table(self.table_name).put_item(Item=model)
return self.check_status(result)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_update'))
def update(self, key, model):
model.id = key
return self.create(model)
def get_converted_model(self, model, nest=False):
empty_keys = list()
if not isinstance(model, dict):
model = model.__dict__
for key, value in model.items():
if isinstance(value, float):
model[key] = Decimal(str(value))
continue
if not value:
if key not in empty_keys:
empty_keys.append(key)
continue
if isinstance(value, dict):
model[key] = self.get_converted_model(value, True)
if not model[key]:
if key not in empty_keys:
empty_keys.append(key)
for key in empty_keys:
model.pop(key)
if not nest:
if Constants.ID not in model.keys():
model[Constants.ID] = str(uuid4())
return model
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_get'))
def get(self, key):
self.logger.debug(f'Get item on dynamo for id: {key}')
self.create_table(self.table_name)
return self.resource.Table(self.table_name).get_item(Key={
Constants.ID: key
}).get(Constants.DYNAMO_ITEM_KEY, None)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_get_specific'))
def get_specific(self, id_value, table_name):
self.logger.debug(
f'Get specific item on dynamo for id: {id_value} from {table_name}'
)
self.create_table(self.table_name)
return self.resource.Table(
self.table_name).get_item(Key={
Constants.ID: id_value
}).get(Constants.DYNAMO_ITEM_KEY, None)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_find'))
def find(self, key, value):
self.logger.debug(f'Find called on dynamo key: {key} val: {value}')
all_items = self.get_all()
return self.find_item_in_list(all_items, **{key: value})
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_get_all'))
def get_all(self):
self.logger.debug(f'Get all called on dynamo')
self.create_table(self.table_name)
return self.resource.Table(self.table_name).scan().get(
Constants.DYNAMO_ITEMS_KEY, [])
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_count'))
def count(self):
return self.resource.Table(self.table_name).scan().get(
Constants.DYNAMO_COUNT_KEY, 0)
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_purge'))
def purge(self):
self.logger.debug('Purge called on dynamo...')
success = True
items = self.get_all()
for item in items:
success = success and self.delete(item.get(Constants.ID))
return success
@log_exception
@monitor(monitor_name=MonitoringProvider.get_monitor_prefix(
Settings.SERVICE_NAME, 'dynamo_delete'))
def delete(self, key):
self.logger.debug(f'Delete on dynamo called id: {key}')
result = self.resource.Table(
self.table_name).delete_item(Key={Constants.ID: key})
return self.check_status(result)
@staticmethod
def check_status(result):
return result.get(Constants.DYNAMO_RESPONSE_METADATA, {}).get(
Constants.DYNAMO_SUCCESS_STATUS,
Constants.BAD_REQUEST_CODE) == Constants.SUCCESS_RESPONSE_CODE
@log_exception
def table_exists(self, table_name):
try:
return self.resource.Table(table_name).table_status == 'ACTIVE'
except Exception: # pylint: disable=broad-except
return False
@log_exception
def delete_table(self, table_name):
try:
return self.check_status(
self.client.delete_table(TableName=table_name))
except Exception: # pylint: disable=broad-except
return False
@log_exception
def filter(self, **kwargs):
if not kwargs:
items = self.get_all()
return items, 0, len(items), len(items)
offset, limit, query = get_query(**kwargs)
all_items = self.get_all()
return self.find_items_in_list(all_items, offset, limit, **query)
@log_exception
def delete_db(self):
self.logger.debug('Drop db on dynamo called.')
return self.delete_table(self.table_name)
def create_table(self, table_name):
try:
if not self.table_exists(table_name):
self.resource.create_table(
AttributeDefinitions=[Constants.DYNAMO_ID_TABLE_ATTR],
KeySchema=[Constants.DYNAMO_ID_TABLE_KEY],
ProvisionedThroughput=Constants.DEFAULT_DYNAMO_THROUGHPUT,
TableName=table_name)
return True
except Exception as ex: # pylint: disable=broad-except
self.logger.info(f'Error creating dynamo table. {ex}',
exc_info=True)
return False
@staticmethod
def _get_dictionary_set(items_dictionary):
converted_list = list()
for key, value in items_dictionary.items():
converted_list.append(f'{key}|{value}')
return set(converted_list)
def find_item_in_list(self, all_items, **kwargs):
args_set = self._get_dictionary_set(kwargs)
match_list = list()
for item in all_items:
item_set = self._get_dictionary_set(item)
if len(item_set.intersection(args_set)) == len(args_set):
match_list.append(item)
return match_list
def find_items_in_list(self, all_items, offset, limit, **kwargs):
self.logger.info(f'Find called on dynamo with {offset}, {limit}')
match_list = self.find_item_in_list(all_items, **kwargs)
return match_list, offset, limit, len(match_list)
def _tag_table(self, table_arn):
try:
self.client.tag_resource(ResourceArn=table_arn,
Tags=Constants.DYNAMO_TAGS)
except Exception: # pylint: disable=broad-except
pass
def _initialize_tables(self):
if self.create_table(self.table_name):
self._tag_table(self.resource.Table(self.table_name).table_arn)
def get_status(self):
return self.table_exists(self.table_name)