class AbstractTask:
def __init__(self, priority=0, type=''):
'''Initializates the AbstractTask and generates assigns an
unique task_id as combintaion of:
task_{type}_{uuid4}
:return Class instance
'''
self.priority = priority
self.creation_time = datetime.now()
self.type = type
self.execution_information = None
# For instance:
# task_Api-request_63f1bd71-f441-4519-8a41-44643ccb4dad
self._task_id = \
f'task_{type}_{uuid.uuid4()}'
self.logger = CustomLogger(__name__)
#print("LOGGER", self.logger)
self.logger.info(f"{self._task_id}")
@property
def task_id(self):
return self._task_id
@task_id.setter
def task_id(self, value):
self._task_id = value
@abstractmethod
def execute(self):
pass
def __init__(self, priority, config: ConfigApiRequestTask):
''' Instantiates an Api request task
:param priority: from 0 (most important) to any greater integer
:param config(ConfigApiRequestTask): Configuration parameters
'''
super().__init__(priority, type='Api-request')
self.config = config
self.logger = CustomLogger(__name__)
def __init__(self, params, db_connection: MongoDbConnection=None,
dynamic_configs=None):
self.type_task = params["type_task"]
self.configuration_id = params["configuration_id"]
if db_connection is None:
self.connection_to_db = current_app.mongo_connection
# self.connection_to_db = g.db # global db connection
else:
self.connection_to_db = db_connection
self.dynamic_configs = params.get('dynamic_configs', {})
self.result = None
self.errors = None
self.logger = CustomLogger(__name__)
def __init__(self, priority, config:ConfigDbTask):
"""Instantiates a Database task
...
Attributes
----------
priority : int
a number from 0 (most important) to any greater integer
config : ConfigDbTask
a configuration object containing the parameters nedded to run the task
"""
super().__init__(priority, type='Db')
self.config = config
self.config.db_connection.connect()
self.logger = CustomLogger(__name__)
class DbTask(AbstractTask):
"""This class defines the operations to be performed with a specific DB
...
Attributes
----------
priority : int
number assigns a level of priority to run the task
config : ConfigDbTask
a configuration object containing the parameters nedded to run the task
Methods
-------
validate():
Verifies the content of a given data before makes a query to the DB.
insert():
Inserts a given data into a specific DB.
get():
Get a given data from a specific DB.
delete():
Deletes a given data from a specific DB.
update():
Updates a given data from a specific DB.
"""
def __init__(self, priority, config:ConfigDbTask):
"""Instantiates a Database task
...
Attributes
----------
priority : int
a number from 0 (most important) to any greater integer
config : ConfigDbTask
a configuration object containing the parameters nedded to run the task
"""
super().__init__(priority, type='Db')
self.config = config
self.config.db_connection.connect()
self.logger = CustomLogger(__name__)
def insert(self):
return self.config.db_connection.insert(self.config.query, self.config.key_id)
def get(self):
return self.config.db_connection.get(self.config.key_id)
def update(self):
return self.config.db_connection.update(self.config.key_id, self.config.query)
def delete(self):
return self.config.db_connection.delete(self.config.key_id)
def execute(self):
if self.config is None:
self.logger.error("Need a configuration object to do query to the DB")
raise ConfigurationError("Need a configuration object to do query to the DB")
else:
if self.config.query_type.upper() == "INSERT":
return self.insert()
elif self.config.query_type.upper() == "GET":
return self.get()
elif self.config.query_type.upper() == "UPDATE":
return self.update()
elif self.config.query_type.upper() == "DELETE":
return self.delete()
class ApiRequestTask(AbstractTask):
def __init__(self, priority, config: ConfigApiRequestTask):
''' Instantiates an Api request task
:param priority: from 0 (most important) to any greater integer
:param config(ConfigApiRequestTask): Configuration parameters
'''
super().__init__(priority, type='Api-request')
self.config = config
self.logger = CustomLogger(__name__)
def do_request(self):
'''Do api request using requests library using config object
:return: dict with response objects cotaining:
{
'json': <dict>, # json convertion or {}
'text': <str>, # string response
'status_code': <str>,
'headers': <dict>,
'url': <str>
}
If an error occurs it raises an HTTPError exceptio
'''
if self.config is None:
raise Exception("Need a config object to do request")
response = None
try:
# TODO: add support to send api_token
response = requests.request(
self.config.http_method.upper(),
self.config.url,
params=None,
data=self.config.body,
headers=self.config.headers,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None
)
response_json = {}
try:
response_json = response.json()
except JSONDecodeError as json_err:
pass
res_dict = {
'json': response_json,
'text': response.text,
'status_code': response.status_code,
'headers': response.headers,
'url': response.url
}
#self.logger.debug("Api-request executed")
#self.logger.debug(res_dict)
return res_dict
# TODO: Log all exceptions
except HTTPError as http_err:
self.logger.info(f'HTTPError: {http_err}')
raise http_err
except RequestException as request_err:
self.logger.info(f'RequestException: {request_err}')
raise request_err
except ConnectionError as conn_err:
self.logger.info(f'ConnectionError: {conn_err}')
raise conn_err
except URLRequired as url_err:
self.logger.info(f'URLRequired: {url_err}')
raise url_err
def execute(self):
''' Performs an API request and returns
a dictionary with response results or raises exception
'''
try:
response = self.do_request()
return response
except Exception as E:
raise E
class TaskManager:
"""
Class used to execute a specific task with its configuration through the
RunTask endpoint request.
...
Attributes
----------
params : dict
params[type_task]: str
it is the type of the task e.g. "Api-request" or "Db" or "File" to be searched for
params[configuration_id]: str
it is the ID of the configuration to be searched for
db_connection: MongoDbConnection
the connection has to be passed to the Mongo DB
Methods
-------
execute():
Executes the task with its configuration
instantiate():
Initializes the configuration object and the task object to be executed
"""
def __init__(self, params, db_connection: MongoDbConnection=None,
dynamic_configs=None):
self.type_task = params["type_task"]
self.configuration_id = params["configuration_id"]
if db_connection is None:
self.connection_to_db = current_app.mongo_connection
# self.connection_to_db = g.db # global db connection
else:
self.connection_to_db = db_connection
self.dynamic_configs = params.get('dynamic_configs', {})
self.result = None
self.errors = None
self.logger = CustomLogger(__name__)
def run(self):
"""Executes the task and saves its results to the DB.
It creates a config object and task result to be into Db using self.save_into_db()
:returns: True if there is no error and False if some error occurs (logs it)
"""
if self.type_task == "Api-request":
self.config = ConfigApiRequestTask(**self.dynamic_configs)
self.task = ApiRequestTask(
priority=0, # fixed priority
config=self.config
)
elif self.type_task == 'Db':
self.config = ConfigDbTask(self.dynamic_configs)
self.task = DbTask(
priority=0,
config=self.config
)
elif self.type_task == 'File':
self.config = ConfigFileTask(self.dynamic_configs)
self.task = FileTask(
priority=0,
config=self.config
)
try:
self.result = self.task.execute()
except Exception as e:
self.errors = str(e)
self.logger.error(f'Error executing task: {self.errors}')
return False
res = self.save_into_db()
return res
def save_into_db(self):
'''Saves the task execution results to the DB. Creates both documents
config and task_results following the schema defined at docs/Mongo-schema.md
:returns: A dict with task_result_id and config_id.
Empty values if error occurs, errors are logged.
'''
config_dbobj = {}
if self.type_task == "Api-request":
config_dbobj = {
#'_id': ObjectId(self.config['config_id']),
'config_id': self.config.config_id,
'url': self.config.url,
'http_method': self.config.http_method,
'headers': json.dumps(self.config.headers),
'body': json.dumps(self.config.body),
'api_token': self.config.api_token
}
elif self.type_task == 'Db':
config_dbobj = {
"config_id": self.config.config_id,
"config_type": self.type_task,
"key_id": self.config.key_id,
"query_type": self.config.query_type,
"query": json.dumps(self.config.query),
"connector": json.dumps(self.config.connector)
}
elif self.type_task == 'File':
# TODO: implement
pass
task_resultdb = {
"task_result_id": f"task-result_{uuid.uuid4()}",
"runBy": "user",
"time": dt.now().strftime('%d/%m/%Y %H:%M:%S'),
"error_message": str(self.errors),
"result": str(self.result)
}
task_obj = {
"task_id": self.task.task_id,
"creation_time": self.task.creation_time.strftime('%d/%m/%Y %H:%M:%S'),
"priority": 0,
"type": self.task.type,
"config": self.config.config_id,
"tasks_results": [task_resultdb["task_result_id"]]
}
try:
self.connection_to_db.connect()
self.connection_to_db.insert('config', config_dbobj)
self.connection_to_db.insert('tasks', task_obj)
self.connection_to_db.insert('task_result', task_resultdb)
self.logger.info(f'Task execution saved to DB')
return {
'config_id': config_dbobj['config_id'],
'task_id': task_obj['task_id'],
'task_result_id': task_resultdb['task_result_id']
}
except Exception as err:
self.logger.error(f'Error registering task results into db: {err}')
return {'task_result_id': '', 'config_id': '', 'task_id': ''}
# def execute(self):
# """Executes the task given the configs and saves its results in self object
# :return:
# """
# run_arg = {
# "type": self.type_task,
# "configs": ObjectId(self.configuration_id)
# }
# task_args = self.connection_to_db.get("tasks", run_arg)[0]
# config_args = self.connection_to_db.get("configs", {"_id":ObjectId(self.configuration_id)})[0]
# self.instantiate(task_args, config_args)
# try:
# self.result = self.task.execute()
# return self.result
# except Exception as e:
# self.errors = str(e)
# self.logger.info(f'Error executing task: {self.errors}')
# return self.errors
# def instantiate(self, task_args, config_args):
# if self.type_task == "Api-request":
# self.config = ConfigApiRequestTask(config_args)
# self.task = ApiRequestTask(
# priority=task_args["priority"],
# config=self.config
# )
# elif self.type_task == "Db":
# self.config = ConfigDbTask(config_args)
# self.task = DbTask(
# priority=task_args["priority"],
# config=self.config
# )
# elif self.type_task == "File":
# self.config = ConfigFileTask(config_args)
# self.task = FileTask(
# priority=task_args["priority"],
# config=self.config
# )