def getDateAndTimeByKey(parameters, dt_key, task_id):
dateValue = None
timeValue = None
if parameters.get("".join(["date", dt_key])):
try:
dateValue = parse(parameters.get("".join(["date", dt_key]))).date()
except ValueError:
Logger.warn(
f"Bad date format. Continue running task without it. task_id: {task_id} \n"
f"key=<{''.join(['date', dt_key])}>; value=<{parameters.get(''.join(['date', dt_key]))}>"
)
if parameters.get("".join(["time", dt_key])):
try:
timeValue = parse(" ".join(
["1970-01-01",
parameters.get("".join(["time", dt_key]))])).time()
except ValueError:
Logger.warn(
f"Bad time format. Continue running task without it. task_id: {task_id} \n"
f"key=<{''.join(['time', dt_key])}>; value=<{parameters.get(''.join(['time', dt_key]))}>"
)
Logger.debug(dateValue)
Logger.debug(timeValue)
return (dateValue, timeValue)
def ml_task_runner(self, parameters):
start_time = time.perf_counter()
self.update_state(state='PROGRESS',
meta={
'current': 1,
'total': 100,
'status': 'Importing started.'
})
importlib.invalidate_caches()
try:
model_spec = importlib.util.spec_from_file_location(
'Model', parameters.get('model'))
class_model = importlib.util.module_from_spec(model_spec)
model_spec.loader.exec_module(class_model)
model = class_model.Model()
except (ImportError, AttributeError, TypeError) as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Bad preprocessor file.")
try:
prep_spec = importlib.util.spec_from_file_location(
'Preprocessor', parameters.get('preprocessor'))
class_prep = importlib.util.module_from_spec(prep_spec)
prep_spec.loader.exec_module(class_prep)
prep = class_prep.Preprocessor()
except (ImportError, AttributeError, TypeError) as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Bad preprocessor file.")
self.update_state(state='PROGRESS',
meta={
'current': 3,
'total': 100,
'status': 'Importing finished. Receiving cursor.'
})
# взять курсор
dt_key = 'From'
(dateFrom, timeFrom) = getDateAndTimeByKey(parameters, dt_key,
self.request.id)
dt_key = 'To'
(dateTo, timeTo) = getDateAndTimeByKey(parameters, dt_key, self.request.id)
tickers = parameters.get('ticker')
if tickers and not isinstance(tickers, list):
tickers = [tickers]
expr = "SELECT * FROM Data"
if tickers or dateFrom or timeFrom or dateTo or timeTo:
expr = addWhereToExpression(expr, tickers, dateFrom, timeFrom, dateTo,
timeTo)
if not expr:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
"CombineError(Not an exception)")
raise RuntimeError("Bad structure of finder fields.")
expr = " ".join([expr, "ORDER BY date DESC, time DESC;"])
Logger.debug(expr)
from app.fl_app import application
with application.app_context():
try:
cursor = db.engine.raw_connection().cursor()
cursor.execute(expr)
except Exception as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError(
"Exception during getting cursor from database.")
self.update_state(state='PROGRESS',
meta={
'current':
10,
'total':
100,
'status':
'Cursor has been received. Preprocessing started.'
})
try:
data = prep.preprocess(cursor)
except (AttributeError, TypeError) as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Bad preprocessor file.")
except Exception as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Exception in runtime of preprocessing.")
self.update_state(state='PROGRESS',
meta={
'current': 35,
'total': 100,
'status':
'Preprocessing finished. Loading data started.'
})
try:
model.load(parameters.get('resource'))
except (AttributeError, TypeError) as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Bad model file.")
except Exception as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Exception in runtime of loading data.")
self.update_state(state='PROGRESS',
meta={
'current': 65,
'total': 100,
'status':
'Loading data finished. Predicting started.'
})
try:
prediction = model.predict(data)
except (AttributeError, TypeError) as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Bad model file.")
except Exception as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Exception in runtime of predicting.")
self.update_state(state='PROGRESS',
meta={
'current': 93,
'total': 100,
'status': 'Prediction made. Adding it to database.'
})
# Вставка в db
with application.app_context():
try:
with transaction():
full_result = Results()
full_result.model = os.path.basename(parameters.get('model'))
full_result.preprocessor = os.path.basename(
parameters.get('preprocessor'))
full_result.resource = os.path.basename(
parameters.get('resource'))
full_result.personEmail = parameters.get('personEmail')
for key in prediction.keys():
try:
setattr(full_result, key, prediction.get(key))
except (AttributeError, TypeError):
Logger.warn(
f"Wrong attribute or type in predicton. "
f"Continue running task. task_id: {self.request.id} \n"
f"key=<{key}>; value=<{prediction.get(key)}>")
db.session.add(full_result)
except Exception as ex:
celeryLogFailAndEmail(self.request.id, start_time,
parameters.get('personEmail'),
type(ex).__name__)
raise RuntimeError("Exception during insertion into database.")
db.session.commit()
self.update_state(state='PROGRESS',
meta={
'current': 97,
'total': 100,
'status':
'Inserting to database finished. Sending mail'
})
# Отправка сообщения на почту
celeryLogSuccessAndEmail(self.request.id, start_time,
parameters.get('personEmail'), prediction)
return {
'current': 100,
'total': 100,
'status': 'Task completed!',
'result': prediction
}
