def model_multivariate(list_var,num_fut,desv_mse):
engines_output={}
debug = {}
try:
engines_output['LSTM'] = anomaly_LSTM(list_var,num_fut,desv_mse)
debug['LSTM'] = engines_output['LSTM']['debug']
print (engines_output['LSTM'])
except Exception as e:
print(e)
print ('ERROR: exception executing LSTM')
try:
engines_output['VAR'] = anomaly_VAR(list_var,num_fut)
debug['VAR'] = engines_output['VAR']['debug']
print (engines_output['VAR'])
except Exception as e:
print(Exception)
print("type error: " + str(e))
print(traceback.format_exc())
print ('ERROR: exception executing VAR')
best_mae=999999999
winner='LSTM'
print ('The size is ')
print (len(engines_output))
print (debug)
for key, value in engines_output.items():
print (key)
print(str(value['mae']))
if value['mae'] < best_mae:
print (key + " " + str(value['mae']) + " best:" + str(best_mae) )
best_mae=value['mae']
winner=key
print ("el ganador es " + winner)
temp= {}
temp['debug']=debug
return merge_two_dicts(engines_output[winner] , temp)
def back_model_multivariate(self,
list_var,
num_fut,
desv_mse,
train=True,
name='Test'):
engines_output = {}
debug = {}
temp_info = {}
self.update_state(state='PROGRESS',
meta={
'running': 'LSTM',
'status': temp_info,
'total': 2,
'finish': 2
})
try:
engines_output['LSTM'] = anomaly_LSTM(list_var, num_fut, desv_mse)
debug['LSTM'] = engines_output['LSTM']['debug']
temp_info['LSTM'] = engines_output['LSTM']
self.update_state(state='PROGRESS',
meta={
'running': 'VAR',
'status': temp_info,
'total': 2,
'finish': 1
})
print(engines_output['LSTM'])
except Exception as e:
print(e)
print('ERROR: exception executing LSTM')
try:
engines_output['VAR'] = anomaly_VAR(list_var, num_fut)
debug['VAR'] = engines_output['VAR']['debug']
temp_info['VAR'] = engines_output['VAR']
self.update_state(state='PROGRESS',
meta={
'running': 'VAR',
'status': temp_info,
'total': 2,
'finish': 2
})
print(engines_output['VAR'])
except Exception as e:
print(Exception)
print("type error: " + str(e))
print('ERROR: exception executing VAR')
try:
engines_output['VECM'] = anomaly_vecm(list_var, num_fut, desv_mse)
debug['VECM'] = engines_output['VECM']['debug']
temp_info['VECM'] = engines_output['VECM']
self.update_state(state='PROGRESS',
meta={
'running': 'VECM',
'status': temp_info,
'total': 2,
'finish': 1
})
print(engines_output['VECM'])
except Exception as e:
print(e)
print('ERROR: exception executing VECM')
best_mae = 999999999
winner = 'LSTM'
print('The size is ')
print(len(engines_output))
print(debug)
for key, value in engines_output.items():
print(key)
print(str(value['mae']))
if value['mae'] < best_mae:
print(key + " " + str(value['mae']) + " best:" + str(best_mae))
best_mae = value['mae']
winner = key
print("el ganador es " + winner)
temp = {}
temp['debug'] = debug
#return merge_two_dicts(engines_output[winner] , temp)
salida = merge_two_dicts(engines_output[winner], temp_info)
salida['winner'] = winner
salida['trend'] = trendline(list_var[0])
salida_temp = {}
salida_temp['status'] = salida
salida_temp['current'] = 100
salida_temp['total'] = 4
salida_temp['finish'] = 4
salida_temp['result'] = 'Task completed'
return salida_temp
def back_model_univariate(self, lista_datos, num_fut, desv_mse, train, name):
engines = {
'LSTM': 'anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)',
'VAR': 'anomaly_var(lista_datos,num_fut,desv_mse,train,name)',
'nbeats': 'anomaly_nbeats(lista_datos,num_fut,desv_mse,train,name)',
'gluonts': 'anomaly_gluonts(lista_datos,num_fut,desv_mse,train,name)',
'fbprophet':
'anomaly_fbprophet(lista_datos,num_fut,desv_mse,train,name)',
'arima': 'anomaly_AutoArima(lista_datos,num_fut,desv_mse,train,name)',
'Holtwinters': 'anomaly_holt(lista_datos_holt,num_fut,desv_mse,name)',
'tcn': 'anomaly_tcn(lista_datos,num_fut,desv_mse,train,name)'
}
engines_output = {}
debug = {}
temp_info = {}
starttime = datetime.now()
# Holtwinters workaround, must to solve
if (len(lista_datos) > 2000):
lista_datos_holt = lista_datos[len(lista_datos) - 2000:]
else:
lista_datos_holt = lista_datos
counter = 0
for engine_name, engine_function in engines.items():
self.update_state(state='PROGRESS',
meta={
'running': engine_name,
'status': temp_info,
'total': len(engines),
'finish': counter
})
try:
engines_output[engine_name] = eval(engine_function)
debug[engine_name] = engines_output[engine_name]['debug']
temp_info[engine_name] = engines_output[engine_name]
self.update_state(state='PROGRESS',
meta={
'running': engine_name,
'status': temp_info,
'total': 6,
'finish': 1
})
except Exception as e:
print('ERROR: ' + engine_name + ' univariate: ' + str(e))
counter = counter + 1
if (train):
best_mae = 999999999
winner = 'VAR'
print('The size is: ')
print(len(engines_output))
for key, value in engines_output.items():
print(key + " " + str(value['mae']))
if value['mae'] < best_mae:
best_mae = value['mae']
winner = key
print(winner)
db.new_model('winner_' + name, winner, pack('N', 365), '', 0)
print(winner)
# self.update_state(state='PROGRESS',
# meta={'running': 'LSTM',
# 'status': '',
# 'total': 6,
# 'finish': 0 })
# if not train:
#
# (model_name,model,params)=db.get_best_model('winner_'+name)
# # print ("recupero el motor " )
# winner= model_name
# if winner == 'LSTM':
# try:
# engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
# debug['LSTM'] = engines_output['LSTM']['debug']
# except Exception as e:
# print(e)
# print ('ERROR: exception executing LSTM univariate')
# elif winner == 'VAR':
# engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
# debug['VAR'] = engines_output['VAR']['debug']
# elif winner == 'Holtwinters':
# engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
# debug['Holtwinters'] = engines_output['Holtwinters']['debug']
# else:
# print ("Error")
#
# else:
#
#
#
#
# try:
# engines_output['nbeats'] = anomaly_nbeats(lista_datos,num_fut,desv_mse,train,name)
# debug['nbeats'] = engines_output['nbeats']['debug']
# temp_info['nbeats']=engines_output['nbeats']
# self.update_state(state='PROGRESS',
# meta={'running': 'nbeats',
# 'status': temp_info,
# 'total': 7,
# 'finish': 1})
# except Exception as e:
#
# print ('ERROR: nbeats univariate: ' + str(e) )
#
#
#
# try:
# engines_output['gluonts'] = anomaly_gluonts(lista_datos,num_fut,desv_mse,train,name)
# debug['gluonts'] = engines_output['gluonts']['debug']
# temp_info['gluonts']=engines_output['gluonts']
# self.update_state(state='PROGRESS',
# meta={'running': 'gluonts',
# 'status': temp_info,
# 'total': 6,
# 'finish': 1})
# except Exception as e:
#
# print ('ERROR: gluonts univariate: ' + str(e) )
#
#
# try:
# engines_output['fbprophet'] = anomaly_fbprophet(lista_datos,num_fut,desv_mse,train,name)
# debug['fbprophet'] = engines_output['fbprophet']['debug']
# temp_info['fbprophet']=engines_output['fbprophet']
# self.update_state(state='PROGRESS',
# meta={'running': 'fbprophet',
# 'status': temp_info,
# 'total': 6,
# 'finish': 2})
# except Exception as e:
#
# print ('ERROR: fbprophet univariate: ' + str(e) )
#
#
# try:
#
# engines_output['arima'] = anomaly_AutoArima(lista_datos,num_fut,desv_mse,train,name)
# debug['arima'] = engines_output['arima']['debug']
# temp_info['arima']=engines_output['arima']
# self.update_state(state='PROGRESS',
# meta={'running': 'VAR',
# 'status': temp_info,
# 'total': 6,
# 'finish': 3})
# except Exception as e:
# print(e)
# print ('ERROR: exception executing Autoarima')
#
# try:
# if (train):
# engines_output['VAR'] = univariate_anomaly_VAR(lista_datos,num_fut,name)
# debug['VAR'] = engines_output['VAR']['debug']
# else:
# engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
# debug['VAR'] = engines_output['VAR']['debug']
# temp_info['VAR'] = engines_output['VAR']
# self.update_state(state='PROGRESS',
# meta={'running': 'Holtwinters',
# 'status': temp_info,
# 'total': 6,
# 'finish': 4})
#
# except Exception as e:
# print(e)
# print ('ERROR: exception executing VAR')
#
# try:
# if (train ):
# if (len(lista_datos) > 2000):
# #new_length=
# lista_datos_holt=lista_datos[len(lista_datos)-2000:]
# else:
# lista_datos_holt = lista_datos
# engines_output['Holtwinters'] = anomaly_holt(lista_datos_holt,num_fut,desv_mse,name)
# debug['Holtwinters'] = engines_output['Holtwinters']['debug']
# else:
# print ("entra en forecast")
# if (len(lista_datos) > 2000):
# #new_length=
# lista_datos_holt=lista_datos[len(lista_datos)-2000:]
# else:
# lista_datos_holt = lista_datos
# engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
# debug['Holtwinters'] = engines_output['Holtwinters']['debug']
#
# temp_info['Holtwinters'] = engines_output['Holtwinters']
# self.update_state(state='PROGRESS',
# meta={'running': 'Holtwinters',
# 'status': temp_info,
# 'total': 6,
# 'finish': 5})
#
# except Exception as e:
# print(e)
# print ('ERROR: exception executing Holtwinters')
#
# try:
# engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
# debug['LSTM'] = engines_output['LSTM']['debug']
# temp_info['LSTM']=engines_output['LSTM']
# self.update_state(state='PROGRESS',
# meta={'running': 'anomaly_AutoArima',
# 'status': temp_info,
# 'total': 6,
# 'finish': 6})
# except Exception as e:
# print(e)
# print ('ERROR: exception executing LSTM univariate')
#
#
# best_mae=999999999
# winner='VAR'
# print ('The size is: ')
# print (len(engines_output))
# for key, value in engines_output.items():
# print (key + " " + str(value['mae']))
#
# if value['mae'] < best_mae:
# best_mae=value['mae']
# winner=key
# print(winner)
#
# db.new_model('winner_'+name, winner, pack('N', 365),'',0)
#
#
# print (winner)
print("el ganador es " + str(winner))
print(engines_output[winner])
temp = {}
temp['debug'] = debug
temp['trend'] = trendline(lista_datos)
# return merge_two_dicts(engines_output[winner] , temp)
salida = merge_two_dicts(engines_output[winner], temp_info)
finishtime = datetime.now()
diff_time = finishtime - starttime
salida['time'] = diff_time.total_seconds()
salida['changepoint'] = find_changepoints(lista_datos)
salida['winner'] = winner
salida['trend'] = trendline(lista_datos)
salida_temp = {}
salida_temp['status'] = salida
salida_temp['current'] = 100
salida_temp['total'] = 5
salida_temp['finish'] = 5
salida_temp['result'] = 'Task completed'
# insert json output to mongodb
import pymongo
from pymongo import MongoClient
import os
import pandas as pd
import numpy as np
timecop_backend = os.getenv('mongodb_backend')
if timecop_backend != None:
client = MongoClient(timecop_backend)
# database
mongo_db = client["ts"]
timecop_db = mongo_db["timecop"]
# data_dict = resultado.to_dict("records")
lista_puntos = np.arange(0, len(lista_datos), 1)
df = pd.DataFrame(list(zip(lista_puntos, lista_datos)),
columns=['step', 'value'])
timecop_db.insert_one({
"name": name,
"data": salida_temp,
"ts": df.to_dict(orient='record')
})
return salida_temp
def model_univariate(self, lista_datos, num_fut, desv_mse, train, name):
engines_output = {}
debug = {}
if not train:
# filename = './models_temp/'+name
# with open(filename,'r') as f:
# winner = f.read()
# f.close()
(model_name, model, params) = get_best_model('winner_' + name)
# print ("recupero el motor " )
winner = model_name
if winner == 'LSTM':
try:
engines_output['LSTM'] = anomaly_uni_LSTM(
lista_datos, num_fut, desv_mse, train, name)
debug['LSTM'] = engines_output['LSTM']['debug']
except Exception as e:
print(e)
print('ERROR: exception executing LSTM univariate')
elif winner == 'VAR':
engines_output['VAR'] = univariate_forecast_VAR(
lista_datos, num_fut, name)
debug['VAR'] = engines_output['VAR']['debug']
elif winner == 'Holtwinters':
engines_output['Holtwinters'] = forecast_holt(
lista_datos, num_fut, desv_mse, name)
debug['Holtwinters'] = engines_output['Holtwinters']['debug']
else:
print("Error")
else:
try:
engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos, num_fut,
desv_mse, train, name)
debug['LSTM'] = engines_output['LSTM']['debug']
except Exception as e:
print(e)
print('ERROR: exception executing LSTM univariate:' + str(e))
try:
engines_output['fbprophet'] = anomaly_fbprophet(
lista_datos, num_fut, desv_mse, train, name)
debug['fbprophet'] = engines_output['fbprophet']['debug']
except Exception as e:
print('ERROR: fbprophet univariate: ' + str(e))
try:
if (len(lista_datos) > 100):
##new_length=
lista_datos_ari = lista_datos[len(lista_datos) - 100:]
engines_output['arima'] = anomaly_AutoArima(
lista_datos_ari, num_fut, len(lista_datos), desv_mse)
debug['arima'] = engines_output['arima']['debug']
except Exception as e:
print('ERROR: exception executing Autoarima: ' + str(e))
try:
if (train):
engines_output['VAR'] = univariate_anomaly_VAR(
lista_datos, num_fut, name)
debug['VAR'] = engines_output['VAR']['debug']
else:
engines_output['VAR'] = univariate_forecast_VAR(
lista_datos, num_fut, name)
debug['VAR'] = engines_output['VAR']['debug']
except Exception as e:
print(e)
print('ERROR: exception executing VAR: ' + str(e))
try:
if (train):
engines_output['Holtwinters'] = anomaly_holt(
lista_datos, num_fut, desv_mse, name)
debug['Holtwinters'] = engines_output['Holtwinters']['debug']
else:
print("entra en forecast")
engines_output['Holtwinters'] = forecast_holt(
lista_datos, num_fut, desv_mse, name)
debug['Holtwinters'] = engines_output['Holtwinters']['debug']
except Exception as e:
print(e)
print('ERROR: exception executing Holtwinters: ' + str(e))
best_mae = 999999999
winner = 'VAR'
print('The size is: ')
print(len(engines_output))
for key, value in engines_output.items():
print(key + " " + str(value['mae']))
if value['mae'] < best_mae:
best_mae = value['mae']
winner = key
print(winner)
# filename = './models_temp/'+name
# with open(filename,'w') as f:
# f.write(winner)
# f.close()
new_model('winner_' + name, winner, pack('N', 365), '', 0)
print(winner)
print("el ganador es " + str(winner))
print(engines_output[winner])
temp = {}
temp['debug'] = debug
return merge_two_dicts(engines_output[winner], temp)
def back_model_univariate(self, lista_datos,num_fut,desv_mse,train,name):
engines_output={}
debug = {}
temp_info = {}
self.update_state(state='PROGRESS',
meta={'running': 'LSTM',
'status': '',
'total': 4,
'finish': 0 })
if not train:
(model_name,model,params)=db.get_best_model('winner_'+name)
# print ("recupero el motor " )
winner= model_name
if winner == 'LSTM':
try:
engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
debug['LSTM'] = engines_output['LSTM']['debug']
except Exception as e:
print(e)
print ('ERROR: exception executing LSTM univariate')
elif winner == 'VAR':
engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
debug['VAR'] = engines_output['VAR']['debug']
elif winner == 'Holtwinters':
engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
debug['Holtwinters'] = engines_output['Holtwinters']['debug']
else:
print ("Error")
else:
try:
engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
debug['LSTM'] = engines_output['LSTM']['debug']
temp_info['LSTM']=engines_output['LSTM']
self.update_state(state='PROGRESS',
meta={'running': 'anomaly_AutoArima',
'status': temp_info,
'total': 4,
'finish': 1})
except Exception as e:
print(e)
print ('ERROR: exception executing LSTM univariate')
try:
if (len(lista_datos) > 100):
#new_length=
lista_datos_ari=lista_datos[len(lista_datos)-100:]
engines_output['arima'] = anomaly_AutoArima(lista_datos_ari,num_fut,len(lista_datos),desv_mse)
debug['arima'] = engines_output['arima']['debug']
temp_info['arima']=engines_output['arima']
self.update_state(state='PROGRESS',
meta={'running': 'VAR',
'status': temp_info,
'total': 4,
'finish': 2})
except Exception as e:
print(e)
print ('ERROR: exception executing Autoarima')
try:
if (train):
engines_output['VAR'] = univariate_anomaly_VAR(lista_datos,num_fut,name)
debug['VAR'] = engines_output['VAR']['debug']
else:
engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
debug['VAR'] = engines_output['VAR']['debug']
temp_info['VAR'] = engines_output['VAR']
self.update_state(state='PROGRESS',
meta={'running': 'Holtwinters',
'status': temp_info,
'total': 4,
'finish': 3})
except Exception as e:
print(e)
print ('ERROR: exception executing VAR')
try:
if (train ):
engines_output['Holtwinters'] = anomaly_holt(lista_datos,num_fut,desv_mse,name)
debug['Holtwinters'] = engines_output['Holtwinters']['debug']
else:
print ("entra en forecast")
engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
debug['Holtwinters'] = engines_output['Holtwinters']['debug']
temp_info['Holtwinters'] = engines_output['Holtwinters']
self.update_state(state='PROGRESS',
meta={'running': 'Holtwinters',
'status': temp_info,
'total': 4,
'finish': 4})
except Exception as e:
print(e)
print ('ERROR: exception executing Holtwinters')
best_mae=999999999
winner='VAR'
print ('The size is: ')
print (len(engines_output))
for key, value in engines_output.items():
print (key + " " + str(value['mae']))
if value['mae'] < best_mae:
best_mae=value['mae']
winner=key
print(winner)
db.new_model('winner_'+name, winner, pack('N', 365),'',0)
print (winner)
print ("el ganador es " + str(winner))
print (engines_output[winner])
temp= {}
temp['debug']=debug
# return merge_two_dicts(engines_output[winner] , temp)
salida = merge_two_dicts(engines_output[winner], temp_info)
salida['winner'] = winner
salida_temp= {}
salida_temp['status'] = salida
salida_temp['current'] = 100
salida_temp['total']=4
salida_temp['finish'] =4
salida_temp['result'] ='Task completed'
return salida_temp