예제 #1
0
def back_univariate_engine():
    db.init_database()

    if not request.json:
        abort(400)

    timedata = request.get_json()
    print(timedata)
    lista = timedata['data']

    num_fut = int(timedata.get('num_future', 5))
    desv_mae = int(timedata.get('desv_metric', 2))
    name = timedata.get('name', 'NA')
    train = timedata.get('train', True)
    restart = timedata.get('restart', False)

    print("train?" + str(train))
    print("restart?" + str(restart))
    print("Received TS")

    if (name != 'NA'):
        filename = './lst/' + name + '.lst'
        try:
            # with open(filename, 'r') as filehandle:
            #     previousList = json.load(filehandle)
            previousList = db.get_ts(name).split(',')
            previousList = list(map(int, previousList))
        except Exception:
            previousList = []
        print("previous list")

        if not restart:
            print("Lista append")
            lista = previousList + lista
        # with open(filename, 'w') as filehandle:
        #     json.dump(lista,filehandle)
        str_lista = ",".join(str(v) for v in lista)
        db.set_ts(name, str_lista)

    #desv_mse = 0
    print("la lista al final es " + str(type(lista)))
    print(lista)
    print(name)

    print("invoco el backend")
    salida = back_model_univariate.s(lista_datos=lista,
                                     num_fut=num_fut,
                                     desv_mse=desv_mae,
                                     train=train,
                                     name=name).apply_async()

    print(salida.id)

    #task = long_task.apply_async()
    valor = {'task_id': salida.id}
    return jsonify(valor), 200
예제 #2
0
def univariate_engine():
    if not request.json:
        abort(400)

    timedata = request.get_json()
    print(timedata)
    lista = timedata['data']

    num_fut = int(timedata.get('num_future', 5))
    desv_mae = int(timedata.get('desv_metric', 2))
    name = timedata.get('name', 'NA')
    train = timedata.get('train', True)
    restart = timedata.get('restart', False)

    print("train?" + str(train))
    print("restart?" + str(restart))
    print("Received TS")

    if (name != 'NA'):
        filename = './lst/' + name + '.lst'
        try:
            with open(filename, 'r') as filehandle:
                previousList = json.load(filehandle)
            previousList = db.get_ts(name).split(',')
            previousList = list(map(int, previousList))
        except Exception:
            previousList = []
        print("previous list")

        if not restart:
            print("Lista append")
            lista = previousList + lista
        with open(filename, 'w') as filehandle:
            json.dump(lista, filehandle)
        str_lista = ",".join(str(v) for v in lista)
        db.set_ts(name, str_lista)

    #desv_mse = 0
    print("la lista al final es " + str(type(lista)))
    print(lista)

    salida = ft.model_univariate(lista, num_fut, desv_mae, train, name)

    return jsonify(salida), 201
예제 #3
0
def monitoring_winners():
    model_name = request.args.get('model_name', default='%', type=str)
    data_models = db.get_winners(model_name)
    return jsonify(data_models.to_dict(orient='record')), 201
예제 #4
0
def back_model_univariate(self, lista_datos, num_fut, desv_mse, train, name):
    engines = {
        'LSTM': 'anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)',
        'VAR': 'anomaly_var(lista_datos,num_fut,desv_mse,train,name)',
        'nbeats': 'anomaly_nbeats(lista_datos,num_fut,desv_mse,train,name)',
        'gluonts': 'anomaly_gluonts(lista_datos,num_fut,desv_mse,train,name)',
        'fbprophet':
        'anomaly_fbprophet(lista_datos,num_fut,desv_mse,train,name)',
        'arima': 'anomaly_AutoArima(lista_datos,num_fut,desv_mse,train,name)',
        'Holtwinters': 'anomaly_holt(lista_datos_holt,num_fut,desv_mse,name)',
        'tcn': 'anomaly_tcn(lista_datos,num_fut,desv_mse,train,name)'
    }

    engines_output = {}
    debug = {}

    temp_info = {}

    starttime = datetime.now()

    # Holtwinters workaround, must to solve
    if (len(lista_datos) > 2000):

        lista_datos_holt = lista_datos[len(lista_datos) - 2000:]
    else:
        lista_datos_holt = lista_datos

    counter = 0
    for engine_name, engine_function in engines.items():
        self.update_state(state='PROGRESS',
                          meta={
                              'running': engine_name,
                              'status': temp_info,
                              'total': len(engines),
                              'finish': counter
                          })
        try:
            engines_output[engine_name] = eval(engine_function)
            debug[engine_name] = engines_output[engine_name]['debug']
            temp_info[engine_name] = engines_output[engine_name]
            self.update_state(state='PROGRESS',
                              meta={
                                  'running': engine_name,
                                  'status': temp_info,
                                  'total': 6,
                                  'finish': 1
                              })
        except Exception as e:

            print('ERROR: ' + engine_name + ' univariate: ' + str(e))
        counter = counter + 1

        if (train):

            best_mae = 999999999
            winner = 'VAR'
            print('The size is: ')
            print(len(engines_output))
            for key, value in engines_output.items():
                print(key + "   " + str(value['mae']))

                if value['mae'] < best_mae:
                    best_mae = value['mae']
                    winner = key
                print(winner)

            db.new_model('winner_' + name, winner, pack('N', 365), '', 0)

            print(winner)

    # self.update_state(state='PROGRESS',
    #                   meta={'running': 'LSTM',
    #                         'status': '',
    #                         'total': 6,
    #                         'finish': 0 })
    # if not train:
    #
    #     (model_name,model,params)=db.get_best_model('winner_'+name)
    #     # print ("recupero el motor " )
    #     winner= model_name
    #     if winner == 'LSTM':
    #         try:
    #             engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
    #             debug['LSTM'] = engines_output['LSTM']['debug']
    #         except Exception as e:
    #             print(e)
    #             print ('ERROR: exception executing LSTM univariate')
    #     elif winner == 'VAR':
    #         engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
    #         debug['VAR'] = engines_output['VAR']['debug']
    #     elif winner == 'Holtwinters':
    #        engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
    #        debug['Holtwinters'] = engines_output['Holtwinters']['debug']
    #     else:
    #         print ("Error")
    #
    # else:
    #
    #
    #
    #
    #     try:
    #         engines_output['nbeats'] = anomaly_nbeats(lista_datos,num_fut,desv_mse,train,name)
    #         debug['nbeats'] = engines_output['nbeats']['debug']
    #         temp_info['nbeats']=engines_output['nbeats']
    #         self.update_state(state='PROGRESS',
    #                   meta={'running': 'nbeats',
    #                         'status': temp_info,
    #                         'total': 7,
    #                         'finish': 1})
    #     except Exception as e:
    #
    #         print ('ERROR: nbeats univariate: ' + str(e) )
    #
    #
    #
    #     try:
    #         engines_output['gluonts'] = anomaly_gluonts(lista_datos,num_fut,desv_mse,train,name)
    #         debug['gluonts'] = engines_output['gluonts']['debug']
    #         temp_info['gluonts']=engines_output['gluonts']
    #         self.update_state(state='PROGRESS',
    #                   meta={'running': 'gluonts',
    #                         'status': temp_info,
    #                         'total': 6,
    #                         'finish': 1})
    #     except Exception as e:
    #
    #         print ('ERROR: gluonts univariate: ' + str(e) )
    #
    #
    #     try:
    #         engines_output['fbprophet'] = anomaly_fbprophet(lista_datos,num_fut,desv_mse,train,name)
    #         debug['fbprophet'] = engines_output['fbprophet']['debug']
    #         temp_info['fbprophet']=engines_output['fbprophet']
    #         self.update_state(state='PROGRESS',
    #                   meta={'running': 'fbprophet',
    #                         'status': temp_info,
    #                         'total': 6,
    #                         'finish': 2})
    #     except Exception as e:
    #
    #         print ('ERROR: fbprophet univariate: ' + str(e) )
    #
    #
    #     try:
    #
    #         engines_output['arima'] = anomaly_AutoArima(lista_datos,num_fut,desv_mse,train,name)
    #         debug['arima'] = engines_output['arima']['debug']
    #         temp_info['arima']=engines_output['arima']
    #         self.update_state(state='PROGRESS',
    #                   meta={'running': 'VAR',
    #                         'status': temp_info,
    #                         'total': 6,
    #                         'finish': 3})
    #     except  Exception as e:
    #         print(e)
    #         print ('ERROR: exception executing Autoarima')
    #
    #     try:
    #         if (train):
    #             engines_output['VAR'] = univariate_anomaly_VAR(lista_datos,num_fut,name)
    #             debug['VAR'] = engines_output['VAR']['debug']
    #         else:
    #             engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
    #             debug['VAR'] = engines_output['VAR']['debug']
    #         temp_info['VAR'] = engines_output['VAR']
    #         self.update_state(state='PROGRESS',
    #                   meta={'running': 'Holtwinters',
    #                         'status': temp_info,
    #                         'total': 6,
    #                         'finish': 4})
    #
    #     except  Exception as e:
    #         print(e)
    #         print ('ERROR: exception executing VAR')
    #
    #     try:
    #         if (train ):
    #                 if (len(lista_datos) > 2000):
    #                     #new_length=
    #                     lista_datos_holt=lista_datos[len(lista_datos)-2000:]
    #                 else:
    #                     lista_datos_holt = lista_datos
    #                 engines_output['Holtwinters'] = anomaly_holt(lista_datos_holt,num_fut,desv_mse,name)
    #                 debug['Holtwinters'] = engines_output['Holtwinters']['debug']
    #         else:
    #                print ("entra en forecast")
    #                if (len(lista_datos) > 2000):
    #                    #new_length=
    #                    lista_datos_holt=lista_datos[len(lista_datos)-2000:]
    #                else:
    #                    lista_datos_holt = lista_datos
    #                engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
    #                debug['Holtwinters'] = engines_output['Holtwinters']['debug']
    #
    #         temp_info['Holtwinters'] = engines_output['Holtwinters']
    #         self.update_state(state='PROGRESS',
    #                   meta={'running': 'Holtwinters',
    #                         'status': temp_info,
    #                         'total': 6,
    #                         'finish': 5})
    #
    #     except  Exception as e:
    #            print(e)
    #            print ('ERROR: exception executing Holtwinters')
    #
    #     try:
    #         engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
    #         debug['LSTM'] = engines_output['LSTM']['debug']
    #         temp_info['LSTM']=engines_output['LSTM']
    #         self.update_state(state='PROGRESS',
    #                   meta={'running': 'anomaly_AutoArima',
    #                         'status': temp_info,
    #                         'total': 6,
    #                         'finish': 6})
    #     except Exception as e:
    #         print(e)
    #         print ('ERROR: exception executing LSTM univariate')
    #
    #
    #     best_mae=999999999
    #     winner='VAR'
    #     print ('The size is: ')
    #     print (len(engines_output))
    #     for key, value in engines_output.items():
    #         print (key + "   " + str(value['mae']))
    #
    #         if value['mae'] < best_mae:
    #             best_mae=value['mae']
    #             winner=key
    #         print(winner)
    #
    #     db.new_model('winner_'+name, winner, pack('N', 365),'',0)
    #
    #
    #     print (winner)

    print("el ganador es " + str(winner))
    print(engines_output[winner])
    temp = {}
    temp['debug'] = debug
    temp['trend'] = trendline(lista_datos)

    #    return merge_two_dicts(engines_output[winner] , temp)
    salida = merge_two_dicts(engines_output[winner], temp_info)
    finishtime = datetime.now()
    diff_time = finishtime - starttime
    salida['time'] = diff_time.total_seconds()
    salida['changepoint'] = find_changepoints(lista_datos)
    salida['winner'] = winner
    salida['trend'] = trendline(lista_datos)
    salida_temp = {}
    salida_temp['status'] = salida
    salida_temp['current'] = 100
    salida_temp['total'] = 5
    salida_temp['finish'] = 5
    salida_temp['result'] = 'Task completed'

    # insert json output to mongodb
    import pymongo
    from pymongo import MongoClient
    import os
    import pandas as pd
    import numpy as np

    timecop_backend = os.getenv('mongodb_backend')
    if timecop_backend != None:
        client = MongoClient(timecop_backend)
        # database
        mongo_db = client["ts"]
        timecop_db = mongo_db["timecop"]
        # data_dict = resultado.to_dict("records")
        lista_puntos = np.arange(0, len(lista_datos), 1)
        df = pd.DataFrame(list(zip(lista_puntos, lista_datos)),
                          columns=['step', 'value'])
        timecop_db.insert_one({
            "name": name,
            "data": salida_temp,
            "ts": df.to_dict(orient='record')
        })

    return salida_temp
예제 #5
0
from engines.nbeats import anomaly_nbeats
from engines.vecm import anomaly_vecm
from engines.tcn import anomaly_tcn

from engines.changepointdetection import find_changepoints

from datetime import datetime

from struct import *

app = Flask(__name__)
CORS(app)

app.config.from_pyfile(os.path.join(".", "config/app.cfg"), silent=False)

db.init_database()

DB_NAME = app.config.get("DB_NAME")
PORT = app.config.get("PORT")

# Celery configuration
app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0'

# Initialize Celery
celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
#broker=main.config['CELERY_BROKER_URL'])

celery.conf.update(app.config)

예제 #6
0
파일: app.py 프로젝트: triplekill/timecop
def back_model_univariate(self, lista_datos,num_fut,desv_mse,train,name):
    engines_output={}
    debug = {}

    temp_info = {}

    self.update_state(state='PROGRESS',
                      meta={'running': 'LSTM',
                            'status': '',
                            'total': 4,
                            'finish': 0 })
    if not train:

        (model_name,model,params)=db.get_best_model('winner_'+name)
        # print ("recupero el motor " )
        winner= model_name
        if winner == 'LSTM':
            try:
                engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
                debug['LSTM'] = engines_output['LSTM']['debug']
            except Exception as e:
                print(e)
                print ('ERROR: exception executing LSTM univariate')
        elif winner == 'VAR':
            engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
            debug['VAR'] = engines_output['VAR']['debug']
        elif winner == 'Holtwinters':
           engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
           debug['Holtwinters'] = engines_output['Holtwinters']['debug']
        else:
            print ("Error")

    else:

        try:
            engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train,name)
            debug['LSTM'] = engines_output['LSTM']['debug']
            temp_info['LSTM']=engines_output['LSTM']
            self.update_state(state='PROGRESS',
                      meta={'running': 'anomaly_AutoArima',
                            'status': temp_info,
                            'total': 4,
                            'finish': 1})
        except Exception as e:
            print(e)
            print ('ERROR: exception executing LSTM univariate')


        try:
            if (len(lista_datos) > 100):
                #new_length=
                lista_datos_ari=lista_datos[len(lista_datos)-100:]
            engines_output['arima'] = anomaly_AutoArima(lista_datos_ari,num_fut,len(lista_datos),desv_mse)
            debug['arima'] = engines_output['arima']['debug']
            temp_info['arima']=engines_output['arima']
            self.update_state(state='PROGRESS',
                      meta={'running': 'VAR',
                            'status': temp_info,
                            'total': 4,
                            'finish': 2})
        except  Exception as e:
            print(e)
            print ('ERROR: exception executing Autoarima')

        try:
            if (train):
                engines_output['VAR'] = univariate_anomaly_VAR(lista_datos,num_fut,name)
                debug['VAR'] = engines_output['VAR']['debug']
            else:
                engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
                debug['VAR'] = engines_output['VAR']['debug']
            temp_info['VAR'] = engines_output['VAR']
            self.update_state(state='PROGRESS',
                      meta={'running': 'Holtwinters',
                            'status': temp_info,
                            'total': 4,
                            'finish': 3})

        except  Exception as e:
            print(e)
            print ('ERROR: exception executing VAR')

        try:
            if (train ):
                   engines_output['Holtwinters'] = anomaly_holt(lista_datos,num_fut,desv_mse,name)
                   debug['Holtwinters'] = engines_output['Holtwinters']['debug']
            else:
                   print ("entra en forecast")
                   engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
                   debug['Holtwinters'] = engines_output['Holtwinters']['debug']

            temp_info['Holtwinters'] = engines_output['Holtwinters']
            self.update_state(state='PROGRESS',
                      meta={'running': 'Holtwinters',
                            'status': temp_info,
                            'total': 4,
                            'finish': 4})

        except  Exception as e:
               print(e)
               print ('ERROR: exception executing Holtwinters')


        best_mae=999999999
        winner='VAR'
        print ('The size is: ')
        print (len(engines_output))
        for key, value in engines_output.items():
            print (key + "   " + str(value['mae']))

            if value['mae'] < best_mae:
                best_mae=value['mae']
                winner=key
            print(winner)

        db.new_model('winner_'+name, winner, pack('N', 365),'',0)


        print (winner)

    print ("el ganador es " + str(winner))
    print (engines_output[winner])
    temp= {}
    temp['debug']=debug

#    return merge_two_dicts(engines_output[winner] , temp)
    salida = merge_two_dicts(engines_output[winner], temp_info)
    salida['winner'] = winner
    salida_temp= {}
    salida_temp['status'] = salida
    salida_temp['current'] = 100
    salida_temp['total']=4
    salida_temp['finish'] =4
    salida_temp['result'] ='Task completed'

    return  salida_temp