def wrapper(*args, **kwargs): try: ret = func(*args, **kwargs) db.session().commit() return ret except Exception as e: logger.exception("Exception:") db.session().rollback() return (500, str(e))
def get_searcher(searcher_id): session = db.session() searcher = session.query(db.ParamSearcher).get(searcher_id) if searcher: return create_response(searcher_to_dict(searcher)) else: return create_response({}, 404, err='not found')
def create_dataset(): name = request.params.name description = request.params.description explanatory_column_ids = json.loads(request.params.explanatory_column_ids) target_column_ids = json.loads(request.params.target_column_ids) selected_scaling = int(request.params.selected_scaling) labels = json.loads(request.params.labels) train_ratio = float(request.params.train_ratio) train_index = json.loads(request.params.train_index) valid_index = json.loads(request.params.valid_index) true_histogram = json.loads(request.params.true_histogram) filename_y = request.params.filename_y filename_X = request.params.filename_X dataset = db.DatasetDef( name=name, description=description, explanatory_column_ids=pickle.dumps(explanatory_column_ids), target_column_ids=pickle.dumps(target_column_ids), labels=pickle.dumps(labels), train_ratio=train_ratio, train_index=pickle.dumps(train_index), valid_index=pickle.dumps(valid_index), true_histogram=pickle.dumps(true_histogram), selected_scaling=selected_scaling, filename_y=filename_y, filename_X=filename_X) session = db.session() session.add(dataset) session.commit() return create_response({'dataset': _dataset_to_dict(dataset)})
def get_deployed_model_info(): model = db.session().query(db.Model).filter(db.Model.deployed == 1).one() if model: model_dict = _model_to_dict(model) model_dict["explanatory_column_ids"] = pickle.loads( model.dataset.explanatory_column_ids) return create_response(model_dict) else: return create_response({}, 404, err='not found')
def delete_model(model_id): session = db.session() session.query(db.ParamSearcherModel).filter_by(model_id=model_id).delete() q = session.query(db.Model).filter_by(id=model_id) n = q.delete() if n: session.commit() return create_response({}) else: return create_response({}, 404, err='model not found')
def create_searcher(): info = json.loads(request.params.info) searcher = db.ParamSearcher(info=pickle.dumps(info)) session = db.session() session.add(searcher) session.commit() return create_response({'id': searcher.id})
def delete_searcher(searcher_id): session = db.session() session.query( db.ParamSearcherModel).filter_by(searcher_id=searcher_id).delete() n = session.query(db.ParamSearcher).filter_by(id=searcher_id).delete() if n: session.commit() return create_response({}) else: return create_response({}, 404, err='searcher not found')
def undeploy_model(model_id): session = db.session() model = session.query(db.Model).get(model_id) if model: model.deployed = 0 session.add(model) session.commit() return create_response({'model': _model_to_dict(model)}) else: return create_response({}, 404, err='not found')
def create_model(): session = db.session() searcher = None searcher_id = request.params.get('searcher_id', None) if searcher_id is not None: searcher = (session.query( db.ParamSearcher).filter_by(id=int(searcher_id)).one()) dataset_id = int(request.params.dataset_id) algorithm = int(request.params.algorithm) algorithm_params = json.loads(request.params.algorithm_params) algorithm_params['num_neighbors'] = int(algorithm_params['num_neighbors']) batch_size = int(request.params.batch_size) epoch = int(request.params.epoch) try: # if user input file is not exists if algorithm_params['script_file_name'] is not '': script = algorithm_params['script_file_name'] scriptdir = os.path.abspath(SCRIPT_DIR) if not scriptdir.endswith(os.path.sep): scriptdir += os.path.sep scriptfile = os.path.abspath(os.path.join(scriptdir, script)) if not scriptfile.startswith(scriptdir): raise ValueError('Invalid script name: %s' % scriptfile) elif not os.path.exists(scriptfile): raise ValueError('Invalid script name: %s' % scriptfile) print(algorithm_params['script_file_name']) model = db.Model(dataset_id=dataset_id, algorithm=algorithm, algorithm_params=pickle.dumps(algorithm_params), batch_size=batch_size, epoch=epoch) session.add(model) session.commit() if searcher: searchermodel = db.ParamSearcherModel(searcher=searcher, model=model) session.add(searchermodel) session.commit() return create_response(_model_to_dict(model)) except Exception as e: traceback.print_exc() return create_response({}, 500, err=str(e))
def train_model(model_id): model = db.session().query(db.Model).get(model_id) if not model.best_epoch_r2: if not model: return create_response({}, 404, err='model not found') taskstate = train_task.TaskState.add_task(model) executor = Executor() submit_task(executor, train_task.train, taskstate, model.id) if renom.cuda.has_cuda(): release_mem_pool() return create_response({'result': 'ok'})
def get_deployed_model_weight(): model = db.session().query(db.Model).filter(db.Model.deployed == 1).one() if model: if model.weight: file_name = model.weight return static_file(file_name, root=DB_DIR_TRAINED_WEIGHT, download='deployed_model.h5') elif model.model_pickle: m_file_name = model.model_pickle return static_file(m_file_name, root=DB_DIR_ML_MODELS, download='deployed_model.pickle') else: return create_response({}, 404, err='model not found')
def delete_dataset(dataset_id): session = db.session() ds = session.query(db.DatasetDef).get(dataset_id) if ds: q = session.query( db.Model.id).filter(db.Model.dataset_id == dataset_id) (session.query(db.ParamSearcherModel).filter( db.ParamSearcherModel.model_id.in_(q)).delete( synchronize_session=False)) session.query( db.Model).filter(db.Model.dataset_id == dataset_id).delete() session.delete(ds) session.commit() return create_response({}) else: return create_response({}, 404, err='dataset not found')
def get_datasets(): q = db.session().query(db.DatasetDef).all() ret = [_dataset_to_dict(ds) for ds in q] return create_response({'datasets': ret})
def get_searchers(): session = db.session() searchers = session.query(db.ParamSearcher) dicts = [searcher_to_dict(searcher) for searcher in searchers] return create_response({'searchers': dicts})
def get_models(): q = db.session().query(db.Model).order_by(db.Model.id.desc()).all() ret = [_model_to_dict(model) for model in q] return create_response({'models': ret})
def get_model(model_id): model = db.session().query(db.Model).get(model_id) if model: return create_response(_model_to_dict(model)) else: return create_response({}, 404, err='not found')
def get_dataset(dataset_id): ds = db.session().query(db.DatasetDef).get(dataset_id) if ds: return create_response({'dataset': _dataset_to_dict(ds)}) else: return create_response({}, 404, err='not found')
def predict_model(model_id): explanatory_column = request.params.explanatory_column target_column = request.params.target_column explanatory_column_ids = json.loads(request.params.explanatory_column_ids) model = db.session().query(db.Model).get(model_id) if not model: return create_response({}, 404, err='model not found') try: with open(os.path.join(DATASRC_DIR, 'prediction_set', 'pred.pickle'), mode='rb') as f: p_all_data = pickle.load(f) p_X = split_target(p_all_data, pickle.loads(model.dataset.labels), explanatory_column_ids) n_X = np.array(p_X) selected_scaling = model.dataset.selected_scaling if selected_scaling != 1: filename_X = model.dataset.filename_X n_X_scaling = scaling_again(n_X, filename_X) else: n_X_scaling = n_X except Exception as e: traceback.print_exc() return create_response({}, 404, err=str(e)) executor = Executor() f = submit_task(executor, pred_task.prediction, model.id, n_X_scaling) try: result = f.result() if selected_scaling != 1: filename_y = model.dataset.filename_y result = re_scaling(result, filename_y) CSV_DIR = os.path.join(DATASRC_PREDICTION_OUT, 'csv') if not os.path.isdir(CSV_DIR): os.makedirs(CSV_DIR) now = datetime.datetime.now() filename = 'model' + str(model_id) + '_{0:%Y%m%d%H%M%S}'.format( now) + '.csv' filepath = os.path.join(CSV_DIR, filename) target_labels = target_column.split(',') explanatory_labels = explanatory_column.split(',') labels = target_labels + explanatory_labels np_xy = np.round(np.c_[result, n_X], 3) pred_x_y = pd.DataFrame(np_xy) pred_x_y.columns = labels pred_x_y.to_csv(filepath, index=False) SAMPLING_SIZE = 1000 sampled_n_X = [] sampled_result = [] p = np.random.permutation(n_X.shape[0]) n_X = n_X.tolist() result = result.tolist() sampled_p = p[:SAMPLING_SIZE] for sp_n in sampled_p: sampled_n_X.append(n_X[sp_n]) sampled_result.append(result[sp_n]) body = { 'pred_x': n_X, 'pred_y': result, 'sampled_pred_x': sampled_n_X, 'sampled_pred_y': sampled_result, 'pred_csv': filename } return create_response(body) except Exception as e: traceback.print_exc() return create_response({}, 404, err=str(e)) finally: if renom.cuda.has_cuda(): release_mem_pool()