def wrapper(*args, **kwargs):
try:
ret = func(*args, **kwargs)
db.session().commit()
return ret
except Exception as e:
logger.exception("Exception:")
db.session().rollback()
return (500, str(e))
def get_searcher(searcher_id):
session = db.session()
searcher = session.query(db.ParamSearcher).get(searcher_id)
if searcher:
return create_response(searcher_to_dict(searcher))
else:
return create_response({}, 404, err='not found')
def create_dataset():
name = request.params.name
description = request.params.description
explanatory_column_ids = json.loads(request.params.explanatory_column_ids)
target_column_ids = json.loads(request.params.target_column_ids)
selected_scaling = int(request.params.selected_scaling)
labels = json.loads(request.params.labels)
train_ratio = float(request.params.train_ratio)
train_index = json.loads(request.params.train_index)
valid_index = json.loads(request.params.valid_index)
true_histogram = json.loads(request.params.true_histogram)
filename_y = request.params.filename_y
filename_X = request.params.filename_X
dataset = db.DatasetDef(
name=name,
description=description,
explanatory_column_ids=pickle.dumps(explanatory_column_ids),
target_column_ids=pickle.dumps(target_column_ids),
labels=pickle.dumps(labels),
train_ratio=train_ratio,
train_index=pickle.dumps(train_index),
valid_index=pickle.dumps(valid_index),
true_histogram=pickle.dumps(true_histogram),
selected_scaling=selected_scaling,
filename_y=filename_y,
filename_X=filename_X)
session = db.session()
session.add(dataset)
session.commit()
return create_response({'dataset': _dataset_to_dict(dataset)})
def get_deployed_model_info():
model = db.session().query(db.Model).filter(db.Model.deployed == 1).one()
if model:
model_dict = _model_to_dict(model)
model_dict["explanatory_column_ids"] = pickle.loads(
model.dataset.explanatory_column_ids)
return create_response(model_dict)
else:
return create_response({}, 404, err='not found')
def delete_model(model_id):
session = db.session()
session.query(db.ParamSearcherModel).filter_by(model_id=model_id).delete()
q = session.query(db.Model).filter_by(id=model_id)
n = q.delete()
if n:
session.commit()
return create_response({})
else:
return create_response({}, 404, err='model not found')
def create_searcher():
info = json.loads(request.params.info)
searcher = db.ParamSearcher(info=pickle.dumps(info))
session = db.session()
session.add(searcher)
session.commit()
return create_response({'id': searcher.id})
def delete_searcher(searcher_id):
session = db.session()
session.query(
db.ParamSearcherModel).filter_by(searcher_id=searcher_id).delete()
n = session.query(db.ParamSearcher).filter_by(id=searcher_id).delete()
if n:
session.commit()
return create_response({})
else:
return create_response({}, 404, err='searcher not found')
def undeploy_model(model_id):
session = db.session()
model = session.query(db.Model).get(model_id)
if model:
model.deployed = 0
session.add(model)
session.commit()
return create_response({'model': _model_to_dict(model)})
else:
return create_response({}, 404, err='not found')
def create_model():
session = db.session()
searcher = None
searcher_id = request.params.get('searcher_id', None)
if searcher_id is not None:
searcher = (session.query(
db.ParamSearcher).filter_by(id=int(searcher_id)).one())
dataset_id = int(request.params.dataset_id)
algorithm = int(request.params.algorithm)
algorithm_params = json.loads(request.params.algorithm_params)
algorithm_params['num_neighbors'] = int(algorithm_params['num_neighbors'])
batch_size = int(request.params.batch_size)
epoch = int(request.params.epoch)
try:
# if user input file is not exists
if algorithm_params['script_file_name'] is not '':
script = algorithm_params['script_file_name']
scriptdir = os.path.abspath(SCRIPT_DIR)
if not scriptdir.endswith(os.path.sep):
scriptdir += os.path.sep
scriptfile = os.path.abspath(os.path.join(scriptdir, script))
if not scriptfile.startswith(scriptdir):
raise ValueError('Invalid script name: %s' % scriptfile)
elif not os.path.exists(scriptfile):
raise ValueError('Invalid script name: %s' % scriptfile)
print(algorithm_params['script_file_name'])
model = db.Model(dataset_id=dataset_id,
algorithm=algorithm,
algorithm_params=pickle.dumps(algorithm_params),
batch_size=batch_size,
epoch=epoch)
session.add(model)
session.commit()
if searcher:
searchermodel = db.ParamSearcherModel(searcher=searcher,
model=model)
session.add(searchermodel)
session.commit()
return create_response(_model_to_dict(model))
except Exception as e:
traceback.print_exc()
return create_response({}, 500, err=str(e))
def train_model(model_id):
model = db.session().query(db.Model).get(model_id)
if not model.best_epoch_r2:
if not model:
return create_response({}, 404, err='model not found')
taskstate = train_task.TaskState.add_task(model)
executor = Executor()
submit_task(executor, train_task.train, taskstate, model.id)
if renom.cuda.has_cuda():
release_mem_pool()
return create_response({'result': 'ok'})
def get_deployed_model_weight():
model = db.session().query(db.Model).filter(db.Model.deployed == 1).one()
if model:
if model.weight:
file_name = model.weight
return static_file(file_name,
root=DB_DIR_TRAINED_WEIGHT,
download='deployed_model.h5')
elif model.model_pickle:
m_file_name = model.model_pickle
return static_file(m_file_name,
root=DB_DIR_ML_MODELS,
download='deployed_model.pickle')
else:
return create_response({}, 404, err='model not found')
def delete_dataset(dataset_id):
session = db.session()
ds = session.query(db.DatasetDef).get(dataset_id)
if ds:
q = session.query(
db.Model.id).filter(db.Model.dataset_id == dataset_id)
(session.query(db.ParamSearcherModel).filter(
db.ParamSearcherModel.model_id.in_(q)).delete(
synchronize_session=False))
session.query(
db.Model).filter(db.Model.dataset_id == dataset_id).delete()
session.delete(ds)
session.commit()
return create_response({})
else:
return create_response({}, 404, err='dataset not found')
def get_datasets():
q = db.session().query(db.DatasetDef).all()
ret = [_dataset_to_dict(ds) for ds in q]
return create_response({'datasets': ret})
def get_searchers():
session = db.session()
searchers = session.query(db.ParamSearcher)
dicts = [searcher_to_dict(searcher) for searcher in searchers]
return create_response({'searchers': dicts})
def get_models():
q = db.session().query(db.Model).order_by(db.Model.id.desc()).all()
ret = [_model_to_dict(model) for model in q]
return create_response({'models': ret})
def get_model(model_id):
model = db.session().query(db.Model).get(model_id)
if model:
return create_response(_model_to_dict(model))
else:
return create_response({}, 404, err='not found')
def get_dataset(dataset_id):
ds = db.session().query(db.DatasetDef).get(dataset_id)
if ds:
return create_response({'dataset': _dataset_to_dict(ds)})
else:
return create_response({}, 404, err='not found')
def predict_model(model_id):
explanatory_column = request.params.explanatory_column
target_column = request.params.target_column
explanatory_column_ids = json.loads(request.params.explanatory_column_ids)
model = db.session().query(db.Model).get(model_id)
if not model:
return create_response({}, 404, err='model not found')
try:
with open(os.path.join(DATASRC_DIR, 'prediction_set', 'pred.pickle'),
mode='rb') as f:
p_all_data = pickle.load(f)
p_X = split_target(p_all_data, pickle.loads(model.dataset.labels),
explanatory_column_ids)
n_X = np.array(p_X)
selected_scaling = model.dataset.selected_scaling
if selected_scaling != 1:
filename_X = model.dataset.filename_X
n_X_scaling = scaling_again(n_X, filename_X)
else:
n_X_scaling = n_X
except Exception as e:
traceback.print_exc()
return create_response({}, 404, err=str(e))
executor = Executor()
f = submit_task(executor, pred_task.prediction, model.id, n_X_scaling)
try:
result = f.result()
if selected_scaling != 1:
filename_y = model.dataset.filename_y
result = re_scaling(result, filename_y)
CSV_DIR = os.path.join(DATASRC_PREDICTION_OUT, 'csv')
if not os.path.isdir(CSV_DIR):
os.makedirs(CSV_DIR)
now = datetime.datetime.now()
filename = 'model' + str(model_id) + '_{0:%Y%m%d%H%M%S}'.format(
now) + '.csv'
filepath = os.path.join(CSV_DIR, filename)
target_labels = target_column.split(',')
explanatory_labels = explanatory_column.split(',')
labels = target_labels + explanatory_labels
np_xy = np.round(np.c_[result, n_X], 3)
pred_x_y = pd.DataFrame(np_xy)
pred_x_y.columns = labels
pred_x_y.to_csv(filepath, index=False)
SAMPLING_SIZE = 1000
sampled_n_X = []
sampled_result = []
p = np.random.permutation(n_X.shape[0])
n_X = n_X.tolist()
result = result.tolist()
sampled_p = p[:SAMPLING_SIZE]
for sp_n in sampled_p:
sampled_n_X.append(n_X[sp_n])
sampled_result.append(result[sp_n])
body = {
'pred_x': n_X,
'pred_y': result,
'sampled_pred_x': sampled_n_X,
'sampled_pred_y': sampled_result,
'pred_csv': filename
}
return create_response(body)
except Exception as e:
traceback.print_exc()
return create_response({}, 404, err=str(e))
finally:
if renom.cuda.has_cuda():
release_mem_pool()
