def get(self, datasource_id, page_size, page):
user_id = get_jwt_identity()
datasource = DatasourceModel.find_one_by_id(user_id, datasource_id)
dataframe = datasource.get_datasource_per_page(page, page_size)
dataset = Dataset(dataframe)
header = DatasourceModel.get_columns(datasource.user_schema_name,
datasource.user_table_name)
return {'header': header, 'data': dataset.get_json()}
def post(self, experiment_name):
user_id = get_jwt_identity()
data = Prediction.parser.parse_args()
raw_data = data['data']
dataset = File.read_json(raw_data)
result = ExperimentModel.find_by_name(user_id, experiment_name)
y_col = result.y_col
datasource_id = result.datasource_id
project_id = result.project_id
result = DatasourceModel.find_one_by_id(user_id, datasource_id)
user_schema_name = result.user_schema_name
user_table_name = result.user_table_name
columns = [
col['column_name'] for col in DatasourceModel.get_columns(
user_schema_name, user_table_name)
]
try:
columns_filter_id = ProjectModel.find_one_by_id(
project_id, user_id).columns_filter_id
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format(project_id)
}, 500
try:
columns_filter = ProjectUnstructureModel.find_by_object_id(
ObjectId(columns_filter_id))['columns']
except Exception as e:
return {
'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id)
}, 500
try:
predict = Predicting(experiment_name, dataset, columns,
columns_filter, y_col, user_id)
predict.decorate_dataframe()
predict.load_model()
predict.predict()
dataset = predict.merge_predict_to_dataframe()
json_data = Dataset(dataset).get_json()
except Exception as e:
print(e)
return {
'message':
ERROR_CANNOT_GET_PREDICT_WITH_DATA.format(experiment_name)
}, 500
# try:
# DatasourceModel.append_datasource(dataset, user_schema_name, user_table_name)
# except Exception as e:
# return {'message': ERROR_CANNOT_INSERT_DB}, 500
return {'data': json_data}, 200
def get(self, datasource_name):
user_id = get_jwt_identity()
user_schema_name = 'u{}'.format(user_id)
user_table_name = datasource_name
datasource_description = ''
datasource = DatasourceModel(datasource_name, datasource_description,
user_schema_name, user_table_name,
user_id)
dataframe = datasource.get_datasource()
dataset = Dataset(dataframe)
header = DatasourceModel.get_columns(user_schema_name, user_table_name)
return {'header': header, 'data': dataset.get_json()}
def get(self, project_id, experiment_name):
user_id = get_jwt_identity()
experiment = ExperimentModel.find_by_experiment_name(
user_id, experiment_name)
datasource = DatasourceModel.find_one_by_id(user_id,
experiment.datasource_id)
data_columns = DatasourceModel.get_columns(datasource.user_schema_name,
datasource.user_table_name)
return {
'datasource_name': datasource.datasource_name,
'header': data_columns
}, 200
def get(self, datasource_id):
user_id = get_jwt_identity()
datasource = DatasourceModel.find_one_by_id(user_id, datasource_id)
if (datasource):
datasource_name = datasource.datasource_name
user_schema_name = 'u{}'.format(user_id)
user_table_name = datasource_name
header = DatasourceModel.get_columns(user_schema_name,
user_table_name)
return {'message': 'sucess', 'header': header}, 200
else:
return {'message': 'Cannot fetch columns'}, 500
def post(self):
self.parser.add_argument("data", type=str, required=True)
args = self.parser.parse_args()
data = json.loads(args["data"])
obj = DatasourceModel(**data)
try:
obj.save()
except Exception as e:
print(e)
return {"msg": "Insert error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
def get(self, datasource_id):
user_id = get_jwt_identity()
datasource = DatasourceModel.find_one_by_id(user_id, datasource_id)
metadata = {
'datasource_id': datasource.datasource_id,
'datasource_name': datasource.datasource_name,
'datasource_description': datasource.datasource_description,
'user_schema_name': datasource.user_schema_name,
'user_table_name': datasource.user_table_name,
'created_at': str(datasource.created_at)
}
header = DatasourceModel.get_columns(datasource.user_schema_name,
datasource.user_table_name)
return {'header': header, 'metadata': metadata}
def get(self, project_id):
user_id = get_jwt_identity()
results = ExperimentModel.find_all_by_id(project_id)
data = [{
'experiment_id':
row.experiment_id,
'experiment_name':
row.experiment_name,
'experiment_description':
row.experiment_description,
'cal_score_id':
row.cal_score_id,
'datasource_id':
row.datasource_id,
'datasource_name':
DatasourceModel.find_one_by_id(user_id,
row.datasource_id).datasource_name,
'model': {
'model_id':
row.model_id,
'model_name':
ModelModel.find_first_by_id(row.model_id).model_name,
'model_type':
ModelModel.find_first_by_id(
row.model_id).model_type.model_type_name
},
'y_col':
row.y_col,
'model_parameter_id':
row.model_parameter_id,
'viz_id':
row.viz_id
} for row in results]
return {'message': 'list of all experiments', 'data': data}, 200
def get(self, id):
obj = DatasourceModel.get_by_id(id)
if obj:
s = obj.to_dict()
return {"data": s, "msg": ""}
return {"msg": "Item not found!", "data": ""}, 404
def post(self, datasource_name):
data = AddPrediction.parser.parse_args()
predicted_data = data['predicted_data']
datasource = DatasourceModel.find_by_name(datasource_name)
user_schema_name = datasource.user_schema_name
user_table_name = datasource.user_table_name
dataset = File.read_json(predicted_data)
try:
DatasourceModel.append_datasource(dataset, user_schema_name,
user_table_name)
except Exception as e:
return {'message': ERROR_CANNOT_INSERT_DB}, 500
return {
'message':
'This Dataset has append to datasource ({})'.format(
datasource_name)
}, 200
def post(self, datasource_name):
# url /datasource/<string:user_schema_name>/<string:datasource_name> {
# "file_name": "cassava.csv",
# }
user_id = get_jwt_identity()
user_schema_name = 'u{}'.format(user_id)
user_table_name = datasource_name
data = NewDatasource.parser.parse_args()
datasource_description = data['datasource_description']
upload_file = data['upload_file']
file_extension = upload_file.filename.split('.')[1]
datasource = DatasourceModel(datasource_name, datasource_description,
user_schema_name, user_table_name,
user_id)
if datasource.find_by_name(datasource_name, user_id):
return {
'message': ERROR_DUPLICATE_NAME.format(datasource_name)
}, 500
upload = UploadModel(upload_file)
try:
file_id = str(upload.save_file_to_db())
except Exception as e:
return {'message': ERROR_CANNOT_UPLOAD_FILE}, 500
file = File(file_id, file_extension)
file.read_file_to_dataset()
dataset_dataframe = file.get_dataframe()
dataset_json = file.get_json()
datasource = DatasourceModel(datasource_name, datasource_description,
user_schema_name, user_table_name,
user_id)
try:
datasource.new_datasource(dataset_dataframe)
except Exception as e:
return {'message': ERROR_CANNOT_CONVERT_TO_DATAFRAME}, 500
try:
datasource.save_to_db(file_id)
except Exception as e:
return {'message': ERROR_CANNOT_INSERT_DB}, 500
return {'message': 'sucess', 'dataset': dataset_json}, 200
def delete(self, id):
obj = DatasourceModel.get_by_id(id)
if obj:
try:
obj.delete()
except Exception as e:
print(e)
return {"msg": "Delete error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
return {"msg": "Item not found!", "data": ""}, 404
def get(self):
user_id = get_jwt_identity()
results = DatasourceModel.find_all(user_id)
data = [{
'datasource_id': row.datasource_id,
'datasource_name': row.datasource_name,
'datasource_description': row.datasource_description,
'user_schema_name': row.user_schema_name,
'user_table_name': row.user_table_name,
'created_at': str(row.created_at)
} for row in results]
return {'message': 'list of all datasources', 'data': data}, 200
def put(self, id):
obj = DatasourceModel.get_by_id(id)
if obj:
self.add_argument("data", type=str, required=True)
args = self.parser.parse_args()
data = json.loads(args["data"])
try:
obj.update(data)
except Exception as e:
print(e)
return {"msg": "Update error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
return {"msg": "Item not found!", "data": ""}, 404
def get(self, datasource_id, page_size):
user_id = get_jwt_identity()
datasource = DatasourceModel.find_one_by_id(user_id, datasource_id)
pages, rows = datasource.get_datasource_pages(page_size)
return {
'message': SUCCESS_MESSAGE.format(datasource_id),
'data': {
'datasource_id': datasource_id,
'datasource_name': datasource.datasource_name,
'pages': pages,
'rows': rows,
'page_size': page_size
}
}
def delete(self, datasource_id):
user_id = get_jwt_identity()
datasource = DatasourceModel.find_one_by_id(user_id, datasource_id)
user_schema_name = datasource.user_schema_name
user_table_name = datasource.user_table_name
try:
datasource.delete_from_db()
except Exception as e:
return {'message': ERROR_CANNOT_DELETE_DATASOURCE_META}, 500
try:
datasource.delete_datasource(user_schema_name, user_table_name)
except Exception as e:
return {'message': ERROR_CANNOT_DELETE_DATASOURCE}, 500
return {'message': SUCCESS_MESSAGE.format(datasource_id)}, 200
def get(self, project_id):
user_id = get_jwt_identity()
data = ProjectModel.find_one_by_id(project_id, user_id)
collection = CollectionModel.find_one_by_id(data.collection_id,
user_id)
datasource = DatasourceModel.find_one_by_id(user_id,
data.datasource_id)
return {
'project_id': data.project_id,
'project_name': data.project_name,
'project_description': data.project_description,
'datasource': {
'datasource_id': datasource.datasource_id,
'datasource_name': datasource.datasource_name,
'datasource_description': datasource.datasource_description
},
'collection': {
'collection_id': collection.collection_id,
'collection_name': collection.collection_name,
'collection_description': collection.collection_description,
}
}
def post(self, project_id, experiment_name):
user_id = get_jwt_identity()
data = Experiment.parser.parse_args()
experiment_description = data['experiment_description']
datasource_id = data['datasoruce_id']
model_id = data['model_id']
model_parameter = data['model_parameter']
y_col = data['y_col']
# --START--Checking retrived data
experiment = ExperimentModel(experiment_name, experiment_description,
datasource_id, model_id, y_col,
project_id, user_id)
if experiment.find_by_name(user_id, experiment_name):
return {
'message': ERROR_DUPLICATE_NAME.format(experiment_name)
}, 500
try:
result = DatasourceModel.find_one_by_id(user_id, datasource_id)
datasource_data = {
'datasource_name': result.datasource_name,
'datasource_description': result.datasource_description,
'user_schema_name': result.user_schema_name,
'user_table_name': result.user_table_name,
'user_id': str(result.created_at)
}
except Exception as e:
return {
'message':
ERROR_CANNOT_FIND_DATASOURCE_ID.format(datasource_id)
}, 500
try:
result = ModelModel.find_first_by_id(model_id)
model_name = result.model_name
except Exception as e:
return {
'message': ERROR_CANNOT_FIND_MODEL_ID.format(model_id)
}, 500
# --END--Checking retrived data
try:
dataframe = DatasourceModel(**datasource_data).get_datasource()
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_DATASET.format(
datasource_data['user_table_name'])
}, 500
try:
columns_filter_id = ProjectModel.find_one_by_id(
project_id, user_id).columns_filter_id
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format(project_id)
}, 500
try:
columns_filter = ProjectUnstructureModel.find_by_object_id(
ObjectId(columns_filter_id))['columns']
except Exception as e:
return {
'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id)
}, 500
try:
training = Training(experiment_name, dataframe, model_name,
model_parameter, columns_filter, y_col,
user_id)
training.train_test()
training.create_and_fit_model()
training.save_model()
training.load_model()
y_pred = training.predict()
cal_score = training.cal_score()
except Exception as e:
print(e)
return {'message': ERROR_CANNOT_TRAIN_MODEL_WITH_DATA}, 500
try:
mongo = ExperimentUnstructureModel(cal_score, model_parameter)
cal_score_id = str(mongo.save_cal_score_to_db())
model_parameter_id = str(mongo.save_model_parameter_to_db())
except Exception as e:
return {
'message': ERROR_CANNOT_INSRET_MONGO.format(experiment_name)
}, 500
try:
experiment.save_to_db()
except Exception as e:
return {'message': ERROR_CANNOT_INSERT_DB}, 500
try:
result = ExperimentModel.find_by_name(user_id, experiment_name)
if result:
result.save_cal_score_to_db(cal_score_id)
result.save_model_parameter_to_db(model_parameter_id)
except Exception as e:
return {'message': ERROR_CANNOT_INSRET_SCORE.format(e)}, 500
return {
'message': '{} have already trained'.format(experiment_name)
}, 200
def put(self, experiment_id):
user_id = get_jwt_identity()
experiment = ExperimentModel.find_by_id(experiment_id)
try:
result = DatasourceModel.find_one_by_id(user_id,
experiment.datasource_id)
datasource_data = {
'datasource_name': result.datasource_name,
'datasource_description': result.datasource_description,
'user_schema_name': result.user_schema_name,
'user_table_name': result.user_table_name,
'user_id': str(result.created_at)
}
except Exception as e:
return {
'message':
ERROR_CANNOT_FIND_DATASOURCE_ID.format(
experiment.datasource_id)
}, 500
try:
result = ModelModel.find_first_by_id(experiment.model_id)
model_name = result.model_name
except Exception as e:
return {
'message':
ERROR_CANNOT_FIND_MODEL_ID.format(experiment.model_id)
}, 500
try:
dataframe = DatasourceModel(**datasource_data).get_datasource()
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_DATASET.format(
datasource_data['user_table_name'])
}, 500
try:
columns_filter_id = ProjectModel.find_one_by_id(
experiment.project_id, user_id).columns_filter_id
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format(
experiment.project_id)
}, 500
try:
columns_filter = ProjectUnstructureModel.find_by_object_id(
ObjectId(columns_filter_id))['columns']
except Exception as e:
return {
'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id)
}, 500
try:
model_parameter = ExperimentUnstructureModel.find_model_parameter_by_object_id(
ObjectId(experiment.model_parameter_id))['model_parameter']
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_MODEL_PARAMETER.format(
experiment.model_parameter_id)
}, 500
training = Training(experiment.experiment_name, dataframe, model_name,
model_parameter, columns_filter, experiment.y_col)
training.train_test()
training.create_and_fit_model()
training.save_model()
training.load_model()
y_pred = training.predict()
cal_score = training.cal_score()
try:
mongo = ExperimentUnstructureModel(cal_score, model_parameter)
cal_score_id = str(mongo.save_cal_score_to_db())
except Exception as e:
return {
'message':
ERROR_CANNOT_INSRET_MONGO.format(experiment.experiment_name)
}, 500
try:
experiment.cal_score_id = cal_score_id
experiment.save_to_db()
except Exception as e:
return {'message': ERROR_CANNOT_INSERT_DB}, 500
return {'message': SUCCESS_MESSAGE.format(experiment_id)}, 200
def get(self, datasource_name):
user_id = get_jwt_identity()
user_schema_name = 'u{}'.format(user_id)
user_table_name = datasource_name
header = DatasourceModel.get_columns(user_schema_name, user_table_name)
return {'message': 'sucess', 'header': header}, 200