def post(self, datasource_name):
# url /datasource/<string:user_schema_name>/<string:datasource_name> {
# "file_name": "cassava.csv",
# }
user_id = get_jwt_identity()
user_schema_name = 'u{}'.format(user_id)
user_table_name = datasource_name
data = NewDatasource.parser.parse_args()
datasource_description = data['datasource_description']
upload_file = data['upload_file']
file_extension = upload_file.filename.split('.')[1]
datasource = DatasourceModel(datasource_name, datasource_description,
user_schema_name, user_table_name,
user_id)
if datasource.find_by_name(datasource_name, user_id):
return {
'message': ERROR_DUPLICATE_NAME.format(datasource_name)
}, 500
upload = UploadModel(upload_file)
try:
file_id = str(upload.save_file_to_db())
except Exception as e:
return {'message': ERROR_CANNOT_UPLOAD_FILE}, 500
file = File(file_id, file_extension)
file.read_file_to_dataset()
dataset_dataframe = file.get_dataframe()
dataset_json = file.get_json()
datasource = DatasourceModel(datasource_name, datasource_description,
user_schema_name, user_table_name,
user_id)
try:
datasource.new_datasource(dataset_dataframe)
except Exception as e:
return {'message': ERROR_CANNOT_CONVERT_TO_DATAFRAME}, 500
try:
datasource.save_to_db(file_id)
except Exception as e:
return {'message': ERROR_CANNOT_INSERT_DB}, 500
return {'message': 'sucess', 'dataset': dataset_json}, 200
def get(self, datasource_name):
user_id = get_jwt_identity()
user_schema_name = 'u{}'.format(user_id)
user_table_name = datasource_name
datasource_description = ''
datasource = DatasourceModel(datasource_name, datasource_description,
user_schema_name, user_table_name,
user_id)
dataframe = datasource.get_datasource()
dataset = Dataset(dataframe)
header = DatasourceModel.get_columns(user_schema_name, user_table_name)
return {'header': header, 'data': dataset.get_json()}
def post(self):
self.parser.add_argument("data", type=str, required=True)
args = self.parser.parse_args()
data = json.loads(args["data"])
obj = DatasourceModel(**data)
try:
obj.save()
except Exception as e:
print(e)
return {"msg": "Insert error!", "data": ""}, 404
return {"data": obj.to_dict(), "msg": ""}
def post(self, project_id, experiment_name):
user_id = get_jwt_identity()
data = Experiment.parser.parse_args()
experiment_description = data['experiment_description']
datasource_id = data['datasoruce_id']
model_id = data['model_id']
model_parameter = data['model_parameter']
y_col = data['y_col']
# --START--Checking retrived data
experiment = ExperimentModel(experiment_name, experiment_description,
datasource_id, model_id, y_col,
project_id, user_id)
if experiment.find_by_name(user_id, experiment_name):
return {
'message': ERROR_DUPLICATE_NAME.format(experiment_name)
}, 500
try:
result = DatasourceModel.find_one_by_id(user_id, datasource_id)
datasource_data = {
'datasource_name': result.datasource_name,
'datasource_description': result.datasource_description,
'user_schema_name': result.user_schema_name,
'user_table_name': result.user_table_name,
'user_id': str(result.created_at)
}
except Exception as e:
return {
'message':
ERROR_CANNOT_FIND_DATASOURCE_ID.format(datasource_id)
}, 500
try:
result = ModelModel.find_first_by_id(model_id)
model_name = result.model_name
except Exception as e:
return {
'message': ERROR_CANNOT_FIND_MODEL_ID.format(model_id)
}, 500
# --END--Checking retrived data
try:
dataframe = DatasourceModel(**datasource_data).get_datasource()
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_DATASET.format(
datasource_data['user_table_name'])
}, 500
try:
columns_filter_id = ProjectModel.find_one_by_id(
project_id, user_id).columns_filter_id
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format(project_id)
}, 500
try:
columns_filter = ProjectUnstructureModel.find_by_object_id(
ObjectId(columns_filter_id))['columns']
except Exception as e:
return {
'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id)
}, 500
try:
training = Training(experiment_name, dataframe, model_name,
model_parameter, columns_filter, y_col,
user_id)
training.train_test()
training.create_and_fit_model()
training.save_model()
training.load_model()
y_pred = training.predict()
cal_score = training.cal_score()
except Exception as e:
print(e)
return {'message': ERROR_CANNOT_TRAIN_MODEL_WITH_DATA}, 500
try:
mongo = ExperimentUnstructureModel(cal_score, model_parameter)
cal_score_id = str(mongo.save_cal_score_to_db())
model_parameter_id = str(mongo.save_model_parameter_to_db())
except Exception as e:
return {
'message': ERROR_CANNOT_INSRET_MONGO.format(experiment_name)
}, 500
try:
experiment.save_to_db()
except Exception as e:
return {'message': ERROR_CANNOT_INSERT_DB}, 500
try:
result = ExperimentModel.find_by_name(user_id, experiment_name)
if result:
result.save_cal_score_to_db(cal_score_id)
result.save_model_parameter_to_db(model_parameter_id)
except Exception as e:
return {'message': ERROR_CANNOT_INSRET_SCORE.format(e)}, 500
return {
'message': '{} have already trained'.format(experiment_name)
}, 200
def put(self, experiment_id):
user_id = get_jwt_identity()
experiment = ExperimentModel.find_by_id(experiment_id)
try:
result = DatasourceModel.find_one_by_id(user_id,
experiment.datasource_id)
datasource_data = {
'datasource_name': result.datasource_name,
'datasource_description': result.datasource_description,
'user_schema_name': result.user_schema_name,
'user_table_name': result.user_table_name,
'user_id': str(result.created_at)
}
except Exception as e:
return {
'message':
ERROR_CANNOT_FIND_DATASOURCE_ID.format(
experiment.datasource_id)
}, 500
try:
result = ModelModel.find_first_by_id(experiment.model_id)
model_name = result.model_name
except Exception as e:
return {
'message':
ERROR_CANNOT_FIND_MODEL_ID.format(experiment.model_id)
}, 500
try:
dataframe = DatasourceModel(**datasource_data).get_datasource()
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_DATASET.format(
datasource_data['user_table_name'])
}, 500
try:
columns_filter_id = ProjectModel.find_one_by_id(
experiment.project_id, user_id).columns_filter_id
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format(
experiment.project_id)
}, 500
try:
columns_filter = ProjectUnstructureModel.find_by_object_id(
ObjectId(columns_filter_id))['columns']
except Exception as e:
return {
'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id)
}, 500
try:
model_parameter = ExperimentUnstructureModel.find_model_parameter_by_object_id(
ObjectId(experiment.model_parameter_id))['model_parameter']
except Exception as e:
return {
'message':
ERROR_CANNOT_GET_MODEL_PARAMETER.format(
experiment.model_parameter_id)
}, 500
training = Training(experiment.experiment_name, dataframe, model_name,
model_parameter, columns_filter, experiment.y_col)
training.train_test()
training.create_and_fit_model()
training.save_model()
training.load_model()
y_pred = training.predict()
cal_score = training.cal_score()
try:
mongo = ExperimentUnstructureModel(cal_score, model_parameter)
cal_score_id = str(mongo.save_cal_score_to_db())
except Exception as e:
return {
'message':
ERROR_CANNOT_INSRET_MONGO.format(experiment.experiment_name)
}, 500
try:
experiment.cal_score_id = cal_score_id
experiment.save_to_db()
except Exception as e:
return {'message': ERROR_CANNOT_INSERT_DB}, 500
return {'message': SUCCESS_MESSAGE.format(experiment_id)}, 200