def get(self, datasource_id, page_size, page): user_id = get_jwt_identity() datasource = DatasourceModel.find_one_by_id(user_id, datasource_id) dataframe = datasource.get_datasource_per_page(page, page_size) dataset = Dataset(dataframe) header = DatasourceModel.get_columns(datasource.user_schema_name, datasource.user_table_name) return {'header': header, 'data': dataset.get_json()}
def post(self, experiment_name): user_id = get_jwt_identity() data = Prediction.parser.parse_args() raw_data = data['data'] dataset = File.read_json(raw_data) result = ExperimentModel.find_by_name(user_id, experiment_name) y_col = result.y_col datasource_id = result.datasource_id project_id = result.project_id result = DatasourceModel.find_one_by_id(user_id, datasource_id) user_schema_name = result.user_schema_name user_table_name = result.user_table_name columns = [ col['column_name'] for col in DatasourceModel.get_columns( user_schema_name, user_table_name) ] try: columns_filter_id = ProjectModel.find_one_by_id( project_id, user_id).columns_filter_id except Exception as e: return { 'message': ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format(project_id) }, 500 try: columns_filter = ProjectUnstructureModel.find_by_object_id( ObjectId(columns_filter_id))['columns'] except Exception as e: return { 'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id) }, 500 try: predict = Predicting(experiment_name, dataset, columns, columns_filter, y_col, user_id) predict.decorate_dataframe() predict.load_model() predict.predict() dataset = predict.merge_predict_to_dataframe() json_data = Dataset(dataset).get_json() except Exception as e: print(e) return { 'message': ERROR_CANNOT_GET_PREDICT_WITH_DATA.format(experiment_name) }, 500 # try: # DatasourceModel.append_datasource(dataset, user_schema_name, user_table_name) # except Exception as e: # return {'message': ERROR_CANNOT_INSERT_DB}, 500 return {'data': json_data}, 200
def get(self, datasource_name): user_id = get_jwt_identity() user_schema_name = 'u{}'.format(user_id) user_table_name = datasource_name datasource_description = '' datasource = DatasourceModel(datasource_name, datasource_description, user_schema_name, user_table_name, user_id) dataframe = datasource.get_datasource() dataset = Dataset(dataframe) header = DatasourceModel.get_columns(user_schema_name, user_table_name) return {'header': header, 'data': dataset.get_json()}
def get(self, project_id, experiment_name): user_id = get_jwt_identity() experiment = ExperimentModel.find_by_experiment_name( user_id, experiment_name) datasource = DatasourceModel.find_one_by_id(user_id, experiment.datasource_id) data_columns = DatasourceModel.get_columns(datasource.user_schema_name, datasource.user_table_name) return { 'datasource_name': datasource.datasource_name, 'header': data_columns }, 200
def get(self, datasource_id): user_id = get_jwt_identity() datasource = DatasourceModel.find_one_by_id(user_id, datasource_id) if (datasource): datasource_name = datasource.datasource_name user_schema_name = 'u{}'.format(user_id) user_table_name = datasource_name header = DatasourceModel.get_columns(user_schema_name, user_table_name) return {'message': 'sucess', 'header': header}, 200 else: return {'message': 'Cannot fetch columns'}, 500
def post(self): self.parser.add_argument("data", type=str, required=True) args = self.parser.parse_args() data = json.loads(args["data"]) obj = DatasourceModel(**data) try: obj.save() except Exception as e: print(e) return {"msg": "Insert error!", "data": ""}, 404 return {"data": obj.to_dict(), "msg": ""}
def get(self, datasource_id): user_id = get_jwt_identity() datasource = DatasourceModel.find_one_by_id(user_id, datasource_id) metadata = { 'datasource_id': datasource.datasource_id, 'datasource_name': datasource.datasource_name, 'datasource_description': datasource.datasource_description, 'user_schema_name': datasource.user_schema_name, 'user_table_name': datasource.user_table_name, 'created_at': str(datasource.created_at) } header = DatasourceModel.get_columns(datasource.user_schema_name, datasource.user_table_name) return {'header': header, 'metadata': metadata}
def get(self, project_id): user_id = get_jwt_identity() results = ExperimentModel.find_all_by_id(project_id) data = [{ 'experiment_id': row.experiment_id, 'experiment_name': row.experiment_name, 'experiment_description': row.experiment_description, 'cal_score_id': row.cal_score_id, 'datasource_id': row.datasource_id, 'datasource_name': DatasourceModel.find_one_by_id(user_id, row.datasource_id).datasource_name, 'model': { 'model_id': row.model_id, 'model_name': ModelModel.find_first_by_id(row.model_id).model_name, 'model_type': ModelModel.find_first_by_id( row.model_id).model_type.model_type_name }, 'y_col': row.y_col, 'model_parameter_id': row.model_parameter_id, 'viz_id': row.viz_id } for row in results] return {'message': 'list of all experiments', 'data': data}, 200
def get(self, id): obj = DatasourceModel.get_by_id(id) if obj: s = obj.to_dict() return {"data": s, "msg": ""} return {"msg": "Item not found!", "data": ""}, 404
def post(self, datasource_name): data = AddPrediction.parser.parse_args() predicted_data = data['predicted_data'] datasource = DatasourceModel.find_by_name(datasource_name) user_schema_name = datasource.user_schema_name user_table_name = datasource.user_table_name dataset = File.read_json(predicted_data) try: DatasourceModel.append_datasource(dataset, user_schema_name, user_table_name) except Exception as e: return {'message': ERROR_CANNOT_INSERT_DB}, 500 return { 'message': 'This Dataset has append to datasource ({})'.format( datasource_name) }, 200
def post(self, datasource_name): # url /datasource/<string:user_schema_name>/<string:datasource_name> { # "file_name": "cassava.csv", # } user_id = get_jwt_identity() user_schema_name = 'u{}'.format(user_id) user_table_name = datasource_name data = NewDatasource.parser.parse_args() datasource_description = data['datasource_description'] upload_file = data['upload_file'] file_extension = upload_file.filename.split('.')[1] datasource = DatasourceModel(datasource_name, datasource_description, user_schema_name, user_table_name, user_id) if datasource.find_by_name(datasource_name, user_id): return { 'message': ERROR_DUPLICATE_NAME.format(datasource_name) }, 500 upload = UploadModel(upload_file) try: file_id = str(upload.save_file_to_db()) except Exception as e: return {'message': ERROR_CANNOT_UPLOAD_FILE}, 500 file = File(file_id, file_extension) file.read_file_to_dataset() dataset_dataframe = file.get_dataframe() dataset_json = file.get_json() datasource = DatasourceModel(datasource_name, datasource_description, user_schema_name, user_table_name, user_id) try: datasource.new_datasource(dataset_dataframe) except Exception as e: return {'message': ERROR_CANNOT_CONVERT_TO_DATAFRAME}, 500 try: datasource.save_to_db(file_id) except Exception as e: return {'message': ERROR_CANNOT_INSERT_DB}, 500 return {'message': 'sucess', 'dataset': dataset_json}, 200
def delete(self, id): obj = DatasourceModel.get_by_id(id) if obj: try: obj.delete() except Exception as e: print(e) return {"msg": "Delete error!", "data": ""}, 404 return {"data": obj.to_dict(), "msg": ""} return {"msg": "Item not found!", "data": ""}, 404
def get(self): user_id = get_jwt_identity() results = DatasourceModel.find_all(user_id) data = [{ 'datasource_id': row.datasource_id, 'datasource_name': row.datasource_name, 'datasource_description': row.datasource_description, 'user_schema_name': row.user_schema_name, 'user_table_name': row.user_table_name, 'created_at': str(row.created_at) } for row in results] return {'message': 'list of all datasources', 'data': data}, 200
def put(self, id): obj = DatasourceModel.get_by_id(id) if obj: self.add_argument("data", type=str, required=True) args = self.parser.parse_args() data = json.loads(args["data"]) try: obj.update(data) except Exception as e: print(e) return {"msg": "Update error!", "data": ""}, 404 return {"data": obj.to_dict(), "msg": ""} return {"msg": "Item not found!", "data": ""}, 404
def get(self, datasource_id, page_size): user_id = get_jwt_identity() datasource = DatasourceModel.find_one_by_id(user_id, datasource_id) pages, rows = datasource.get_datasource_pages(page_size) return { 'message': SUCCESS_MESSAGE.format(datasource_id), 'data': { 'datasource_id': datasource_id, 'datasource_name': datasource.datasource_name, 'pages': pages, 'rows': rows, 'page_size': page_size } }
def delete(self, datasource_id): user_id = get_jwt_identity() datasource = DatasourceModel.find_one_by_id(user_id, datasource_id) user_schema_name = datasource.user_schema_name user_table_name = datasource.user_table_name try: datasource.delete_from_db() except Exception as e: return {'message': ERROR_CANNOT_DELETE_DATASOURCE_META}, 500 try: datasource.delete_datasource(user_schema_name, user_table_name) except Exception as e: return {'message': ERROR_CANNOT_DELETE_DATASOURCE}, 500 return {'message': SUCCESS_MESSAGE.format(datasource_id)}, 200
def get(self, project_id): user_id = get_jwt_identity() data = ProjectModel.find_one_by_id(project_id, user_id) collection = CollectionModel.find_one_by_id(data.collection_id, user_id) datasource = DatasourceModel.find_one_by_id(user_id, data.datasource_id) return { 'project_id': data.project_id, 'project_name': data.project_name, 'project_description': data.project_description, 'datasource': { 'datasource_id': datasource.datasource_id, 'datasource_name': datasource.datasource_name, 'datasource_description': datasource.datasource_description }, 'collection': { 'collection_id': collection.collection_id, 'collection_name': collection.collection_name, 'collection_description': collection.collection_description, } }
def post(self, project_id, experiment_name): user_id = get_jwt_identity() data = Experiment.parser.parse_args() experiment_description = data['experiment_description'] datasource_id = data['datasoruce_id'] model_id = data['model_id'] model_parameter = data['model_parameter'] y_col = data['y_col'] # --START--Checking retrived data experiment = ExperimentModel(experiment_name, experiment_description, datasource_id, model_id, y_col, project_id, user_id) if experiment.find_by_name(user_id, experiment_name): return { 'message': ERROR_DUPLICATE_NAME.format(experiment_name) }, 500 try: result = DatasourceModel.find_one_by_id(user_id, datasource_id) datasource_data = { 'datasource_name': result.datasource_name, 'datasource_description': result.datasource_description, 'user_schema_name': result.user_schema_name, 'user_table_name': result.user_table_name, 'user_id': str(result.created_at) } except Exception as e: return { 'message': ERROR_CANNOT_FIND_DATASOURCE_ID.format(datasource_id) }, 500 try: result = ModelModel.find_first_by_id(model_id) model_name = result.model_name except Exception as e: return { 'message': ERROR_CANNOT_FIND_MODEL_ID.format(model_id) }, 500 # --END--Checking retrived data try: dataframe = DatasourceModel(**datasource_data).get_datasource() except Exception as e: return { 'message': ERROR_CANNOT_GET_DATASET.format( datasource_data['user_table_name']) }, 500 try: columns_filter_id = ProjectModel.find_one_by_id( project_id, user_id).columns_filter_id except Exception as e: return { 'message': ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format(project_id) }, 500 try: columns_filter = ProjectUnstructureModel.find_by_object_id( ObjectId(columns_filter_id))['columns'] except Exception as e: return { 'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id) }, 500 try: training = Training(experiment_name, dataframe, model_name, model_parameter, columns_filter, y_col, user_id) training.train_test() training.create_and_fit_model() training.save_model() training.load_model() y_pred = training.predict() cal_score = training.cal_score() except Exception as e: print(e) return {'message': ERROR_CANNOT_TRAIN_MODEL_WITH_DATA}, 500 try: mongo = ExperimentUnstructureModel(cal_score, model_parameter) cal_score_id = str(mongo.save_cal_score_to_db()) model_parameter_id = str(mongo.save_model_parameter_to_db()) except Exception as e: return { 'message': ERROR_CANNOT_INSRET_MONGO.format(experiment_name) }, 500 try: experiment.save_to_db() except Exception as e: return {'message': ERROR_CANNOT_INSERT_DB}, 500 try: result = ExperimentModel.find_by_name(user_id, experiment_name) if result: result.save_cal_score_to_db(cal_score_id) result.save_model_parameter_to_db(model_parameter_id) except Exception as e: return {'message': ERROR_CANNOT_INSRET_SCORE.format(e)}, 500 return { 'message': '{} have already trained'.format(experiment_name) }, 200
def put(self, experiment_id): user_id = get_jwt_identity() experiment = ExperimentModel.find_by_id(experiment_id) try: result = DatasourceModel.find_one_by_id(user_id, experiment.datasource_id) datasource_data = { 'datasource_name': result.datasource_name, 'datasource_description': result.datasource_description, 'user_schema_name': result.user_schema_name, 'user_table_name': result.user_table_name, 'user_id': str(result.created_at) } except Exception as e: return { 'message': ERROR_CANNOT_FIND_DATASOURCE_ID.format( experiment.datasource_id) }, 500 try: result = ModelModel.find_first_by_id(experiment.model_id) model_name = result.model_name except Exception as e: return { 'message': ERROR_CANNOT_FIND_MODEL_ID.format(experiment.model_id) }, 500 try: dataframe = DatasourceModel(**datasource_data).get_datasource() except Exception as e: return { 'message': ERROR_CANNOT_GET_DATASET.format( datasource_data['user_table_name']) }, 500 try: columns_filter_id = ProjectModel.find_one_by_id( experiment.project_id, user_id).columns_filter_id except Exception as e: return { 'message': ERROR_CANNOT_GET_COLUMNS_FILTER_ID.format( experiment.project_id) }, 500 try: columns_filter = ProjectUnstructureModel.find_by_object_id( ObjectId(columns_filter_id))['columns'] except Exception as e: return { 'message': ERROR_CANNOT_GET_COLUMNS_FILTER.format(project_id) }, 500 try: model_parameter = ExperimentUnstructureModel.find_model_parameter_by_object_id( ObjectId(experiment.model_parameter_id))['model_parameter'] except Exception as e: return { 'message': ERROR_CANNOT_GET_MODEL_PARAMETER.format( experiment.model_parameter_id) }, 500 training = Training(experiment.experiment_name, dataframe, model_name, model_parameter, columns_filter, experiment.y_col) training.train_test() training.create_and_fit_model() training.save_model() training.load_model() y_pred = training.predict() cal_score = training.cal_score() try: mongo = ExperimentUnstructureModel(cal_score, model_parameter) cal_score_id = str(mongo.save_cal_score_to_db()) except Exception as e: return { 'message': ERROR_CANNOT_INSRET_MONGO.format(experiment.experiment_name) }, 500 try: experiment.cal_score_id = cal_score_id experiment.save_to_db() except Exception as e: return {'message': ERROR_CANNOT_INSERT_DB}, 500 return {'message': SUCCESS_MESSAGE.format(experiment_id)}, 200
def get(self, datasource_name): user_id = get_jwt_identity() user_schema_name = 'u{}'.format(user_id) user_table_name = datasource_name header = DatasourceModel.get_columns(user_schema_name, user_table_name) return {'message': 'sucess', 'header': header}, 200