def get_data_details(project_id: str): """ Route to get the details for all data for a given project matching the project_id. Raises an HTTPNotFoundError if the project is not found in the database. :param project_id: the project_id to get the data details for :return: a tuple containing (json response, http status code) """ args = {key: val for key, val in request.args.items()} _LOGGER.info( "getting all the data for project_id {} and request args {}".format( project_id, args)) args = SearchProjectDataSchema().load(args) # Validate project and model get_project_by_id(project_id) get_project_model_by_project_id(project_id) project_data = (ProjectData.select().where( ProjectData.project_id == project_id).group_by(ProjectData).order_by( ProjectData.created).paginate(args["page"], args["page_length"])) resp_data = data_dump_and_validation(ResponseProjectDataSchema(), {"data": project_data}) _LOGGER.info("sending project data {}".format(resp_data)) return jsonify(resp_data), HTTPStatus.OK.value
def delete_analysis(project_id: str) -> Tuple[Response, int]: """ Route for deleting the model analysis for a given project matching the project_id. Raises an HTTPNotFoundError if the project is not found in the database or a model doesn't exit. :param project_id: the project_id to delete the analysis for :return: a tuple containing (json response, http status code) """ _LOGGER.info( "deleting analysis for project_id {} model".format(project_id)) get_project_by_id(project_id) project_model = get_project_model_by_project_id(project_id) project_model.analysis = None project_model.save() resp_deleted = data_dump_and_validation( ResponseProjectModelDeletedSchema(), { "project_id": project_id, "model_id": project_model.model_id }, ) _LOGGER.info( "deleted model analysis for project_id {} and model_id {} from path {}" .format(project_id, project_model.model_id, project_model.file_path)) return jsonify(resp_deleted), HTTPStatus.OK.value
def get_analysis(project_id: str) -> Tuple[Response, int]: """ Route for getting the model analysis for a given project matching the project_id. Raises an HTTPNotFoundError if the project is not found in the database or a model doesn't exit. :param project_id: the project_id to get the analysis for :return: a tuple containing (json response, http status code) """ _LOGGER.info("getting model analysis for project_id {}".format(project_id)) get_project_by_id(project_id) project_model = get_project_model_by_project_id(project_id) analysis = (ProjectModelAnalysisSchema().dump(project_model.analysis) if project_model.analysis else None) if analysis is None: raise ValidationError("analysis must be created first") resp_analysis = data_dump_and_validation( ResponseProjectModelAnalysisSchema(), {"analysis": analysis}) _LOGGER.info( "retrieved model analysis for project_id {} and model_id {}".format( project_id, project_model.model_id)) return jsonify(resp_analysis), HTTPStatus.OK.value
def create_analysis(project_id: str) -> Tuple[Response, int]: """ Route for creating a model analysis for a given project matching the project_id. If one exists, will overwrite the previous. Raises an HTTPNotFoundError if the project is not found in the database or a model doesn't exit. :param project_id: the project_id to create the analysis for :return: a tuple containing (json response, http status code) """ _LOGGER.info( "creating analysis for project_id {} model".format(project_id)) get_project_by_id(project_id) project_model = get_project_model_by_project_id(project_id) project_model.validate_filesystem() analyzer = ModelAnalyzer(project_model.file_path) analysis = ProjectModelAnalysisSchema().load(analyzer.dict()) project_model.analysis = analysis project_model.save() resp_analysis = data_dump_and_validation( ResponseProjectModelAnalysisSchema(), {"analysis": analysis}) _LOGGER.info( "analyzed model for project_id {} and model_id {} from path {}".format( project_id, project_model.model_id, project_model.file_path)) return jsonify(resp_analysis), HTTPStatus.OK.value
def get_data_single_details(project_id: str, data_id: str): """ Route to get the details for all data for a given project matching the project_id. Raises an HTTPNotFoundError if the project or data is not found in the database. :param project_id: the project_id to get the data details for :param data_id: the data_id to get the data details for :return: a tuple containing (json response, http status code) """ _LOGGER.info("getting the data with data_id {} for project_id {}".format( data_id, project_id)) get_project_by_id(project_id) get_project_model_by_project_id(project_id) project_data = get_project_data_by_ids(project_id, data_id) resp_data = data_dump_and_validation(ResponseProjectDataSingleSchema(), {"data": project_data}) _LOGGER.info("sending project data from {}".format(project_data.file_path)) return jsonify(resp_data), HTTPStatus.OK.value
def _add_model_check(project_id: str) -> Project: project = get_project_by_id(project_id) project_model = get_project_model_by_project_id(project_id, raise_not_found=False) if project_model is not None: raise ValidationError( ("A model is already set for the project with id {}, " "can only have one model per project").format(project_id)) return project
def delete_data(project_id: str, data_id: str): """ Route to delete a data file for a given project matching the project_id. Raises an HTTPNotFoundError if the project or data is not found in the database. :param project_id: the project_id to get the data for :param data_id: the data_id to get the data for :return: a tuple containing (json response, http status code) """ _LOGGER.info("deleting data with data_id {} for project_id {}".format( data_id, project_id)) get_project_by_id(project_id) get_project_model_by_project_id(project_id) project_data = get_project_data_by_ids(project_id, data_id) args = {key: val for key, val in request.args.items()} try: project_data.delete_instance() project_data.delete_filesystem() except Exception as err: _LOGGER.error( "error while deleting project data for {}, rolling back: {}". format(data_id, err)) if not args["force"]: raise err resp_deleted = data_dump_and_validation( ResponseProjectDataDeletedSchema(), { "project_id": project_id, "data_id": data_id }, ) _LOGGER.info( "deleted data for project_id {} and data_id {} from path {}".format( project_id, project_data.data_id, project_data.dir_path)) return jsonify(resp_deleted), HTTPStatus.OK.value
def get_model_file(project_id: str) -> Tuple[Response, int]: """ Route to get the model details for a given project matching the project_id. Raises an HTTPNotFoundError if the project is not found in the database or a model doesn't exit. :param project_id: the project_id to get the model details for :return: a tuple containing (json response, http status code) """ _LOGGER.info("getting the model file for project_id {}".format(project_id)) get_project_by_id(project_id) project_model = get_project_model_by_project_id(project_id) project_model.validate_filesystem() _LOGGER.info("sending project model file from {}".format( project_model.file_path)) return send_file(project_model.file_path, mimetype="application/octet-stream")
def delete_model(project_id: str) -> Tuple[Response, int]: """ Route to delete the model for a given project matching the project_id. Raises an HTTPNotFoundError if the project is not found in the database or a model doesn't exit. :param project_id: the project_id to delete the model for :return: a tuple containing (json response, http status code) """ _LOGGER.info("deleting the model for project_id {}".format(project_id)) args = DeleteProjectModelSchema().load( {key: val for key, val in request.args.items()}) get_project_by_id(project_id) project_model = get_project_model_by_project_id(project_id) model_id = project_model.model_id try: project_model.delete_instance() project_model.delete_filesystem() except Exception as err: _LOGGER.error( "error while deleting project model for {}, rolling back: {}". format(project_id, err)) if not args["force"]: raise err resp_deleted = data_dump_and_validation( ResponseProjectModelDeletedSchema(), { "project_id": project_id, "model_id": model_id }, ) _LOGGER.info( "deleted model for project_id {} and model_id {} from path {}".format( project_id, project_model.model_id, project_model.dir_path)) return jsonify(resp_deleted), HTTPStatus.OK.value
def load_data_from_repo(project_id: str): """ Route for loading data file(s) for a project from the Neural Magic model repo. Starts a background job in the JobWorker setup to run. The state of the job can be checked after. Raises an HTTPNotFoundError if the project is not found in the database. :param project_id: the id of the project to load the data for :return: a tuple containing (json response, http status code) """ _LOGGER.info( "loading data from repo for project {} for request json {}".format( project_id, request.json)) project = get_project_by_id(project_id) get_project_model_by_project_id(project_id) data = SetProjectDataFromSchema().load(request.get_json(force=True)) project_data = None job = None try: project_data = ProjectData.create(project=project, source="downloaded_path", job=None) job = Job.create( project_id=project.project_id, type_=DataFromRepoJobWorker.get_type(), worker_args=DataFromRepoJobWorker.format_args( data_id=project_data.data_id, uri=data["uri"]), ) project_data.job = job project_data.save() project_data.setup_filesystem() project_data.validate_filesystem() except Exception as err: if project_data: try: os.remove(project_data.file_path) except OSError: pass try: project_data.delete_instance() except Exception as rollback_err: _LOGGER.error("error while rolling back new data: {}".format( rollback_err)) if job: try: job.delete_instance() except Exception as rollback_err: _LOGGER.error("error while rolling back new data: {}".format( rollback_err)) _LOGGER.error( "error while creating new project data, rolling back: {}".format( err)) raise err # call into JobWorkerManager to kick off job if it's not already running JobWorkerManager().refresh() resp_data = data_dump_and_validation(ResponseProjectDataSingleSchema(), {"data": project_data}) _LOGGER.info("created project data from path {}".format(resp_data)) return jsonify(resp_data), HTTPStatus.OK.value
def upload_data(project_id: str): """ Route for uploading a data file to a project. Raises an HTTPNotFoundError if the project is not found in the database. :param project_id: the id of the project to upload the data for :return: a tuple containing (json response, http status code) """ _LOGGER.info("uploading data file for project {}".format(project_id)) project = get_project_by_id(project_id) project_model = get_project_model_by_project_id(project_id) if "data_file" not in request.files: _LOGGER.error("missing uploaded file 'data_file'") raise ValidationError("missing uploaded file 'data_file'") data_file = request.files["data_file"] with NamedTemporaryFile() as temp: data_path = gettempdir() tempname = os.path.join(data_path, temp.name) data_file.save(tempname) try: _LOGGER.info(project_model.file_path) validate_model_data(os.path.join(data_path, "*"), project_model.file_path) data = CreateUpdateProjectDataSchema().dump({ "source": "uploaded", "job": None }) project_data = ProjectData.create(project=project, **data) project_data.file = "{}.npz".format(project_data) project_data.setup_filesystem() shutil.copy(tempname, project_data.file_path) project_data.validate_filesystem() validate_model_data(project_data.file_path, project_model.file_path) project_data.save() except Exception as err: if project_data: try: os.remove(project_data.file_path) except OSError: pass try: project_data.delete_instance() except Exception as rollback_err: _LOGGER.error( "error while rolling back new data: {}".format( rollback_err)) _LOGGER.error( "error while creating new project data, rolling back: {}". format(err)) raise err resp_data = data_dump_and_validation(ResponseProjectDataSingleSchema(), {"data": project_data}) _LOGGER.info("created project data {}".format(resp_data)) return jsonify(resp_data), HTTPStatus.OK.value