def get_data_details(project_id: str):
"""
Route to get the details for all data for a given project matching the project_id.
Raises an HTTPNotFoundError if the project is not found in the database.
:param project_id: the project_id to get the data details for
:return: a tuple containing (json response, http status code)
"""
args = {key: val for key, val in request.args.items()}
_LOGGER.info(
"getting all the data for project_id {} and request args {}".format(
project_id, args))
args = SearchProjectDataSchema().load(args)
# Validate project and model
get_project_by_id(project_id)
get_project_model_by_project_id(project_id)
project_data = (ProjectData.select().where(
ProjectData.project_id == project_id).group_by(ProjectData).order_by(
ProjectData.created).paginate(args["page"], args["page_length"]))
resp_data = data_dump_and_validation(ResponseProjectDataSchema(),
{"data": project_data})
_LOGGER.info("sending project data {}".format(resp_data))
return jsonify(resp_data), HTTPStatus.OK.value
def delete_analysis(project_id: str) -> Tuple[Response, int]:
"""
Route for deleting the model analysis for a given project matching the project_id.
Raises an HTTPNotFoundError if the project is not found in the database
or a model doesn't exit.
:param project_id: the project_id to delete the analysis for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info(
"deleting analysis for project_id {} model".format(project_id))
get_project_by_id(project_id)
project_model = get_project_model_by_project_id(project_id)
project_model.analysis = None
project_model.save()
resp_deleted = data_dump_and_validation(
ResponseProjectModelDeletedSchema(),
{
"project_id": project_id,
"model_id": project_model.model_id
},
)
_LOGGER.info(
"deleted model analysis for project_id {} and model_id {} from path {}"
.format(project_id, project_model.model_id, project_model.file_path))
return jsonify(resp_deleted), HTTPStatus.OK.value
def get_analysis(project_id: str) -> Tuple[Response, int]:
"""
Route for getting the model analysis for a given project matching the project_id.
Raises an HTTPNotFoundError if the project is not found in the database
or a model doesn't exit.
:param project_id: the project_id to get the analysis for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info("getting model analysis for project_id {}".format(project_id))
get_project_by_id(project_id)
project_model = get_project_model_by_project_id(project_id)
analysis = (ProjectModelAnalysisSchema().dump(project_model.analysis)
if project_model.analysis else None)
if analysis is None:
raise ValidationError("analysis must be created first")
resp_analysis = data_dump_and_validation(
ResponseProjectModelAnalysisSchema(), {"analysis": analysis})
_LOGGER.info(
"retrieved model analysis for project_id {} and model_id {}".format(
project_id, project_model.model_id))
return jsonify(resp_analysis), HTTPStatus.OK.value
def create_analysis(project_id: str) -> Tuple[Response, int]:
"""
Route for creating a model analysis for a given project matching the project_id.
If one exists, will overwrite the previous.
Raises an HTTPNotFoundError if the project is not found in the database
or a model doesn't exit.
:param project_id: the project_id to create the analysis for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info(
"creating analysis for project_id {} model".format(project_id))
get_project_by_id(project_id)
project_model = get_project_model_by_project_id(project_id)
project_model.validate_filesystem()
analyzer = ModelAnalyzer(project_model.file_path)
analysis = ProjectModelAnalysisSchema().load(analyzer.dict())
project_model.analysis = analysis
project_model.save()
resp_analysis = data_dump_and_validation(
ResponseProjectModelAnalysisSchema(), {"analysis": analysis})
_LOGGER.info(
"analyzed model for project_id {} and model_id {} from path {}".format(
project_id, project_model.model_id, project_model.file_path))
return jsonify(resp_analysis), HTTPStatus.OK.value
def get_data_single_details(project_id: str, data_id: str):
"""
Route to get the details for all data for a given project matching the project_id.
Raises an HTTPNotFoundError if the project or data is not found in the database.
:param project_id: the project_id to get the data details for
:param data_id: the data_id to get the data details for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info("getting the data with data_id {} for project_id {}".format(
data_id, project_id))
get_project_by_id(project_id)
get_project_model_by_project_id(project_id)
project_data = get_project_data_by_ids(project_id, data_id)
resp_data = data_dump_and_validation(ResponseProjectDataSingleSchema(),
{"data": project_data})
_LOGGER.info("sending project data from {}".format(project_data.file_path))
return jsonify(resp_data), HTTPStatus.OK.value
def _add_model_check(project_id: str) -> Project:
project = get_project_by_id(project_id)
project_model = get_project_model_by_project_id(project_id,
raise_not_found=False)
if project_model is not None:
raise ValidationError(
("A model is already set for the project with id {}, "
"can only have one model per project").format(project_id))
return project
def delete_data(project_id: str, data_id: str):
"""
Route to delete a data file for a given project matching the project_id.
Raises an HTTPNotFoundError if the project or data is not found in the database.
:param project_id: the project_id to get the data for
:param data_id: the data_id to get the data for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info("deleting data with data_id {} for project_id {}".format(
data_id, project_id))
get_project_by_id(project_id)
get_project_model_by_project_id(project_id)
project_data = get_project_data_by_ids(project_id, data_id)
args = {key: val for key, val in request.args.items()}
try:
project_data.delete_instance()
project_data.delete_filesystem()
except Exception as err:
_LOGGER.error(
"error while deleting project data for {}, rolling back: {}".
format(data_id, err))
if not args["force"]:
raise err
resp_deleted = data_dump_and_validation(
ResponseProjectDataDeletedSchema(),
{
"project_id": project_id,
"data_id": data_id
},
)
_LOGGER.info(
"deleted data for project_id {} and data_id {} from path {}".format(
project_id, project_data.data_id, project_data.dir_path))
return jsonify(resp_deleted), HTTPStatus.OK.value
def get_model_file(project_id: str) -> Tuple[Response, int]:
"""
Route to get the model details for a given project matching the project_id.
Raises an HTTPNotFoundError if the project is not found in the database
or a model doesn't exit.
:param project_id: the project_id to get the model details for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info("getting the model file for project_id {}".format(project_id))
get_project_by_id(project_id)
project_model = get_project_model_by_project_id(project_id)
project_model.validate_filesystem()
_LOGGER.info("sending project model file from {}".format(
project_model.file_path))
return send_file(project_model.file_path,
mimetype="application/octet-stream")
def delete_model(project_id: str) -> Tuple[Response, int]:
"""
Route to delete the model for a given project matching the project_id.
Raises an HTTPNotFoundError if the project is not found in the database
or a model doesn't exit.
:param project_id: the project_id to delete the model for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info("deleting the model for project_id {}".format(project_id))
args = DeleteProjectModelSchema().load(
{key: val
for key, val in request.args.items()})
get_project_by_id(project_id)
project_model = get_project_model_by_project_id(project_id)
model_id = project_model.model_id
try:
project_model.delete_instance()
project_model.delete_filesystem()
except Exception as err:
_LOGGER.error(
"error while deleting project model for {}, rolling back: {}".
format(project_id, err))
if not args["force"]:
raise err
resp_deleted = data_dump_and_validation(
ResponseProjectModelDeletedSchema(),
{
"project_id": project_id,
"model_id": model_id
},
)
_LOGGER.info(
"deleted model for project_id {} and model_id {} from path {}".format(
project_id, project_model.model_id, project_model.dir_path))
return jsonify(resp_deleted), HTTPStatus.OK.value
def load_data_from_repo(project_id: str):
"""
Route for loading data file(s) for a project from the Neural Magic model repo.
Starts a background job in the JobWorker setup to run.
The state of the job can be checked after.
Raises an HTTPNotFoundError if the project is not found in the database.
:param project_id: the id of the project to load the data for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info(
"loading data from repo for project {} for request json {}".format(
project_id, request.json))
project = get_project_by_id(project_id)
get_project_model_by_project_id(project_id)
data = SetProjectDataFromSchema().load(request.get_json(force=True))
project_data = None
job = None
try:
project_data = ProjectData.create(project=project,
source="downloaded_path",
job=None)
job = Job.create(
project_id=project.project_id,
type_=DataFromRepoJobWorker.get_type(),
worker_args=DataFromRepoJobWorker.format_args(
data_id=project_data.data_id, uri=data["uri"]),
)
project_data.job = job
project_data.save()
project_data.setup_filesystem()
project_data.validate_filesystem()
except Exception as err:
if project_data:
try:
os.remove(project_data.file_path)
except OSError:
pass
try:
project_data.delete_instance()
except Exception as rollback_err:
_LOGGER.error("error while rolling back new data: {}".format(
rollback_err))
if job:
try:
job.delete_instance()
except Exception as rollback_err:
_LOGGER.error("error while rolling back new data: {}".format(
rollback_err))
_LOGGER.error(
"error while creating new project data, rolling back: {}".format(
err))
raise err
# call into JobWorkerManager to kick off job if it's not already running
JobWorkerManager().refresh()
resp_data = data_dump_and_validation(ResponseProjectDataSingleSchema(),
{"data": project_data})
_LOGGER.info("created project data from path {}".format(resp_data))
return jsonify(resp_data), HTTPStatus.OK.value
def upload_data(project_id: str):
"""
Route for uploading a data file to a project.
Raises an HTTPNotFoundError if the project is not found in the database.
:param project_id: the id of the project to upload the data for
:return: a tuple containing (json response, http status code)
"""
_LOGGER.info("uploading data file for project {}".format(project_id))
project = get_project_by_id(project_id)
project_model = get_project_model_by_project_id(project_id)
if "data_file" not in request.files:
_LOGGER.error("missing uploaded file 'data_file'")
raise ValidationError("missing uploaded file 'data_file'")
data_file = request.files["data_file"]
with NamedTemporaryFile() as temp:
data_path = gettempdir()
tempname = os.path.join(data_path, temp.name)
data_file.save(tempname)
try:
_LOGGER.info(project_model.file_path)
validate_model_data(os.path.join(data_path, "*"),
project_model.file_path)
data = CreateUpdateProjectDataSchema().dump({
"source": "uploaded",
"job": None
})
project_data = ProjectData.create(project=project, **data)
project_data.file = "{}.npz".format(project_data)
project_data.setup_filesystem()
shutil.copy(tempname, project_data.file_path)
project_data.validate_filesystem()
validate_model_data(project_data.file_path,
project_model.file_path)
project_data.save()
except Exception as err:
if project_data:
try:
os.remove(project_data.file_path)
except OSError:
pass
try:
project_data.delete_instance()
except Exception as rollback_err:
_LOGGER.error(
"error while rolling back new data: {}".format(
rollback_err))
_LOGGER.error(
"error while creating new project data, rolling back: {}".
format(err))
raise err
resp_data = data_dump_and_validation(ResponseProjectDataSingleSchema(),
{"data": project_data})
_LOGGER.info("created project data {}".format(resp_data))
return jsonify(resp_data), HTTPStatus.OK.value