def get_latest_project_revision(project_id):
stub = get_projects_services_stub()
response = stub.SearchRevision(
project_pb2.SearchRevisionRequest(query={"project": project_id},
page=1,
limit=5))
revision = response.data[0]
return revision.id
def check_modelapi_permission(obj, token_info):
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=obj.metadata.project))
if not permissions.IsProjectMember.has_object_permission(
token_info, project):
raise connexion.ProblemException(
status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action")
def create_model(body, **kwargs):
"""Create a model
:param body: Request payload to create a model
:type body: dict | bytes
:rtype: ModelSerializer
"""
serializer = ModelSerializer.from_dict(body)
description = serializer.description if serializer.description else ""
project_id = serializer.tags.get("ilyde.project")
if not project_id:
raise connexion.ProblemException(
status=400,
title="Bad Request",
detail=
"ilyde.project tag should be provided with the id of the project.")
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=project_id))
if not permissions.IsProjectMember.has_object_permission(
kwargs["token_info"], project):
raise connexion.ProblemException(
status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action")
model_name = util.safe_model_name(serializer.name)
if not model_name.startswith(util.safe_model_name(project.name)):
model_name = "{}-{}".format(util.safe_model_name(project.name),
model_name)
mlflow_client = get_mlflow_client()
model = mlflow_client.create_registered_model(name=model_name,
description=description,
tags=serializer.tags)
return ModelSerializer(
name=model.name,
description=model.description,
tags=model.tags,
latest_versions=[
ModelVersionSerializer(
creation_timestamp=version.creation_timestamp,
current_stage=version.current_stage,
description=version.description,
last_updated_timestamp=version.last_updated_timestamp,
name=version.name,
run_id=version.run_id,
source=version.source,
status=version.status,
status_message=version.status_message,
tags=version.tags,
version=version.version) for version in model.latest_versions
],
last_updated_timestamp=model.last_updated_timestamp,
creation_timestamp=model.creation_timestamp)
def create_project(body, **kwargs):
"""Create a project
:param body: Project object to be created
:type body: dict | bytes
:rtype: ProjectSerializer
"""
stub = get_projects_services_stub()
body["owner"] = kwargs["user"]
response = stub.Create(project_pb2.Project(**body))
return ProjectSerializer.from_dict(util.deserialize_protobuf(response))
def update_project(id_, body, **kwargs):
"""Update a project
:param id_: The ID of the project object to be updated
:type id_: str
:param body: Project object to be updated
:type body: dict | bytes
:rtype: ProjectSerializer
"""
proj = get_project_object(id_)
check_project_permission(proj, kwargs["token_info"])
stub = get_projects_services_stub()
proj.description = body["description"]
response = stub.Update(proj)
return ProjectSerializer.from_dict(util.deserialize_protobuf(response))
def delete_project(id_, **kwargs):
"""Delete a project
:param id_: The ID of the project object to be deleted
:type id_: str
:rtype: StatusSerializer
"""
proj = get_project_object(id_)
check_project_permission(proj, kwargs["token_info"])
stub = get_projects_services_stub()
response = stub.Delete(project_pb2.ID(id=proj.id))
if response.status != 200:
return ErrorSerializer(status=response.status,
title="Api Error",
detail=response.message), response.status
return StatusSerializer.from_dict(util.deserialize_protobuf(response))
def close_project(id_, **kwargs):
"""Close a project
:param id_: The ID of the project object to be updated
:type id_: str
:rtype: ProjectSerializer
"""
proj = get_project_object(id_)
check_project_permission(proj, kwargs["token_info"])
if proj.owner != kwargs["user"]:
raise connexion.ProblemException(
status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action")
proj.state = project_pb2.STATE.CLOSED
stub = get_projects_services_stub()
response = stub.Update(proj)
return ProjectSerializer.from_dict(util.deserialize_protobuf(response))
def list_projects(body=None, **kwargs):
"""List projects
:param body:
:type body: dict | bytes
:rtype: PageLimitListSerializer
"""
query = body.get("query")
if not permissions.IsAdmin.has_permission(
kwargs["token_info"]) and query["visibility"] == "PRIVATE":
query["member"] = kwargs["user"]
stub = get_projects_services_stub()
response = stub.Search(
project_pb2.SearchProjectRequest(query=query,
limit=body.get("limit"),
page=body.get("page")))
return util.deserialize_protobuf(response)
def get_experiment_artifact(id_, path, **kwargs): # noqa: E501
"""Download experiment artifact
Download experiment artifact
:param id_: The ID of the experiment resource
:type id_: str
:param path: File's path to download
:type path: str
:rtype: file
"""
stub = get_experiments_services_stub()
experiment = stub.Get(job_pb2.ID(id=id_))
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=experiment.metadata.project))
if not IsProjectMember.has_object_permission(kwargs["token_info"], project):
return ErrorSerializer(status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action"), 403
mlflow_client = get_mlflow_client()
all_experiments = [exp.experiment_id for exp in mlflow_client.list_experiments()]
run = mlflow_client.search_runs(experiment_ids=all_experiments,
filter_string="tags.`ilyde.job` = '{}'".format(experiment.id),
run_view_type=ViewType.ALL)[0]
local_dir = os.path.dirname(os.path.join(current_app.config.get("BASE_DIR"), "media", run.info.run_id, path))
if not os.path.exists(local_dir):
os.makedirs(local_dir, exist_ok=True)
local_path = mlflow_client.download_artifacts(run.info.run_id, path, local_dir)
file_type, _ = mimetypes.guess_type(local_path)
if file_type is None:
file_type = 'application/octet-stream'
with open(local_path, 'rb') as f:
response = make_response(f.read())
response.headers.set('Content-Type', file_type)
return response, 200
def create_project_revision(id_, **kwargs):
"""Create project's revision
:param id_: The ID of the project resource
:type id_: str
:rtype: ProjectRevisionSerializer
"""
proj = get_project_object(id_)
check_project_permission(proj, kwargs["token_info"])
# upload file in Minio
files = connexion.request.files
client = get_minio_client()
# acquire lock to write
etcd = etcd3.client(host=current_app.config.get("ETCD_HOST"),
port=current_app.config.get("ETCD_PORT"))
# acquire lock
commit_message = "Write"
stub = get_projects_services_stub()
with etcd.lock(proj.id, ttl=1800) as lock:
for file in files.getlist('files'):
filename = secure_filename(
file.filename
) # This is convenient to validate your filename, otherwise just use file.filename
file_type, _ = mimetypes.guess_type(filename)
bio = io.BytesIO(file.read())
client.put_object(data=bio,
bucket_name=proj.repo_bucket,
object_name=filename,
content_type=file_type,
length=len(bio.getvalue()))
commit_message += " {}".format(filename)
response = stub.CreateRevision(
project_pb2.Revision(project=proj.id,
commit=commit_message,
author=kwargs["user"]))
return ProjectRevisionSerializer.from_dict(
util.deserialize_protobuf(response))
def list_project_revisions(id_, limit=None, page=None, **kwargs):
"""List project's revisions
:param id_: The ID of the project resource
:type id_: str
:param limit:
:type limit: int
:param page:
:type page: int
:rtype: PageLimitListSerializer
"""
proj = get_project_object(id_)
check_project_permission(proj, kwargs["token_info"])
stub = get_projects_services_stub()
response = stub.SearchRevision(
project_pb2.SearchRevisionRequest(query={"project": proj.id},
limit=limit,
page=page))
return util.deserialize_protobuf(response)
def remove_project_member(id_, body, **kwargs):
"""Close a project
:param id_: The ID of the project object to be updated
:type id_: str
:param body: User Id
:type body: dict | bytes
:rtype: ProjectSerializer
"""
proj = get_project_object(id_)
check_project_permission(proj, kwargs["token_info"])
if proj.owner != kwargs["user"]:
raise connexion.ProblemException(
status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action")
if body["user"] in proj.members:
proj.members.remove(body["user"])
stub = get_projects_services_stub()
proj = stub.Update(proj)
return ProjectSerializer.from_dict(util.deserialize_protobuf(proj))
def get_project_file(id_, path, version, **kwargs):
"""Download project's file
Download project's file
:param id_: The ID of the project resource
:type id_: str
:param path: File's path to download
:type version: str
:param version: File's version to download
:type version: str
:rtype: file
"""
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=id_))
if not IsProjectMember.has_object_permission(kwargs["token_info"], project):
return ErrorSerializer(status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action"), 403
extra_params = {}
if version is not None:
extra_params['version_id'] = version
client = services.get_minio_client()
data = client.get_object(project.repo_bucket, path, **extra_params)
if int(data.getheader('Content-Length')) > 1024 * 1024 * 30:
return ErrorSerializer(status=400, title="Bad Request",
detail="File is too large."), 400
response = make_response(data.read())
response.headers.set('Content-Type', data.getheader('Content-Type'))
return response, 200
def get_project_object(project_id):
stub = get_projects_services_stub()
return stub.Retrieve(project_pb2.ID(id=project_id))
