def commit_project(project_id, message, author, changes):
channel = grpc.insecure_channel(config.PROJECTS_SERVICES_ENDPOINT)
stub = project_pb2_grpc.ProjectServicesStub(channel=channel)
project = stub.Retrieve(project_pb2.ID(id=project_id))
etcd = etcd3.client(host=config.ETCD_HOST, port=config.ETCD_PORT)
# acquire lock
with etcd.lock(project_id, ttl=1800) as lock:
# copy data to repo_bucket
minio_client = get_minio_client()
for change in changes:
key = os.path.relpath(change["path"],
config.ILYDE_WORKING_DIR).replace('\\', '/')
if change['action'] == "deleted":
minio_client.remove_object(bucket_name=project.repo_bucket,
object_name=key)
else:
file_type, _ = mimetypes.guess_type(change["path"])
if file_type is None:
file_type = 'application/octet-stream'
minio_client.fput_object(project.repo_bucket,
key,
change["path"],
content_type=file_type)
# create new revision
payload = {'project': project_id, 'commit': message, 'author': author}
response = stub.CreateRevision(project_pb2.Revision(**payload))
return response
def check_modelapi_permission(obj, token_info):
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=obj.metadata.project))
if not permissions.IsProjectMember.has_object_permission(
token_info, project):
raise connexion.ProblemException(
status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action")
def create_model(body, **kwargs):
"""Create a model
:param body: Request payload to create a model
:type body: dict | bytes
:rtype: ModelSerializer
"""
serializer = ModelSerializer.from_dict(body)
description = serializer.description if serializer.description else ""
project_id = serializer.tags.get("ilyde.project")
if not project_id:
raise connexion.ProblemException(
status=400,
title="Bad Request",
detail=
"ilyde.project tag should be provided with the id of the project.")
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=project_id))
if not permissions.IsProjectMember.has_object_permission(
kwargs["token_info"], project):
raise connexion.ProblemException(
status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action")
model_name = util.safe_model_name(serializer.name)
if not model_name.startswith(util.safe_model_name(project.name)):
model_name = "{}-{}".format(util.safe_model_name(project.name),
model_name)
mlflow_client = get_mlflow_client()
model = mlflow_client.create_registered_model(name=model_name,
description=description,
tags=serializer.tags)
return ModelSerializer(
name=model.name,
description=model.description,
tags=model.tags,
latest_versions=[
ModelVersionSerializer(
creation_timestamp=version.creation_timestamp,
current_stage=version.current_stage,
description=version.description,
last_updated_timestamp=version.last_updated_timestamp,
name=version.name,
run_id=version.run_id,
source=version.source,
status=version.status,
status_message=version.status_message,
tags=version.tags,
version=version.version) for version in model.latest_versions
],
last_updated_timestamp=model.last_updated_timestamp,
creation_timestamp=model.creation_timestamp)
def copy_project(project_id, revision_id):
channel = grpc.insecure_channel(config.PROJECTS_SERVICES_ENDPOINT)
stub = project_pb2_grpc.ProjectServicesStub(channel=channel)
project = stub.Retrieve(project_pb2.ID(id=project_id))
revision = stub.RetrieveRevision(project_pb2.ID(id=revision_id))
# get minio client
minio_client = get_minio_client()
for file in revision.file_tree:
destination = os.path.join(config.ILYDE_WORKING_DIR, file.name)
try:
obj = minio_client.fget_object(bucket_name=project.repo_bucket,
object_name=file.name,
file_path=destination,
version_id=file.version)
# add modification time
os.utime(destination, (time.mktime(
time.localtime()), time.mktime(obj.last_modified)))
except MinioError as e:
pass
except Exception as e:
pass
def delete_project(id_, **kwargs):
"""Delete a project
:param id_: The ID of the project object to be deleted
:type id_: str
:rtype: StatusSerializer
"""
proj = get_project_object(id_)
check_project_permission(proj, kwargs["token_info"])
stub = get_projects_services_stub()
response = stub.Delete(project_pb2.ID(id=proj.id))
if response.status != 200:
return ErrorSerializer(status=response.status,
title="Api Error",
detail=response.message), response.status
return StatusSerializer.from_dict(util.deserialize_protobuf(response))
def get_experiment_artifact(id_, path, **kwargs): # noqa: E501
"""Download experiment artifact
Download experiment artifact
:param id_: The ID of the experiment resource
:type id_: str
:param path: File's path to download
:type path: str
:rtype: file
"""
stub = get_experiments_services_stub()
experiment = stub.Get(job_pb2.ID(id=id_))
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=experiment.metadata.project))
if not IsProjectMember.has_object_permission(kwargs["token_info"], project):
return ErrorSerializer(status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action"), 403
mlflow_client = get_mlflow_client()
all_experiments = [exp.experiment_id for exp in mlflow_client.list_experiments()]
run = mlflow_client.search_runs(experiment_ids=all_experiments,
filter_string="tags.`ilyde.job` = '{}'".format(experiment.id),
run_view_type=ViewType.ALL)[0]
local_dir = os.path.dirname(os.path.join(current_app.config.get("BASE_DIR"), "media", run.info.run_id, path))
if not os.path.exists(local_dir):
os.makedirs(local_dir, exist_ok=True)
local_path = mlflow_client.download_artifacts(run.info.run_id, path, local_dir)
file_type, _ = mimetypes.guess_type(local_path)
if file_type is None:
file_type = 'application/octet-stream'
with open(local_path, 'rb') as f:
response = make_response(f.read())
response.headers.set('Content-Type', file_type)
return response, 200
def get_project_file(id_, path, version, **kwargs):
"""Download project's file
Download project's file
:param id_: The ID of the project resource
:type id_: str
:param path: File's path to download
:type version: str
:param version: File's version to download
:type version: str
:rtype: file
"""
stub = get_projects_services_stub()
project = stub.Retrieve(project_pb2.ID(id=id_))
if not IsProjectMember.has_object_permission(kwargs["token_info"], project):
return ErrorSerializer(status=403,
title="Permission Denied",
detail="Doesn't have enough permissions to take this action"), 403
extra_params = {}
if version is not None:
extra_params['version_id'] = version
client = services.get_minio_client()
data = client.get_object(project.repo_bucket, path, **extra_params)
if int(data.getheader('Content-Length')) > 1024 * 1024 * 30:
return ErrorSerializer(status=400, title="Bad Request",
detail="File is too large."), 400
response = make_response(data.read())
response.headers.set('Content-Type', data.getheader('Content-Type'))
return response, 200
def retrieve_project_revision(revision_id):
channel = grpc.insecure_channel(config.PROJECTS_SERVICES_ENDPOINT)
stub = project_pb2_grpc.ProjectServicesStub(channel=channel)
return stub.RetrieveRevision(project_pb2.ID(id=revision_id))
def get_project_object(project_id):
stub = get_projects_services_stub()
return stub.Retrieve(project_pb2.ID(id=project_id))