def migrate_project_view(user_data, cache):
"""Migrate specified project."""
ctx = ProjectMigrateRequest().load(request.json)
user = cache.ensure_user(user_data)
project = cache.get_project(user, ctx["project_id"])
commit_message = ctx.get("commit_message", None)
if ctx.get("is_delayed", False):
job = cache.make_job(user,
project=project,
job_data={
"renku_op": "migrate_job",
"client_extras": ctx.get("client_extras")
})
with enqueue_retry(MIGRATIONS_JOB_QUEUE) as queue:
queue.enqueue(
migrate_job,
user_data,
project.project_id,
job.job_id,
commit_message,
)
return result_response(ProjectMigrateAsyncResponseRPC(), job)
messages, was_migrated = execute_migration(project, commit_message)
response = {"messages": messages, "was_migrated": was_migrated}
if was_migrated:
_, response["remote_branch"] = repo_sync(Repo(project.abs_path),
remote="origin")
return result_response(ProjectMigrateResponseRPC(), response)
def to_response(self):
"""Execute controller flow and serialize to service response."""
if "project_id" in self.ctx:
return result_response(MigrationsCheckCtrl.RESPONSE_SERIALIZER,
self.local())
elif "git_url" in self.ctx:
return result_response(MigrationsCheckCtrl.RESPONSE_SERIALIZER,
self.remote())
def job_details(user_data, cache, job_id):
"""Show details for a specific job."""
user = cache.ensure_user(user_data)
job = cache.get_job(user, job_id)
if not job or not job.project_id:
return result_response(JobDetailsResponseRPC(), None)
job.project = cache.get_project(user, job.project_id)
return result_response(JobDetailsResponseRPC(), job)
def unlink_file_view(user_data, cache):
"""Unlink a file from a dataset."""
ctx = DatasetUnlinkRequest().load(request.json)
include = ctx.get('include_filter')
exclude = ctx.get('exclude_filter')
user = cache.ensure_user(user_data)
project = cache.get_project(user, ctx['project_id'])
if ctx.get('commit_message') is None:
if include and exclude:
filters = '-I {0} -X {0}'.format(include, exclude)
elif not include and exclude:
filters = '-X {0}'.format(exclude)
else:
filters = '-I {0}'.format(include)
ctx['commit_message'] = ('service: unlink dataset {0} {1}'.format(
ctx['short_name'], filters))
with chdir(project.abs_path):
records = file_unlink(short_name=ctx['short_name'],
include=ctx.get('include_filters'),
exclude=ctx.get('exclude_filters'),
yes=True,
interactive=False,
commit_message=ctx['commit_message'])
unlinked = [record.path for record in records]
return result_response(DatasetUnlinkResponseRPC(), {'unlinked': unlinked})
def edit_dataset_view(user_data, cache):
"""Edit dataset metadata."""
ctx = DatasetEditRequest().load(request.json)
user = cache.ensure_user(user_data)
project = cache.get_project(user, ctx['project_id'])
if ctx.get('commit_message') is None:
ctx['commit_message'] = 'service: dataset edit {0}'.format(
ctx['short_name'])
with chdir(project.abs_path):
edited, warnings = edit_dataset(ctx['short_name'],
ctx.get('title'),
ctx.get('description'),
ctx.get('creators'),
keywords=ctx.get('keywords'),
commit_message=ctx['commit_message'])
return result_response(
DatasetEditResponseRPC(), {
'edited': {field: ctx.get(field)
for field in edited},
'warnings': warnings
})
def import_dataset_view(user_data, cache):
"""Import a dataset view."""
user = cache.ensure_user(user_data)
ctx = DatasetImportRequest().load(request.json)
project = cache.get_project(user, ctx['project_id'])
user_job = {
'job_id': uuid.uuid4().hex,
'state': USER_JOB_STATE_ENQUEUED,
}
job = cache.make_job(user, user_job, locked=project.project_id)
with enqueue_retry(DATASETS_JOB_QUEUE) as queue:
queue.enqueue(
dataset_import,
user_data,
user_job['job_id'],
project.project_id,
ctx['dataset_uri'],
short_name=ctx.get('short_name'),
extract=ctx.get('extract', False),
timeout=int(os.getenv('WORKER_DATASET_JOBS_TIMEOUT', 1800)),
result_ttl=int(os.getenv('WORKER_DATASET_JOBS_RESULT_TTL', 500)))
return result_response(DatasetImportResponseRPC(), job)
def edit_dataset_view(user_data, cache):
"""Edit dataset metadata."""
ctx = DatasetEditRequest().load(request.json)
user = cache.ensure_user(user_data)
project = cache.get_project(user, ctx["project_id"])
if ctx.get("commit_message") is None:
ctx["commit_message"] = "service: dataset edit {0}".format(ctx["name"])
with chdir(project.abs_path):
edited, warnings = edit_dataset(
ctx["name"],
ctx.get("title"),
ctx.get("description"),
ctx.get("creators"),
keywords=ctx.get("keywords"),
commit_message=ctx["commit_message"],
)
return result_response(
DatasetEditResponseRPC(), {
"edited": {field: ctx.get(field)
for field in edited},
"warnings": warnings
})
def project_clone(user, cache):
"""Clone a remote repository."""
ctx = ProjectCloneContext().load(
(lambda a, b: a.update(b) or a)(request.json, user),
unknown=EXCLUDE,
)
local_path = make_project_path(user, ctx)
user = cache.ensure_user(user)
if local_path.exists():
shutil.rmtree(str(local_path))
for project in cache.get_projects(user):
if project.git_url == ctx['git_url']:
project.delete()
local_path.mkdir(parents=True, exist_ok=True)
renku_clone(ctx['url_with_auth'],
local_path,
depth=ctx['depth'],
raise_git_except=True,
config={
'user.name': ctx['fullname'],
'user.email': ctx['email'],
})
project = cache.make_project(user, ctx)
return result_response(ProjectCloneResponseRPC(), project)
def import_dataset_view(user_data, cache):
"""Import a dataset view."""
user = cache.ensure_user(user_data)
ctx = DatasetImportRequest().load(request.json)
project = cache.get_project(user, ctx['project_id'])
if project is None or project.abs_path is False:
return error_response(
INVALID_PARAMS_ERROR_CODE,
'invalid project_id: {0}'.format(ctx['project_id']))
user_job = {
'job_id': uuid.uuid4().hex,
'state': USER_JOB_STATE_ENQUEUED,
}
job = cache.make_job(user, user_job)
with enqueue_retry(DATASETS_JOB_QUEUE) as queue:
queue.enqueue(
dataset_import,
user_data,
user_job['job_id'],
ctx['project_id'],
ctx['dataset_uri'],
short_name=ctx.get('short_name'),
extract=ctx.get('extract', False),
timeout=int(os.getenv('WORKER_DATASET_JOBS_TIMEOUT', 1800)),
result_ttl=int(os.getenv('WORKER_DATASET_JOBS_RESULT_TTL', 500)))
return result_response(DatasetImportResponseRPC(), job)
def test_result_response_with_empty_tuple(svc_client):
"""Test result response with empty value."""
response = result_response(DatasetListResponseRPC(), ()).json
assert response
assert {'result'} == set(response.keys())
assert {} == response['result']
def test_result_response_with_empty_dict(svc_client):
"""Test result response with empty value."""
response = result_response(DatasetListResponseRPC(), {}).json
assert response
assert {"result"} == set(response.keys())
assert {} == response["result"]
def test_result_response_with_none(svc_client):
"""Test result response with None value."""
response = result_response(DatasetListResponseRPC(), None).json
assert response
assert {'result'} == set(response.keys())
assert response['result'] is None
def unlink_file_view(user_data, cache):
"""Unlink a file from a dataset."""
ctx = DatasetUnlinkRequest().load(request.json)
include = ctx.get("include_filter")
exclude = ctx.get("exclude_filter")
user = cache.ensure_user(user_data)
project = cache.get_project(user, ctx["project_id"])
if ctx.get("commit_message") is None:
if include and exclude:
filters = "-I {0} -X {1}".format(include, exclude)
elif not include and exclude:
filters = "-X {0}".format(exclude)
else:
filters = "-I {0}".format(include)
ctx["commit_message"] = "service: unlink dataset {0} {1}".format(
ctx["name"], filters)
with chdir(project.abs_path):
records = file_unlink(
name=ctx["name"],
include=ctx.get("include_filters"),
exclude=ctx.get("exclude_filters"),
yes=True,
interactive=False,
commit_message=ctx["commit_message"],
)
unlinked = [record.path for record in records]
return result_response(DatasetUnlinkResponseRPC(), {"unlinked": unlinked})
def import_dataset_view(user_data, cache):
"""Import a dataset view."""
user = cache.ensure_user(user_data)
ctx = DatasetImportRequest().load(request.json)
project = cache.get_project(user, ctx["project_id"])
job = cache.make_job(user,
project=project,
job_data={
"renku_op": "dataset_import",
"client_extras": ctx.get("client_extras")
})
with enqueue_retry(DATASETS_JOB_QUEUE) as queue:
queue.enqueue(
dataset_import,
user_data,
job.job_id,
project.project_id,
ctx["dataset_uri"],
name=ctx.get("name"),
extract=ctx.get("extract", False),
timeout=int(os.getenv("WORKER_DATASET_JOBS_TIMEOUT", 1800)),
result_ttl=int(os.getenv("WORKER_DATASET_JOBS_RESULT_TTL", 500)),
)
return result_response(DatasetImportResponseRPC(), job)
def add_file_to_dataset_view(user_data, cache):
"""Add the uploaded file to cloned repository."""
ctx = DatasetAddRequest().load(request.json)
user = cache.ensure_user(user_data)
project = cache.get_project(user, ctx['project_id'])
if not ctx['commit_message']:
ctx['commit_message'] = 'service: dataset add {0}'.format(
ctx['short_name'])
local_paths = []
for _file in ctx['files']:
local_path = None
if 'file_url' in _file:
commit_message = '{0}{1}'.format(ctx['commit_message'],
_file['file_url'])
job = cache.make_job(user)
_file['job_id'] = job.job_id
with enqueue_retry(DATASETS_JOB_QUEUE) as queue:
queue.enqueue(dataset_add_remote_file, user_data, job.job_id,
project.project_id, ctx['create_dataset'],
commit_message, ctx['short_name'],
_file['file_url'])
continue
if 'file_id' in _file:
file = cache.get_file(user, _file['file_id'])
local_path = file.abs_path
elif 'file_path' in _file:
local_path = project.abs_path / Path(_file['file_path'])
if not local_path or not local_path.exists():
return error_response(
INVALID_PARAMS_ERROR_CODE,
'invalid file reference: {0}'.format(json.dumps(_file)))
ctx['commit_message'] += ' {0}'.format(local_path.name)
local_paths.append(str(local_path))
if local_paths:
with chdir(project.abs_path):
add_file(local_paths,
ctx['short_name'],
create=ctx['create_dataset'],
force=ctx['force'],
commit_message=ctx['commit_message'])
try:
_, ctx['remote_branch'] = repo_sync(Repo(project.abs_path),
remote='origin')
except GitCommandError:
return error_response(INTERNAL_FAILURE_ERROR_CODE,
'repo sync failed')
return result_response(DatasetAddResponseRPC(), ctx)
def list_projects_view(user, cache):
"""List cached projects."""
projects = [
project for project in cache.get_projects(cache.ensure_user(user))
if project.abs_path.exists()
]
return result_response(ProjectListResponseRPC(), {"projects": projects})
def test_result_response(svc_client):
"""Test result response utility."""
ctx = {'datasets': [{'name': 'my-dataset'}]}
response = result_response(DatasetListResponseRPC(), ctx).json
assert response
assert {'result'} == set(response.keys())
assert {'datasets'} == set(response['result'].keys())
assert ctx == response['result']
def to_response(self):
"""Serialize to service version response."""
return result_response(
VersionCtrl.RESPONSE_SERIALIZER,
{
"latest_version": __version__,
"supported_project_version": SUPPORTED_PROJECT_VERSION,
},
)
def list_datasets_view(user, cache):
"""List all datasets in project."""
ctx = DatasetListRequest().load(request.args)
project = cache.get_project(cache.ensure_user(user), ctx['project_id'])
with chdir(project.abs_path):
ctx['datasets'] = list_datasets()
return result_response(DatasetListResponseRPC(), ctx)
def list_dataset_files_view(user, cache):
"""List files in a dataset."""
ctx = DatasetFilesListRequest().load(request.args)
project = cache.get_project(cache.ensure_user(user), ctx['project_id'])
with chdir(project.abs_path):
ctx['files'] = list_files(datasets=[ctx['short_name']])
return result_response(DatasetFilesListResponseRPC(), ctx)
def project_clone_view(user_data):
"""Clone a remote repository."""
project_data = ProjectCloneContext().load({
**user_data,
**request.json
},
unknown=EXCLUDE)
project = _project_clone(user_data, project_data)
return result_response(ProjectCloneResponseRPC(), project)
def to_response(self):
"""Execute controller flow and serialize to service response."""
self.ctx["datasets"] = []
if "project_id" in self.ctx:
self.ctx["datasets"] = self.local()
elif "git_url" in self.ctx:
self.ctx["datasets"] = self.remote()
return result_response(DatasetsListCtrl.RESPONSE_SERIALIZER, self.ctx)
def list_uploaded_files_view(user, cache):
"""List uploaded files ready to be added to projects."""
user = cache.ensure_user(user)
files = [f for f in cache.get_files(user) if f.exists()]
response = {
"files": sorted(files, key=lambda rec: (rec.is_dir, rec.relative_path))
}
return result_response(FileListResponseRPC(), response)
def read_manifest_from_template(user, cache):
"""Read templates from the manifest file of a template repository."""
project_data = ManifestTemplatesRequest().load({
**user,
**request.args,
},
unknown=EXCLUDE)
project = _project_clone(user, project_data)
manifest = read_template_manifest(project.abs_path)
return result_response(ManifestTemplatesResponseRPC(),
{"templates": manifest})
def list_jobs(user_data, cache):
"""List user created jobs."""
user = cache.ensure_user(user_data)
jobs = []
for job in cache.get_jobs(user):
if job.project_id:
job.project = cache.get_project(user, job.project_id)
jobs.append(job)
return result_response(JobListResponseRPC(), {"jobs": jobs})
def migration_check_project_view(user_data, cache):
"""Migrate specified project."""
user = cache.ensure_user(user_data)
project = cache.get_project(user, request.json['project_id'])
with chdir(project.abs_path):
migration_required, project_supported = migrations_check()
return result_response(
ProjectMigrationCheckResponseRPC(), {
'migration_required': migration_required,
'project_supported': project_supported
})
def list_dataset_files_view(user, cache):
"""List files in a dataset."""
ctx = DatasetFilesListRequest().load(request.args)
project = cache.get_project(cache.ensure_user(user), ctx['project_id'])
if not project.abs_path.exists():
return error_response(INVALID_PARAMS_ERROR_CODE,
'invalid project_id argument')
with chdir(project.abs_path):
ctx['files'] = list_files(datasets=[ctx['dataset_name']])
return result_response(DatasetFilesListResponseRPC(), ctx)
def test_result_response(svc_client):
"""Test result response utility."""
ctx = {"datasets": [{"name": "my-dataset"}]}
response = result_response(DatasetListResponseRPC(), ctx).json
assert response
assert {"result"} == set(response.keys())
assert {"datasets"} == set(response["result"].keys())
expected = ctx["datasets"][0]
received = response["result"]["datasets"][0]
assert expected["name"] == received["name"]
def add_file_to_dataset_view(user, cache):
"""Add the uploaded file to cloned repository."""
ctx = DatasetAddRequest().load(request.json)
user = cache.ensure_user(user)
project = cache.get_project(user, ctx['project_id'])
if not project.abs_path.exists():
return error_response(
INVALID_PARAMS_ERROR_CODE,
'invalid project_id: {0}'.format(ctx['project_id']))
if not ctx['commit_message']:
ctx['commit_message'] = 'service: dataset add {0}'.format(
ctx['dataset_name'])
local_paths = []
for _file in ctx['files']:
local_path = None
if 'file_id' in _file:
file = cache.get_file(user, _file['file_id'])
local_path = file.abs_path
elif 'file_path' in _file:
local_path = project.abs_path / Path(_file['file_path'])
if not local_path or not local_path.exists():
return error_response(
INVALID_PARAMS_ERROR_CODE,
'invalid file reference: {0}'.format(json.dumps(_file)))
ctx['commit_message'] += ' {0}'.format(local_path.name)
local_paths.append(str(local_path))
with chdir(project.abs_path):
add_file(local_paths,
ctx['dataset_name'],
create=ctx['create_dataset'],
commit_message=ctx['commit_message'])
if not repo_sync(project.abs_path):
return error_response(INTERNAL_FAILURE_ERROR_CODE,
'repo sync failed')
return result_response(DatasetAddResponseRPC(), ctx)
def migrate_project_view(user_data, cache):
"""Migrate specified project."""
user = cache.ensure_user(user_data)
project = cache.get_project(user, request.json['project_id'])
messages = []
def collect_message(msg):
"""Collect migration message."""
messages.append(msg)
with chdir(project.abs_path):
was_migrated = migrate_project(progress_callback=collect_message)
return result_response(ProjectMigrateResponseRPC(), {
'messages': messages,
'was_migrated': was_migrated
})
