def new_repo_push(repo_path,
source_url,
source_name="origin",
source_branch="master"):
"""Push a new repo to origin."""
repo = Repo(repo_path)
repo.create_remote(source_name, source_url)
_, branch = repo_sync(repo, remote=source_name)
return branch == source_branch
def remove_dataset_view(user, cache):
"""Remove a dataset from a project."""
ctx = DatasetRemoveRequest().load(request.json)
project = cache.get_project(cache.ensure_user(user), ctx["project_id"])
if not project.abs_path.exists():
return error_response(INVALID_PARAMS_ERROR_CODE,
"invalid project_id argument")
with chdir(project.abs_path):
dataset_remove([ctx["name"]], commit_message=ctx["commit_message"])
try:
_, ctx["remote_branch"] = repo_sync(Repo(project.abs_path),
remote="origin")
except GitCommandError:
return error_response(INTERNAL_FAILURE_ERROR_CODE, "repo sync failed")
return result_response(DatasetRemoveResponseRPC(), ctx)
def create_dataset_view(user, cache):
"""Create a new dataset in a project."""
ctx = DatasetCreateRequest().load(request.json)
project = cache.get_project(cache.ensure_user(user), ctx['project_id'])
with chdir(project.abs_path):
create_dataset(ctx['short_name'],
title=ctx.get('name'),
creators=ctx.get('creator'),
description=ctx.get('description'),
keywords=ctx.get('keywords'),
commit_message=ctx['commit_message'])
try:
_, ctx['remote_branch'] = repo_sync(Repo(project.abs_path),
remote='origin')
except GitCommandError:
return error_response(INTERNAL_FAILURE_ERROR_CODE,
'push to remote failed silently - try again')
return result_response(DatasetCreateResponseRPC(), ctx)
def create_dataset_view(user, cache):
"""Create a new dataset in a project."""
ctx = DatasetCreateRequest().load(request.json)
project = cache.get_project(cache.ensure_user(user), ctx["project_id"])
with chdir(project.abs_path):
create_dataset(
ctx["name"],
title=ctx.get("title"),
creators=ctx.get("creators"),
description=ctx.get("description"),
keywords=ctx.get("keywords"),
commit_message=ctx["commit_message"],
)
try:
_, ctx["remote_branch"] = repo_sync(Repo(project.abs_path),
remote="origin")
except GitCommandError:
return error_response(INTERNAL_FAILURE_ERROR_CODE, "repo sync failed")
return result_response(DatasetCreateResponseRPC(), ctx)
def dataset_import(
cache,
user,
user_job_id,
project_id,
dataset_uri,
short_name=None,
extract=False,
timeout=None,
):
"""Job for dataset import."""
user = cache.ensure_user(user)
user_job = cache.get_job(user, user_job_id)
project = cache.get_project(user, project_id)
with chdir(project.abs_path):
try:
user_job.in_progress()
import_dataset(
dataset_uri,
short_name,
extract,
commit_message=f'service: dataset import {dataset_uri}',
progress=DatasetImportJobProcess(cache, user_job))
_, remote_branch = repo_sync(Repo(project.abs_path),
remote='origin')
user_job.update_extras('remote_branch', remote_branch)
user_job.complete()
except (HTTPError, ParameterError, DatasetExistsError,
GitCommandError) as exp:
user_job.fail_job(str(exp))
# Reraise exception, so we see trace in job metadata.
raise exp
def dataset_add_remote_file(cache, user, user_job_id, project_id,
create_dataset, commit_message, short_name, url):
"""Add a remote file to a specified dataset."""
user = cache.ensure_user(user)
user_job = cache.get_job(user, user_job_id)
project = cache.get_project(user, project_id)
try:
user_job.in_progress()
with chdir(project.abs_path):
urls = url if isinstance(url, list) else [url]
add_file(urls,
short_name,
create=create_dataset,
commit_message=commit_message)
_, remote_branch = repo_sync(Repo(project.abs_path),
remote='origin')
user_job.update_extras('remote_branch', remote_branch)
user_job.complete()
except (HTTPError, BaseException, GitCommandError) as e:
user_job.fail_job(str(e))
def add_file_to_dataset_view(user_data, cache):
"""Add the uploaded file to cloned repository."""
ctx = DatasetAddRequest().load(request.json)
user = cache.ensure_user(user_data)
project = cache.get_project(user, ctx["project_id"])
if not ctx["commit_message"]:
ctx["commit_message"] = "service: dataset add {0}".format(ctx["name"])
local_paths = []
for _file in ctx["files"]:
local_path = None
if "file_url" in _file:
commit_message = "{0}{1}".format(ctx["commit_message"],
_file["file_url"])
job = cache.make_job(
user,
project=project,
job_data={
"renku_op": "dataset_add_remote_file",
"client_extras": ctx.get("client_extras")
},
)
_file["job_id"] = job.job_id
with enqueue_retry(DATASETS_JOB_QUEUE) as queue:
queue.enqueue(
dataset_add_remote_file,
user_data,
job.job_id,
project.project_id,
ctx["create_dataset"],
commit_message,
ctx["name"],
_file["file_url"],
)
continue
if "file_id" in _file:
file = cache.get_file(user, _file["file_id"])
local_path = file.abs_path
elif "file_path" in _file:
local_path = project.abs_path / Path(_file["file_path"])
if not local_path or not local_path.exists():
return error_response(
INVALID_PARAMS_ERROR_CODE,
"invalid file reference: {0}".format(json.dumps(_file)))
ctx["commit_message"] += " {0}".format(local_path.name)
local_paths.append(str(local_path))
if local_paths:
with chdir(project.abs_path):
add_file(
local_paths,
ctx["name"],
create=ctx["create_dataset"],
force=ctx["force"],
commit_message=ctx["commit_message"],
)
try:
_, ctx["remote_branch"] = repo_sync(Repo(project.abs_path),
remote="origin")
except GitCommandError:
return error_response(INTERNAL_FAILURE_ERROR_CODE,
"repo sync failed")
return result_response(DatasetAddResponseRPC(), ctx)