def dataset_import( cache, user, user_job_id, project_id, dataset_uri, name=None, extract=False, timeout=None, ): """Job for dataset import.""" user = cache.ensure_user(user) worker_log.debug(f"executing dataset import job for {user.user_id}:{user.fullname}") user_job = cache.get_job(user, user_job_id) user_job.in_progress() try: worker_log.debug(f"retrieving metadata for project {project_id}") project = cache.get_project(user, project_id) with chdir(project.abs_path): worker_log.debug(f"project found in cache - importing dataset {dataset_uri}") import_dataset( dataset_uri, name, extract, commit_message=f"service: dataset import {dataset_uri}", progress=DatasetImportJobProcess(cache, user_job), ) worker_log.debug("operation successful - syncing with remote") _, remote_branch = repo_sync(Repo(project.abs_path), remote="origin") user_job.update_extras("remote_branch", remote_branch) user_job.complete() worker_log.debug("job completed") except (HTTPError, ParameterError, RenkuException, GitCommandError) as exp: user_job.fail_job(str(exp)) # Reraise exception, so we see trace in job metadata # and in metrics as failed job. raise exp
def import_(uri, name, extract, yes): """Import data from a 3rd party provider or another renku project. Supported providers: [Dataverse, Renku, Zenodo] """ import_dataset(uri=uri, name=name, extract=extract, with_prompt=True, yes=yes, progress=_DownloadProgressbar) click.secho(" " * 79 + "\r", nl=False) click.secho("OK", fg="green")
def import_(uri, short_name, extract): """Import data from a 3rd party provider. Supported providers: [Zenodo, Dataverse] """ import_dataset( uri=uri, short_name=short_name, extract=extract, with_prompt=True, progress=_DownloadProgressbar ) click.secho('OK', fg='green')
def import_(uri, short_name, extract, yes): """Import data from a 3rd party provider or another renku project. Supported providers: [Dataverse, Renku, Zenodo] """ import_dataset( uri=uri, short_name=short_name, extract=extract, with_prompt=True, yes=yes, progress=_DownloadProgressbar ) click.secho(' ' * 79 + '\r', nl=False) click.secho('OK', fg='green')
def dataset_import( cache, user, user_job_id, project_id, dataset_uri, short_name=None, extract=False, timeout=None, ): """Job for dataset import.""" user = cache.ensure_user(user) user_job = cache.get_job(user, user_job_id) project = cache.get_project(user, project_id) with chdir(project.abs_path): try: user_job.in_progress() import_dataset( dataset_uri, short_name, extract, commit_message=f'service: dataset import {dataset_uri}', progress=DatasetImportJobProcess(cache, user_job)) _, remote_branch = repo_sync(Repo(project.abs_path), remote='origin') user_job.update_extras('remote_branch', remote_branch) user_job.complete() except (HTTPError, ParameterError, DatasetExistsError, GitCommandError) as exp: user_job.fail_job(str(exp)) # Reraise exception, so we see trace in job metadata. raise exp
def dataset_import( cache, user, user_job_id, project_id, dataset_uri, short_name=None, extract=False, timeout=None, ): """Job for dataset import.""" user = cache.ensure_user(user) user_job = cache.get_job(user, user_job_id) project = cache.get_project(user, project_id) with chdir(project.abs_path): try: user_job.in_progress() import_dataset( dataset_uri, short_name, extract, commit_message=f'service: dataset import {dataset_uri}', progress=DatasetImportJobProcess(cache, user_job)) user_job.complete() except (HTTPError, ParameterError, DatasetExistsError) as exp: user_job.fail_job(str(exp)) # Reraise exception, so we see trace in job metadata. raise exp if not repo_sync(project.abs_path): error = 'failed to push refs' user_job.fail_job(error) raise RuntimeError(error)