def create_dataset_view(user, cache):
"""Create a new dataset in a project."""
ctx = DatasetCreateRequest().load(request.json)
project = cache.get_project(user, ctx['project_id'])
project_path = make_project_path(user, project)
if not project_path:
return jsonify(
error={
'code': INVALID_PARAMS_ERROR_CODE,
'message': 'invalid project_id argument',
})
with chdir(project_path):
create_dataset(
ctx['dataset_name'],
commit_message=ctx['commit_message'],
creators=ctx.get('creators'),
description=ctx.get('description'),
)
if not repo_sync(project_path):
return jsonify(
error={
'code': INTERNAL_FAILURE_ERROR_CODE,
'reason': 'push to remote failed silently - try again'
})
return jsonify(DatasetCreateResponseRPC().load(
{'result': DatasetCreateResponse().load(ctx, unknown=EXCLUDE)}))
def add_file_to_dataset_view(user, cache):
"""Add the uploaded file to cloned repository."""
ctx = DatasetAddRequest().load(request.json)
project = cache.get_project(user, ctx['project_id'])
project_path = make_project_path(user, project)
if not project_path:
return jsonify(
error={
'code': INVALID_PARAMS_ERROR_CODE,
'message': 'invalid project_id: {0}'.format(ctx['project_id']),
})
if not ctx['commit_message']:
ctx['commit_message'] = 'service: dataset add {0}'.format(
ctx['dataset_name'])
local_paths = []
for _file in ctx['files']:
local_path = None
if 'file_id' in _file:
file = cache.get_file(user, _file['file_id'])
local_path = make_file_path(user, file)
elif 'file_path' in _file:
local_path = project_path / Path(_file['file_path'])
if not local_path or not local_path.exists():
return jsonify(
error={
'code':
INVALID_PARAMS_ERROR_CODE,
'message':
'invalid file reference: {0}'.format(
local_path.relative_to(project_path))
})
ctx['commit_message'] += ' {0}'.format(local_path.name)
local_paths.append(str(local_path))
with chdir(project_path):
add_file(local_paths,
ctx['dataset_name'],
create=ctx['create_dataset'],
commit_message=ctx['commit_message'])
if not repo_sync(project_path):
return jsonify(error={
'code': INTERNAL_FAILURE_ERROR_CODE,
'message': 'repo sync failed'
})
return jsonify(DatasetAddResponseRPC().load(
{'result': DatasetAddResponse().load(ctx, unknown=EXCLUDE)}))
def add_file_to_dataset_view(user, cache):
"""Add the uploaded file to cloned repository."""
ctx = DatasetAddRequest().load(request.json)
user = cache.ensure_user(user)
project = cache.get_project(user, ctx['project_id'])
if not project.abs_path.exists():
return error_response(
INVALID_PARAMS_ERROR_CODE,
'invalid project_id: {0}'.format(ctx['project_id']))
if not ctx['commit_message']:
ctx['commit_message'] = 'service: dataset add {0}'.format(
ctx['dataset_name'])
local_paths = []
for _file in ctx['files']:
local_path = None
if 'file_id' in _file:
file = cache.get_file(user, _file['file_id'])
local_path = file.abs_path
elif 'file_path' in _file:
local_path = project.abs_path / Path(_file['file_path'])
if not local_path or not local_path.exists():
return error_response(
INVALID_PARAMS_ERROR_CODE,
'invalid file reference: {0}'.format(json.dumps(_file)))
ctx['commit_message'] += ' {0}'.format(local_path.name)
local_paths.append(str(local_path))
with chdir(project.abs_path):
add_file(local_paths,
ctx['dataset_name'],
create=ctx['create_dataset'],
commit_message=ctx['commit_message'])
if not repo_sync(project.abs_path):
return error_response(INTERNAL_FAILURE_ERROR_CODE,
'repo sync failed')
return result_response(DatasetAddResponseRPC(), ctx)
def create_dataset_view(user, cache):
"""Create a new dataset in a project."""
ctx = DatasetCreateRequest().load(request.json)
project = cache.get_project(cache.ensure_user(user), ctx['project_id'])
if not project.abs_path.exists():
return error_response(INVALID_PARAMS_ERROR_CODE,
'invalid project_id argument')
with chdir(project.abs_path):
create_dataset(
ctx['dataset_name'],
commit_message=ctx['commit_message'],
creators=ctx.get('creators'),
description=ctx.get('description'),
)
if not repo_sync(project.abs_path):
return error_response(INTERNAL_FAILURE_ERROR_CODE,
'push to remote failed silently - try again')
return result_response(DatasetCreateResponseRPC(), ctx)
def dataset_import(
cache,
user,
user_job_id,
project_id,
dataset_uri,
short_name=None,
extract=False,
timeout=None,
):
"""Job for dataset import."""
user = cache.ensure_user(user)
user_job = cache.get_job(user, user_job_id)
project = cache.get_project(user, project_id)
with chdir(project.abs_path):
try:
user_job.in_progress()
import_dataset(
dataset_uri,
short_name,
extract,
commit_message=f'service: dataset import {dataset_uri}',
progress=DatasetImportJobProcess(cache, user_job))
user_job.complete()
except (HTTPError, ParameterError, DatasetExistsError) as exp:
user_job.fail_job(str(exp))
# Reraise exception, so we see trace in job metadata.
raise exp
if not repo_sync(project.abs_path):
error = 'failed to push refs'
user_job.fail_job(error)
raise RuntimeError(error)