def test_load_with_existing_dataset():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
# post dataset creation request
dataset_dict_with_id = create_dataset(dataset_dict=dataset_dict,
resources=[dpath],
api=api)
task_path = common.make_upload_task(dataset_dict=dataset_dict_with_id,
resource_paths=[str(dpath)],
resource_names=[dpath.name])
uj = task.load_task(task_path, api=api)
assert uj.dataset_id == dataset_dict_with_id["id"]
# skipping the upload should work, since it's already uploaded
uj.set_state("online")
uj.task_verify_resources()
for ii in range(600):
uj.task_verify_resources()
if uj.state != "done":
time.sleep(.1)
continue
else:
break
else:
raise AssertionError("State not 'done' - No verification within 60s!")
def test_initialize():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath])
assert uj.state == "init"
def test_dataset_create_same_resource():
"""There should be an error when a resource is added twice"""
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="create-with-same-resource")
# post dataset creation request
data = dataset.create_dataset(dataset_dict=dataset_dict, api=api)
dataset.add_resource(dataset_id=data["id"], path=dpath, api=api)
with pytest.raises(APIConflictError):
# Should not be able to upload same resource twice
dataset.add_resource(dataset_id=data["id"], path=dpath, api=api)
def test_save_load():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath],
task_id="hanspeter")
td = pathlib.Path(tempfile.mkdtemp(prefix="task_"))
task_path = td / "test.dcoraid-task"
task.save_task(uj, path=task_path)
uj2 = task.load_task(task_path, api=api)
assert uj.dataset_id == uj2.dataset_id
assert uj.paths[0].samefile(uj2.paths[0])
def test_load_with_existing_dataset_map_from_task_control():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
# post dataset creation request
dataset_dict_with_id = create_dataset(dataset_dict=dataset_dict,
resources=[dpath],
api=api)
task_path = common.make_upload_task(dataset_dict=dataset_dict,
resource_paths=[str(dpath)],
resource_names=[dpath.name],
task_id="xwing")
uj = task.load_task(
task_path,
api=api,
map_task_to_dataset_id={"deathstar": dataset_dict_with_id["id"]})
assert uj.dataset_id != dataset_dict_with_id["id"]
def test_dataset_creation_wrong_resource_supplement():
"""Pass an invalid resource supplement and see if it fails"""
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="basic_test")
# post dataset creation request
data = dataset.create_dataset(
dataset_dict=dataset_dict,
api=api,
)
# simple test
with pytest.raises(APIConflictError):
dataset.add_resource(
dataset_id=data["id"],
resource_dict={"sp:chip:production date": "2020-15-31"},
path=dpath,
api=api)
def test_dataset_creation():
"""Just test whether we can create (and remove) a draft dataset"""
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="basic_test")
# post dataset creation request
data = dataset.create_dataset(
dataset_dict=dataset_dict,
api=api,
)
# simple test
assert "authors" in data
assert data["authors"] == common.USER_NAME
assert data["state"] == "draft"
# remove draft dataset
dataset.remove_draft(
dataset_id=data["id"],
api=api,
)
with pytest.raises(APINotFoundError):
# make sure it is gone
api.get("package_show", id=data["id"])
def test_saveload():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath],
task_id="hanspeter")
state = uj.__getstate__()
assert state["dataset_id"] == dataset_dict["id"]
assert dpath.samefile(state["resource_paths"][0])
assert dpath.name == state["resource_names"][0]
# now create a new job from the state
uj2 = job.UploadJob.from_upload_job_state(state, api=api)
state2 = uj2.__getstate__()
assert state2["dataset_id"] == dataset_dict["id"]
assert dpath.samefile(state2["resource_paths"][0])
assert dpath.name == state2["resource_names"][0]
assert state2["task_id"] == "hanspeter"
def test_full_upload():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath])
assert uj.state == "init"
uj.task_compress_resources()
assert uj.state == "parcel"
uj.task_upload_resources()
assert uj.state == "online"
for ii in range(30):
uj.task_verify_resources()
if uj.state != "done":
time.sleep(.1)
continue
else:
break
else:
raise AssertionError("State not 'done' - No verification within 3s!")
def test_dataset_id_already_exists_active_fails():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
# post dataset creation request
dataset_dict_with_id = create_dataset(dataset_dict=dataset_dict,
resources=[dpath],
api=api,
activate=True)
# create a new task with the same dataset ID but with different data
task_path = common.make_upload_task(
dataset_dict=dataset_dict_with_id,
resource_paths=[str(dpath), str(dpath)],
resource_names=["1.rtdc", "2.rtdc"])
uj = task.load_task(task_path, api=api)
assert len(uj.paths) == 2
assert len(uj.resource_names) == 2
assert uj.dataset_id == dataset_dict_with_id["id"]
# attempt to upload the task
uj.task_compress_resources()
assert uj.state == "parcel"
uj.task_upload_resources()
assert uj.state == "error"
assert "Access denied" in str(uj.traceback)