def test_queue_remove_job():
"""Remove a job from the queue and from the persistent list"""
api = common.get_api()
td = pathlib.Path(tempfile.mkdtemp(prefix="persistent_uj_list_"))
pujl_path = td / "joblistdir"
# create some metadata
dataset_dict = common.make_dataset_dict(hint="create-with-resource")
# post dataset creation request
data = create_dataset(dataset_dict=dataset_dict, api=api)
joblist = UploadQueue(api=api,
path_persistent_job_list=pujl_path)
# disable all daemons, so no uploading happens
joblist.daemon_compress.join()
joblist.daemon_upload.join()
joblist.daemon_verify.join()
uj = joblist.new_job(dataset_id=data["id"],
paths=[dpath])
assert uj.state == "init"
joblist.remove_job(uj.dataset_id)
assert uj not in joblist
assert not joblist.jobs_eternal.job_exists(uj.dataset_id)
# adding it again should work
uj2 = joblist.new_job(dataset_id=data["id"],
paths=[dpath])
assert uj2 in joblist
assert uj2.__getstate__() == uj.__getstate__()
def test_load_with_existing_dataset():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
# post dataset creation request
dataset_dict_with_id = create_dataset(dataset_dict=dataset_dict,
resources=[dpath],
api=api)
task_path = common.make_upload_task(dataset_dict=dataset_dict_with_id,
resource_paths=[str(dpath)],
resource_names=[dpath.name])
uj = task.load_task(task_path, api=api)
assert uj.dataset_id == dataset_dict_with_id["id"]
# skipping the upload should work, since it's already uploaded
uj.set_state("online")
uj.task_verify_resources()
for ii in range(600):
uj.task_verify_resources()
if uj.state != "done":
time.sleep(.1)
continue
else:
break
else:
raise AssertionError("State not 'done' - No verification within 60s!")
def test_initialize():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath])
assert uj.state == "init"
def test_missing_owner_org():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
dataset_dict.pop("owner_org")
task_path = common.make_upload_task(dataset_dict=dataset_dict)
assert not task.task_has_circle(task_path)
with pytest.raises(dcoraid.api.APIConflictError,
match="A circle must be provided"):
task.load_task(task_path, api=api)
def test_no_ids():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
task_path = common.make_upload_task(dataset_dict=dataset_dict,
resource_paths=[str(dpath)],
resource_names=[dpath.name],
task_id=None)
with pytest.raises(ValueError,
match="or pass the dataset_id via the dataset_kwargs"):
task.load_task(task_path, api=api)
def test_dataset_create_same_resource():
"""There should be an error when a resource is added twice"""
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="create-with-same-resource")
# post dataset creation request
data = dataset.create_dataset(dataset_dict=dataset_dict, api=api)
dataset.add_resource(dataset_id=data["id"], path=dpath, api=api)
with pytest.raises(APIConflictError):
# Should not be able to upload same resource twice
dataset.add_resource(dataset_id=data["id"], path=dpath, api=api)
def test_custom_dataset_dict_2():
api = common.get_api()
# post dataset creation request
task_path = common.make_upload_task(dataset_dict=True,
resource_paths=[str(dpath)],
resource_names=[dpath.name])
dataset_dict = common.make_dataset_dict()
dataset_dict["authors"] = "Captain Hook!"
uj = task.load_task(task_path, api=api, dataset_kwargs=dataset_dict)
# now make sure the authors were set correctly
ddict = api.get("package_show", id=uj.dataset_id)
assert ddict["authors"] == "Captain Hook!"
def test_wrong_ids():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
dataset_dict["id"] = "peter"
task_path = common.make_upload_task(
dataset_dict=dataset_dict,
dataset_id="hans", # different id
resource_paths=[str(dpath)],
resource_names=[dpath.name])
with pytest.raises(ValueError,
match="I got the following IDs: from upload job " +
"state: hans; from dataset dict: peter"):
task.load_task(task_path, api=api)
def test_load_with_existing_dataset_map_from_task_dict_update():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
# post dataset creation request
task_path = common.make_upload_task(dataset_dict=dataset_dict,
resource_paths=[str(dpath)],
resource_names=[dpath.name],
task_id="xwing")
map_task_to_dataset_id = {}
uj = task.load_task(task_path,
api=api,
map_task_to_dataset_id=map_task_to_dataset_id)
assert uj.task_id == "xwing"
assert map_task_to_dataset_id["xwing"] == uj.dataset_id
def test_queue_create_dataset_with_resource():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="create-with-resource")
# post dataset creation request
data = create_dataset(dataset_dict=dataset_dict, api=api)
joblist = UploadQueue(api=api)
joblist.new_job(dataset_id=data["id"],
paths=[dpath])
for _ in range(600): # 60 seconds to upload
if joblist[0].state == "done":
break
time.sleep(.1)
else:
assert False, "Job not finished: {}".format(joblist[0].get_status())
def test_save_load():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath],
task_id="hanspeter")
td = pathlib.Path(tempfile.mkdtemp(prefix="task_"))
task_path = td / "test.dcoraid-task"
task.save_task(uj, path=task_path)
uj2 = task.load_task(task_path, api=api)
assert uj.dataset_id == uj2.dataset_id
assert uj.paths[0].samefile(uj2.paths[0])
def test_load_with_existing_dataset_map_from_task_control():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
# post dataset creation request
dataset_dict_with_id = create_dataset(dataset_dict=dataset_dict,
resources=[dpath],
api=api)
task_path = common.make_upload_task(dataset_dict=dataset_dict,
resource_paths=[str(dpath)],
resource_names=[dpath.name],
task_id="xwing")
uj = task.load_task(
task_path,
api=api,
map_task_to_dataset_id={"deathstar": dataset_dict_with_id["id"]})
assert uj.dataset_id != dataset_dict_with_id["id"]
def test_dataset_creation_wrong_resource_supplement():
"""Pass an invalid resource supplement and see if it fails"""
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="basic_test")
# post dataset creation request
data = dataset.create_dataset(
dataset_dict=dataset_dict,
api=api,
)
# simple test
with pytest.raises(APIConflictError):
dataset.add_resource(
dataset_id=data["id"],
resource_dict={"sp:chip:production date": "2020-15-31"},
path=dpath,
api=api)
def test_queue_basic_functionalities():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="create-with-resource")
# post dataset creation request
data = create_dataset(dataset_dict=dataset_dict, api=api)
joblist = UploadQueue(api=api)
uj = joblist.new_job(dataset_id=data["id"],
paths=[dpath])
assert uj.dataset_id == data["id"]
assert uj in joblist
samejob = joblist.get_job(data["id"])
assert samejob is uj
with pytest.raises(KeyError, match="Job 'peter' not found!"):
joblist.get_job("peter")
samejob2 = joblist[0]
assert samejob2 is uj
def test_upload_task_missing_circle(qtbot, monkeypatch):
"""When the organization is missing, DCOR-Aid should ask for it"""
task_id = str(uuid.uuid4())
dataset_dict = common.make_dataset_dict(hint="task_upload_no_org_")
dataset_dict.pop("owner_org")
tpath = common.make_upload_task(task_id=task_id,
dataset_dict=dataset_dict)
mw = DCORAid()
QtWidgets.QApplication.processEvents(QtCore.QEventLoop.AllEvents, 300)
monkeypatch.setattr(QtWidgets.QFileDialog, "getOpenFileNames",
lambda *args: ([tpath], None))
# We actually only need this monkeypatch if there is more than
# one circle for the present user.
monkeypatch.setattr(QtWidgets.QInputDialog, "getItem",
# return the first item in the circle list
lambda *args: (args[3][0], True))
act = QtWidgets.QAction("some unimportant text")
act.setData("single")
mw.panel_upload.on_upload_task(action=act)
uj = mw.panel_upload.jobs[-1]
assert uj.task_id == task_id
mw.close()
def test_saveload():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath],
task_id="hanspeter")
state = uj.__getstate__()
assert state["dataset_id"] == dataset_dict["id"]
assert dpath.samefile(state["resource_paths"][0])
assert dpath.name == state["resource_names"][0]
# now create a new job from the state
uj2 = job.UploadJob.from_upload_job_state(state, api=api)
state2 = uj2.__getstate__()
assert state2["dataset_id"] == dataset_dict["id"]
assert dpath.samefile(state2["resource_paths"][0])
assert dpath.name == state2["resource_names"][0]
assert state2["task_id"] == "hanspeter"
def test_dataset_creation():
"""Just test whether we can create (and remove) a draft dataset"""
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="basic_test")
# post dataset creation request
data = dataset.create_dataset(
dataset_dict=dataset_dict,
api=api,
)
# simple test
assert "authors" in data
assert data["authors"] == common.USER_NAME
assert data["state"] == "draft"
# remove draft dataset
dataset.remove_draft(
dataset_id=data["id"],
api=api,
)
with pytest.raises(APINotFoundError):
# make sure it is gone
api.get("package_show", id=data["id"])
def test_full_upload():
api = common.get_api()
# create some metadata
bare_dict = common.make_dataset_dict(hint="create-with-resource")
# create dataset (to get the "id")
dataset_dict = create_dataset(dataset_dict=bare_dict, api=api)
uj = job.UploadJob(api=api,
dataset_id=dataset_dict["id"],
resource_paths=[dpath])
assert uj.state == "init"
uj.task_compress_resources()
assert uj.state == "parcel"
uj.task_upload_resources()
assert uj.state == "online"
for ii in range(30):
uj.task_verify_resources()
if uj.state != "done":
time.sleep(.1)
continue
else:
break
else:
raise AssertionError("State not 'done' - No verification within 3s!")
def test_dataset_id_already_exists_active_fails():
api = common.get_api()
# create some metadata
dataset_dict = common.make_dataset_dict(hint="task_test")
# post dataset creation request
dataset_dict_with_id = create_dataset(dataset_dict=dataset_dict,
resources=[dpath],
api=api,
activate=True)
# create a new task with the same dataset ID but with different data
task_path = common.make_upload_task(
dataset_dict=dataset_dict_with_id,
resource_paths=[str(dpath), str(dpath)],
resource_names=["1.rtdc", "2.rtdc"])
uj = task.load_task(task_path, api=api)
assert len(uj.paths) == 2
assert len(uj.resource_names) == 2
assert uj.dataset_id == dataset_dict_with_id["id"]
# attempt to upload the task
uj.task_compress_resources()
assert uj.state == "parcel"
uj.task_upload_resources()
assert uj.state == "error"
assert "Access denied" in str(uj.traceback)
