class LibrariesApiTestCase(ApiTestCase):
def setUp(self):
super().setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(
self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
def test_create(self):
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries",
data=data,
admin=True,
json=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "name")
assert library["name"] == "CreateTestLibrary"
def test_delete(self):
library = self.library_populator.new_library("DeleteTestLibrary")
create_response = self._delete(f"libraries/{library['id']}",
admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "deleted")
assert library["deleted"] is True
# Test undeleting
data = dict(undelete=True)
create_response = self._delete(f"libraries/{library['id']}",
data=data,
admin=True,
json=True)
library = create_response.json()
self._assert_status_code_is(create_response, 200)
assert library["deleted"] is False
def test_nonadmin(self):
# Anons can't create libs
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries",
data=data,
admin=False,
anon=True,
json=True)
self._assert_status_code_is(create_response, 403)
# Anons can't delete libs
library = self.library_populator.new_library("AnonDeleteTestLibrary")
create_response = self._delete(f"libraries/{library['id']}",
admin=False,
anon=True)
self._assert_status_code_is(create_response, 403)
# Anons can't update libs
data = dict(name="ChangedName",
description="ChangedDescription",
synopsis='ChangedSynopsis')
create_response = self._patch(f"libraries/{library['id']}",
data=data,
admin=False,
anon=True)
self._assert_status_code_is(create_response, 403)
def test_update(self):
library = self.library_populator.new_library("UpdateTestLibrary")
data = dict(name='ChangedName',
description='ChangedDescription',
synopsis='ChangedSynopsis')
create_response = self._patch(f"libraries/{library['id']}",
data=data,
admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, 'name', 'description', 'synopsis')
assert library['name'] == 'ChangedName'
assert library['description'] == 'ChangedDescription'
assert library['synopsis'] == 'ChangedSynopsis'
def test_create_private_library_legacy_permissions(self):
library = self.library_populator.new_library(
"LegacyPermissionTestLibrary")
library_id = library["id"]
role_id = self.library_populator.user_private_role_id()
self.library_populator.set_permissions(library_id, role_id)
create_response = self._create_folder(library)
self._assert_status_code_is(create_response, 200)
with self._different_user():
create_response = self._create_folder(library)
self._assert_status_code_is(create_response, 403)
def test_create_private_library_permissions(self):
library = self.library_populator.new_library("PermissionTestLibrary")
library_id = library["id"]
role_id = self.library_populator.user_private_role_id()
self.library_populator.set_permissions_with_action(
library_id, role_id, action="set_permissions")
create_response = self._create_folder(library)
self._assert_status_code_is(create_response, 200)
with self._different_user():
create_response = self._create_folder(library)
self._assert_status_code_is(create_response, 403)
def test_get_library_current_permissions(self):
library = self.library_populator.new_library(
"GetCurrentPermissionTestLibrary")
library_id = library["id"]
current = self.library_populator.get_permissions(library_id,
scope="current")
self._assert_has_keys(current, "access_library_role_list",
"modify_library_role_list",
"manage_library_role_list",
"add_library_item_role_list")
role_id = self.library_populator.user_private_role_id()
self.library_populator.set_permissions_with_action(
library_id, role_id, action="set_permissions")
current = self.library_populator.get_permissions(library_id,
scope="current")
assert role_id in current["access_library_role_list"][0]
assert role_id in current["modify_library_role_list"][0]
assert role_id in current["manage_library_role_list"][0]
assert role_id in current["add_library_item_role_list"][0]
def test_get_library_available_permissions(self):
library = self.library_populator.new_library(
"GetAvailablePermissionTestLibrary")
library_id = library["id"]
role_id = self.library_populator.user_private_role_id()
# As we can manage this library our role will be available
available = self.library_populator.get_permissions(library_id,
scope="available")
available_role_ids = [role["id"] for role in available["roles"]]
assert role_id in available_role_ids
def test_get_library_available_permissions_with_query(self):
library = self.library_populator.new_library(
"GetAvailablePermissionWithQueryTestLibrary")
library_id = library["id"]
with self._different_user():
email = self.library_populator.user_email()
# test at least 2 user roles should be available now
available = self.library_populator.get_permissions(library_id,
scope="available")
available_role_ids = [role["id"] for role in available["roles"]]
assert len(available_role_ids) > 1
# test query for specific role/email
available = self.library_populator.get_permissions(library_id,
scope="available",
q=email)
available_role_emails = [role["name"] for role in available["roles"]]
assert available["total"] == 1
assert available_role_emails[0] == email
def test_create_library_dataset_bootstrap_user(
self, library_name="private_dataset", wait=True):
library = self.library_populator.new_private_library(library_name)
payload, files = self.library_populator.create_dataset_request(
library, file_type="txt", contents="create_test")
create_response = self.galaxy_interactor.post("libraries/%s/contents" %
library["id"],
payload,
files=files,
key=self.master_api_key)
self._assert_status_code_is(create_response, 400)
def test_create_dataset_denied(self):
url, payload = self._create_dataset_kwargs()
with self._different_user():
create_response = self._post(url, payload)
self._assert_status_code_is(create_response, 403)
def test_create_dataset_bootstrap_admin_user(self):
url, payload = self._create_dataset_kwargs()
with self._different_user():
create_response = self._post(url, payload, key=self.master_api_key)
self._assert_status_code_is(create_response, 400)
def _create_dataset_kwargs(self):
library = self.library_populator.new_private_library(
"ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id,
content="1 2 3")['id']
return f"folders/{folder_id}/contents", {"from_hda_id": hda_id}
def test_show_private_dataset_permissions(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library(
"ForCreateDatasets", wait=True)
with self._different_user():
with pytest.raises(requests.exceptions.HTTPError) as exc_info:
self.library_populator.show_ld(library["id"],
library_dataset["id"])
# TODO: this should really be 403 and a proper JSON exception.
self._assert_status_code_is(exc_info.value.response, 400)
def test_create_dataset(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library(
"ForCreateDatasets", wait=True)
self._assert_has_keys(library_dataset, "peek", "data_type")
assert library_dataset["peek"].find("create_test") >= 0
assert library_dataset["file_ext"] == "txt", library_dataset[
"file_ext"]
# Get library dataset (same as library_dataset)
library_dataset_from_show = self.library_populator.show_ld(
library["id"], library_dataset["id"])
assert library_dataset_from_show["id"] == library_dataset["id"]
assert library_dataset_from_show["parent_library_id"] == library["id"]
assert library_dataset_from_show["folder_id"] == library[
"root_folder_id"]
# Get library dataset dataset association
ldda = self.library_populator.show_ldda(
library_dataset_from_show["ldda_id"])
assert ldda["library_dataset_id"] == library_dataset["id"]
assert ldda["parent_library_id"] == library["id"]
assert ldda["file_ext"] == "txt"
def test_fetch_upload_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder(
"flat_zip")
items = [{
"src": "files",
"dbkey": "hg19",
"info": "my cool bed",
"created_from_basename": "4.bed"
}]
targets = [{"destination": destination, "items": items}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {
"files_0|file_data":
open(self.test_data_resolver.get_filename("4.bed"))
},
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(
library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
assert dataset["genome_build"] == "hg19", dataset
assert dataset["misc_info"] == "my cool bed", dataset
assert dataset["file_ext"] == "bed", dataset
assert dataset["created_from_basename"] == "4.bed"
def test_fetch_zip_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder(
"flat_zip")
bed_test_data_path = self.test_data_resolver.get_filename("4.bed.zip")
targets = [{
"destination": destination,
"items_from": "archive",
"src": "files",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {
"files_0|file_data": open(bed_test_data_path, 'rb')
}
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(
library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
def test_fetch_single_url_to_folder(self):
library, response = self._fetch_single_url_to_folder()
dataset = self.library_populator.get_library_contents_with_path(
library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
def test_fetch_single_url_with_invalid_datatype(self):
_, response = self._fetch_single_url_to_folder('xxx', assert_ok=False)
self._assert_status_code_is(response, 400)
assert response.json(
)['err_msg'] == "Requested extension 'xxx' unknown, cannot upload dataset."
def _fetch_single_url_to_folder(self, file_type='auto', assert_ok=True):
history_id, library, destination = self._setup_fetch_to_folder(
"single_url")
items = [{
"src": "url",
"url": FILE_URL,
"MD5": FILE_MD5,
"ext": file_type,
}]
targets = [{
"destination": destination,
"items": items,
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"validate_hashes": True
}
return library, self.dataset_populator.fetch(payload,
assert_ok=assert_ok)
def test_legacy_upload_unknown_datatype(self):
library = self.library_populator.new_private_library("ForLegacyUpload")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
payload = {
'folder_id': folder_id,
'create_type': 'file',
'file_type': 'xxx',
'upload_option': 'upload_file',
'files_0|url_paste': FILE_URL,
}
create_response = self._post(f"libraries/{library['id']}/contents",
payload)
self._assert_status_code_is(create_response, 400)
assert create_response.json(
) == "Requested extension 'xxx' unknown, cannot upload dataset."
def test_fetch_failed_validation(self):
# Exception handling is really rough here - we should be creating a dataset in error instead
# of just failing the job like this.
history_id, library, destination = self._setup_fetch_to_folder(
"single_url")
items = [{
"src": "url",
"url":
"https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed",
"MD5": "37b59762b59fff860460522d271bc112",
"name": "4.bed",
}]
targets = [{
"destination": destination,
"items": items,
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"validate_hashes": True
}
tool_response = self.dataset_populator.fetch(payload, assert_ok=False)
job = self.dataset_populator.check_run(tool_response)
self.dataset_populator.wait_for_job(job["id"])
job = tool_response.json()["jobs"][0]
details = self.dataset_populator.get_job_details(job["id"]).json()
assert details["state"] == "ok", details
dataset = self.library_populator.get_library_contents_with_path(
library["id"], "/4.bed")
assert dataset["state"] == "error", dataset
def test_fetch_url_archive_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder(
"single_url")
targets = [{
"destination":
destination,
"items_from":
"archive",
"src":
"url",
"url":
"https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(
library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
@unittest.skip("reference URLs changed, checksums now invalid.")
def test_fetch_bagit_archive_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder(
"bagit_archive")
example_bag_path = self.test_data_resolver.get_filename(
"example-bag.zip")
targets = [{
"destination": destination,
"items_from": "bagit_archive",
"src": "files",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {
"files_0|file_data": open(example_bag_path)
},
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(
library["id"], "/README.txt")
assert dataset["file_size"] == 66, dataset
dataset = self.library_populator.get_library_contents_with_path(
library["id"], "/bdbag-profile.json")
assert dataset["file_size"] == 723, dataset
def _setup_fetch_to_folder(self, test_name):
return self.library_populator.setup_fetch_to_folder(test_name)
def test_create_dataset_in_folder(self):
library = self.library_populator.new_private_library(
"ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id,
content="1 2 3")['id']
payload = {'from_hda_id': hda_id}
create_response = self._post(f"folders/{folder_id}/contents", payload)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "id")
def test_create_dataset_in_subfolder(self):
library = self.library_populator.new_private_library(
"ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
subfolder_response = self._create_subfolder(folder_id)
self._assert_status_code_is(folder_response, 200)
print(subfolder_response.json())
subfolder_id = subfolder_response.json()['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id,
content="1 2 3 sub")['id']
payload = {'from_hda_id': hda_id}
create_response = self._post(f"folders/{subfolder_id}/contents",
payload)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "id")
dataset_update_time = create_response.json()['update_time']
container_fetch_response = self.galaxy_interactor.get(
f"folders/{folder_id}/contents")
container_update_time = container_fetch_response.json(
)['folder_contents'][0]['update_time']
assert dataset_update_time == container_update_time, container_fetch_response
def _patch_library_dataset(self, library_dataset_id, data):
create_response = self._patch(
f"libraries/datasets/{library_dataset_id}", data=data)
self._assert_status_code_is(create_response, 200)
ld = create_response.json()
library_id = ld['parent_library_id']
return self.library_populator.wait_on_library_dataset(
library_id, ld["id"])
def test_update_dataset_in_folder(self):
ld = self._create_dataset_in_folder_in_library("ForUpdateDataset",
content=">seq\nATGC",
wait=True).json()
data = {
'name': 'updated_name',
'file_ext': 'fasta',
'misc_info': 'updated_info',
'genome_build': 'updated_genome_build'
}
ld_updated = self._patch_library_dataset(ld['id'], data)
self._assert_has_keys(ld_updated, "name", "file_ext", "misc_info",
"genome_build")
def test_update_dataset_tags(self):
ld = self._create_dataset_in_folder_in_library("ForTagtestDataset",
wait=True).json()
data = {"tags": ["#Lancelot", "name:Holy Grail", "blue"]}
ld_updated = self._patch_library_dataset(ld['id'], data)
self._assert_has_keys(ld_updated, "tags")
assert ld_updated["tags"] == "name:Lancelot, name:HolyGrail, blue"
def test_invalid_update_dataset_in_folder(self):
ld = self._create_dataset_in_folder_in_library(
"ForInvalidUpdateDataset", wait=True).json()
data = {'file_ext': 'nonexisting_type'}
create_response = self._patch(f"libraries/datasets/{ld['id']}",
data=data)
self._assert_status_code_is(create_response, 400)
assert 'This Galaxy does not recognize the datatype of:' in create_response.json(
)['err_msg']
def test_detect_datatype_of_dataset_in_folder(self):
ld = self._create_dataset_in_folder_in_library("ForDetectDataset",
wait=True).json()
data = {'file_ext': 'data'}
ld_updated = self._patch_library_dataset(ld['id'], data)
self._assert_has_keys(ld_updated, "file_ext")
assert ld_updated["file_ext"] == "data"
data = {'file_ext': 'auto'}
ld_updated = self._patch_library_dataset(ld['id'], data)
self._assert_has_keys(ld_updated, "file_ext")
assert ld_updated["file_ext"] == "txt"
def test_ldda_collection_import_to_history(self):
self._import_to_history(visible=True)
def test_ldda_collection_import_to_history_hide_source(self):
self._import_to_history(visible=False)
def test_import_paired_collection(self):
ld = self._create_dataset_in_folder_in_library(
"ForHistoryImport").json()
history_id = self.dataset_populator.new_history()
url = f"histories/{history_id}/contents"
collection_name = 'Paired-end data (from library)'
payload = {
'name':
collection_name,
'collection_type':
'list:paired',
"type":
"dataset_collection",
'element_identifiers':
json.dumps([{
'src':
'new_collection',
'name':
'pair1',
'collection_type':
'paired',
'element_identifiers': [{
'name': 'forward',
'src': 'ldda',
'id': ld['id']
}, {
'name': 'reverse',
'src': 'ldda',
'id': ld['id']
}]
}])
}
new_collection = self._post(url, payload).json()
assert new_collection['name'] == collection_name
pair = new_collection['elements'][0]
assert pair['element_identifier'] == 'pair1'
assert pair['object']['elements'][0]['object'][
'history_id'] == history_id
def _import_to_history(self, visible=True):
ld = self._create_dataset_in_folder_in_library(
"ForHistoryImport").json()
history_id = self.dataset_populator.new_history()
url = f"histories/{history_id}/contents"
collection_name = 'new_collection_name'
element_identifer = 'new_element_identifier'
payload = {
"collection_type":
"list",
"history_content_type":
"dataset_collection",
"model_class":
"HistoryDatasetCollectionAssociation",
"history_id":
history_id,
"name":
collection_name,
"hide_source_items":
not visible,
"element_identifiers":
json.dumps([{
"id": ld['id'],
"name": element_identifer,
"src": "ldda"
}]),
"type":
"dataset_collection",
"elements": []
}
new_collection = self._post(url, payload).json()
assert new_collection['name'] == collection_name
assert new_collection['element_count'] == 1
element = new_collection['elements'][0]
assert element['element_identifier'] == element_identifer
assert element['object']['visible'] == visible
def test_create_datasets_in_library_from_collection(self):
library = self.library_populator.new_private_library(
"ForCreateDatasetsFromCollection")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["xxx", "yyy"],
direct_upload=True).json()["outputs"][0]["id"]
payload = {
'from_hdca_id': hdca_id,
'create_type': 'file',
'folder_id': folder_id
}
create_response = self._post(f"libraries/{library['id']}/contents",
payload)
self._assert_status_code_is(create_response, 200)
def test_create_datasets_in_folder_from_collection(self):
library = self.library_populator.new_private_library(
"ForCreateDatasetsFromCollection")
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["xxx", "yyy"],
direct_upload=True).json()["outputs"][0]["id"]
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
payload = {'from_hdca_id': hdca_id}
create_response = self._post(f"folders/{folder_id}/contents", payload)
self._assert_status_code_is(create_response, 200)
assert len(create_response.json()) == 2
# Also test that anything different from a flat dataset collection list
# is refused
hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(
history_id).json()["outputs"][0]['id']
payload = {'from_hdca_id': hdca_pair_id}
create_response = self._post(f"folders/{folder_id}/contents", payload)
self._assert_status_code_is(create_response, 501)
assert create_response.json(
)['err_msg'] == 'Cannot add nested collections to library. Please flatten your collection first.'
def _create_folder(self, library):
create_data = dict(
folder_id=library["root_folder_id"],
create_type="folder",
name="New Folder",
)
return self._post(f"libraries/{library['id']}/contents",
data=create_data)
def _create_subfolder(self, containing_folder_id):
create_data = dict(
description="new subfolder desc",
name="New Subfolder",
)
return self._post(f"folders/{containing_folder_id}",
data=create_data,
json=True)
def _create_dataset_in_folder_in_library(self,
library_name,
content="1 2 3",
wait=False):
library = self.library_populator.new_private_library(library_name)
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id,
content=content,
wait=wait)['id']
payload = {
'from_hda_id': hda_id,
'create_type': 'file',
'folder_id': folder_id
}
ld = self._post(f"libraries/{folder_id}/contents", payload)
return ld
class LibrariesApiTestCase(ApiTestCase, TestsDatasets):
def setUp(self):
super(LibrariesApiTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
self.library_populator = LibraryPopulator(self.galaxy_interactor)
def test_create(self):
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries", data=data, admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "name")
assert library["name"] == "CreateTestLibrary"
def test_delete(self):
library = self.library_populator.new_library("DeleteTestLibrary")
create_response = self._delete("libraries/%s" % library["id"], admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, "deleted")
assert library["deleted"] is True
# Test undeleting
data = dict(undelete=True)
create_response = self._delete("libraries/%s" % library["id"], data=data, admin=True)
library = create_response.json()
self._assert_status_code_is(create_response, 200)
assert library["deleted"] is False
def test_nonadmin(self):
# Anons can't create libs
data = dict(name="CreateTestLibrary")
create_response = self._post("libraries", data=data, admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
# Anons can't delete libs
library = self.library_populator.new_library("AnonDeleteTestLibrary")
create_response = self._delete("libraries/%s" % library["id"], admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
# Anons can't update libs
data = dict(name="ChangedName", description="ChangedDescription", synopsis='ChangedSynopsis')
create_response = self._patch("libraries/%s" % library["id"], data=data, admin=False, anon=True)
self._assert_status_code_is(create_response, 403)
def test_update(self):
library = self.library_populator.new_library("UpdateTestLibrary")
data = dict(name='ChangedName', description='ChangedDescription', synopsis='ChangedSynopsis')
create_response = self._patch("libraries/%s" % library["id"], data=data, admin=True)
self._assert_status_code_is(create_response, 200)
library = create_response.json()
self._assert_has_keys(library, 'name', 'description', 'synopsis')
assert library['name'] == 'ChangedName'
assert library['description'] == 'ChangedDescription'
assert library['synopsis'] == 'ChangedSynopsis'
def test_create_private_library_permissions(self):
library = self.library_populator.new_library("PermissionTestLibrary")
library_id = library["id"]
role_id = self.library_populator.user_private_role_id()
self.library_populator.set_permissions(library_id, role_id)
create_response = self._create_folder(library)
self._assert_status_code_is(create_response, 200)
def test_create_dataset_denied(self):
library = self.library_populator.new_private_library("ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
with self._different_user():
payload = {'from_hda_id': hda_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 403)
def test_show_private_dataset_permissions(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
with self._different_user():
response = self.library_populator.show_ldda(library["id"], library_dataset["id"])
# TODO: this should really be 403 and a proper JSON exception.
self._assert_status_code_is(response, 400)
def test_create_dataset(self):
library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
self._assert_has_keys(library_dataset, "peek", "data_type")
assert library_dataset["peek"].find("create_test") >= 0
assert library_dataset["file_ext"] == "txt", library_dataset["file_ext"]
def test_fetch_upload_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "created_from_basename": "4.bed"}]
targets = [{
"destination": destination,
"items": items
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
assert dataset["genome_build"] == "hg19", dataset
assert dataset["misc_info"] == "my cool bed", dataset
assert dataset["file_ext"] == "bed", dataset
assert dataset["created_from_basename"] == "4.bed"
def test_fetch_zip_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("flat_zip")
bed_test_data_path = self.test_data_resolver.get_filename("4.bed.zip")
targets = [{
"destination": destination,
"items_from": "archive", "src": "files",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(bed_test_data_path, 'rb')}
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
def test_fetch_single_url_to_folder(self):
library, response = self._fetch_single_url_to_folder()
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
def test_fetch_single_url_with_invalid_datatype(self):
_, response = self._fetch_single_url_to_folder('xxx', assert_ok=False)
self._assert_status_code_is(response, 400)
assert response.json()['err_msg'] == "Requested extension 'xxx' unknown, cannot upload dataset."
def _fetch_single_url_to_folder(self, file_type='auto', assert_ok=True):
history_id, library, destination = self._setup_fetch_to_folder("single_url")
items = [{
"src": "url",
"url": FILE_URL,
"MD5": FILE_MD5,
"ext": file_type,
}]
targets = [{
"destination": destination,
"items": items,
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"validate_hashes": True
}
return library, self.dataset_populator.fetch(payload, assert_ok=assert_ok)
def test_legacy_upload_unknown_datatype(self):
library = self.library_populator.new_private_library("ForLegacyUpload")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
payload = {
'folder_id': folder_id,
'create_type': 'file',
'file_type': 'xxx',
'upload_option': 'upload_file',
'files_0|url_paste': FILE_URL,
}
create_response = self._post("libraries/%s/contents" % library['id'], payload)
self._assert_status_code_is(create_response, 400)
assert create_response.json() == "Requested extension 'xxx' unknown, cannot upload dataset."
def test_fetch_failed_validation(self):
# Exception handling is really rough here - we should be creating a dataset in error instead
# of just failing the job like this.
history_id, library, destination = self._setup_fetch_to_folder("single_url")
items = [{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed",
"MD5": "37b59762b59fff860460522d271bc112"
}]
targets = [{
"destination": destination,
"items": items,
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"validate_hashes": True
}
tool_response = self.dataset_populator.fetch(payload, assert_ok=False)
job = self.dataset_populator.check_run(tool_response)
self.dataset_populator.wait_for_job(job["id"])
job = tool_response.json()["jobs"][0]
details = self.dataset_populator.get_job_details(job["id"]).json()
assert details["state"] == "error", details
def test_fetch_url_archive_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("single_url")
targets = [{
"destination": destination,
"items_from": "archive",
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
@unittest.skip # reference URLs changed, checksums now invalid.
def test_fetch_bagit_archive_to_folder(self):
history_id, library, destination = self._setup_fetch_to_folder("bagit_archive")
example_bag_path = self.test_data_resolver.get_filename("example-bag.zip")
targets = [{
"destination": destination,
"items_from": "bagit_archive", "src": "files",
}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"__files": {"files_0|file_data": open(example_bag_path)},
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/README.txt")
assert dataset["file_size"] == 66, dataset
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/bdbag-profile.json")
assert dataset["file_size"] == 723, dataset
def _setup_fetch_to_folder(self, test_name):
return self.library_populator.setup_fetch_to_folder(test_name)
def test_create_dataset_in_folder(self):
library = self.library_populator.new_private_library("ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "id")
def test_create_dataset_in_subfolder(self):
library = self.library_populator.new_private_library("ForCreateDatasets")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
subfolder_response = self._create_subfolder(folder_id)
self._assert_status_code_is(folder_response, 200)
print(subfolder_response.json())
subfolder_id = subfolder_response.json()['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3 sub")['id']
payload = {'from_hda_id': hda_id}
create_response = self._post("folders/%s/contents" % subfolder_id, payload)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "id")
dataset_update_time = create_response.json()['update_time']
container_fetch_response = self.galaxy_interactor.get("folders/%s/contents" % folder_id)
container_update_time = container_fetch_response.json()['folder_contents'][0]['update_time']
assert dataset_update_time == container_update_time, container_fetch_response
def test_update_dataset_in_folder(self):
ld = self._create_dataset_in_folder_in_library("ForUpdateDataset")
data = {'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "name", "file_ext", "misc_info", "genome_build")
def test_update_dataset_tags(self):
ld = self._create_dataset_in_folder_in_library("ForTagtestDataset")
data = {"tags": ["#Lancelot", "name:Holy Grail", "blue"]}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "tags")
assert create_response.json()["tags"] == "name:Lancelot, name:HolyGrail, blue"
def test_invalid_update_dataset_in_folder(self):
ld = self._create_dataset_in_folder_in_library("ForInvalidUpdateDataset")
data = {'file_ext': 'nonexisting_type'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 400)
assert 'This Galaxy does not recognize the datatype of:' in create_response.json()['err_msg']
def test_detect_datatype_of_dataset_in_folder(self):
ld = self._create_dataset_in_folder_in_library("ForDetectDataset")
# Wait for metadata job to finish.
time.sleep(2)
data = {'file_ext': 'data'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "file_ext")
assert create_response.json()["file_ext"] == "data"
# Wait for metadata job to finish.
time.sleep(2)
data = {'file_ext': 'auto'}
create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data)
self._assert_status_code_is(create_response, 200)
self._assert_has_keys(create_response.json(), "file_ext")
assert create_response.json()["file_ext"] == "txt"
def test_create_datasets_in_library_from_collection(self):
library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["xxx", "yyy"], direct_upload=True).json()["outputs"][0]["id"]
payload = {'from_hdca_id': hdca_id, 'create_type': 'file', 'folder_id': folder_id}
create_response = self._post("libraries/%s/contents" % library['id'], payload)
self._assert_status_code_is(create_response, 200)
def test_create_datasets_in_folder_from_collection(self):
library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["xxx", "yyy"], direct_upload=True).json()["outputs"][0]["id"]
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
payload = {'from_hdca_id': hdca_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 200)
assert len(create_response.json()) == 2
# Also test that anything different from a flat dataset collection list
# is refused
hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id).json()["outputs"][0]['id']
payload = {'from_hdca_id': hdca_pair_id}
create_response = self._post("folders/%s/contents" % folder_id, payload)
self._assert_status_code_is(create_response, 501)
assert create_response.json()['err_msg'] == 'Cannot add nested collections to library. Please flatten your collection first.'
def _create_folder(self, library):
create_data = dict(
folder_id=library["root_folder_id"],
create_type="folder",
name="New Folder",
)
return self._post("libraries/%s/contents" % library["id"], data=create_data)
def _create_subfolder(self, containing_folder_id):
create_data = dict(
description="new subfolder desc",
name="New Subfolder",
)
return self._post("folders/%s" % containing_folder_id, data=create_data)
def _create_dataset_in_folder_in_library(self, library_name):
library = self.library_populator.new_private_library(library_name)
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
folder_id = folder_response.json()[0]['id']
history_id = self.dataset_populator.new_history()
hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id']
payload = {'from_hda_id': hda_id, 'create_type': 'file', 'folder_id': folder_id}
ld = self._post("libraries/%s/contents" % folder_id, payload)
return ld
