def setUp(self): super(HistoryContentsApiTestCase, self).setUp() self.history_id = self._new_history() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor)
def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history()
def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() self.library = self.library_populator.new_private_library("FolderContentsTestsLibrary") self.root_folder_id = self._create_folder_in_library("Test Folder Contents")
def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) for d in [self.library_dir, self.user_library_dir, self.ftp_upload_dir]: if os.path.exists(d): shutil.rmtree(d) os.mkdir(d)
class MaximumWorkflowJobsPerSchedulingIterationTestCase( integration_util.IntegrationTestCase): framework_tool_and_types = True def setUp(self): super(MaximumWorkflowJobsPerSchedulingIterationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) @classmethod def handle_galaxy_config_kwds(cls, config): config["maximum_workflow_jobs_per_scheduling_iteration"] = 1 def do_test(self): workflow_id = self.workflow_populator.upload_yaml_workflow(""" class: GalaxyWorkflow steps: - type: input_collection - tool_id: collection_creates_pair state: input1: $link: 0 - tool_id: collection_paired_test state: f1: $link: 1#paired_output - tool_id: cat_list state: input1: $link: 2#out1 """) with self.dataset_populator.test_history() as history_id: hdca1 = self.dataset_collection_populator.create_list_in_history( history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]).json() self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { '0': { "src": "hdca", "id": hdca1["id"] }, } invocation_id = self.workflow_populator.invoke_workflow( history_id, workflow_id, inputs) self.workflow_populator.wait_for_workflow(history_id, workflow_id, invocation_id) self.dataset_populator.wait_for_history(history_id, assert_ok=True) self.assertEqual( "a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0))
def setUp(self): super(MaximumWorkflowJobsPerSchedulingIterationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor)
def setUp(self): super(JobsApiTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor)
def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor)
class LibrariesApiTestCase(ApiTestCase, TestsDatasets): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor) def test_create(self): data = dict(name="CreateTestLibrary") create_response = self._post("libraries", data=data, admin=True) self._assert_status_code_is(create_response, 200) library = create_response.json() self._assert_has_keys(library, "name") assert library["name"] == "CreateTestLibrary" def test_delete(self): library = self.library_populator.new_library("DeleteTestLibrary") create_response = self._delete("libraries/%s" % library["id"], admin=True) self._assert_status_code_is(create_response, 200) library = create_response.json() self._assert_has_keys(library, "deleted") assert library["deleted"] is True # Test undeleting data = dict(undelete=True) create_response = self._delete("libraries/%s" % library["id"], data=data, admin=True) library = create_response.json() self._assert_status_code_is(create_response, 200) assert library["deleted"] is False def test_nonadmin(self): # Anons can't create libs data = dict(name="CreateTestLibrary") create_response = self._post("libraries", data=data, admin=False, anon=True) self._assert_status_code_is(create_response, 403) # Anons can't delete libs library = self.library_populator.new_library("AnonDeleteTestLibrary") create_response = self._delete("libraries/%s" % library["id"], admin=False, anon=True) self._assert_status_code_is(create_response, 403) # Anons can't update libs data = dict(name="ChangedName", description="ChangedDescription", synopsis='ChangedSynopsis') create_response = self._patch("libraries/%s" % library["id"], data=data, admin=False, anon=True) self._assert_status_code_is(create_response, 403) def test_update(self): library = self.library_populator.new_library("UpdateTestLibrary") data = dict(name='ChangedName', description='ChangedDescription', synopsis='ChangedSynopsis') create_response = self._patch("libraries/%s" % library["id"], data=data, admin=True) self._assert_status_code_is(create_response, 200) library = create_response.json() self._assert_has_keys(library, 'name', 'description', 'synopsis') assert library['name'] == 'ChangedName' assert library['description'] == 'ChangedDescription' assert library['synopsis'] == 'ChangedSynopsis' def test_create_private_library_permissions(self): library = self.library_populator.new_library("PermissionTestLibrary") library_id = library["id"] role_id = self.library_populator.user_private_role_id() self.library_populator.set_permissions(library_id, role_id) create_response = self._create_folder(library) self._assert_status_code_is(create_response, 200) def test_create_dataset_denied(self): library = self.library_populator.new_private_library( "ForCreateDatasets") folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] history_id = self.dataset_populator.new_history() hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] with self._different_user(): payload = {'from_hda_id': hda_id} create_response = self._post("folders/%s/contents" % folder_id, payload) self._assert_status_code_is(create_response, 403) def test_show_private_dataset_permissions(self): library, library_dataset = self.library_populator.new_library_dataset_in_private_library( "ForCreateDatasets", wait=True) with self._different_user(): response = self.library_populator.show_ldda( library["id"], library_dataset["id"]) # TODO: this should really be 403 and a proper JSON exception. self._assert_status_code_is(response, 400) def test_create_dataset(self): library, library_dataset = self.library_populator.new_library_dataset_in_private_library( "ForCreateDatasets", wait=True) self._assert_has_keys(library_dataset, "peek", "data_type") assert library_dataset["peek"].find("create_test") >= 0 assert library_dataset["file_ext"] == "txt", library_dataset[ "file_ext"] def test_fetch_upload_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder( "flat_zip") items = [{ "src": "files", "dbkey": "hg19", "info": "my cool bed", "created_from_basename": "4.bed" }] targets = [{"destination": destination, "items": items}] payload = { "history_id": history_id, # TODO: Shouldn't be needed :( "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload) dataset = self.library_populator.get_library_contents_with_path( library["id"], "/4.bed") assert dataset["file_size"] == 61, dataset assert dataset["genome_build"] == "hg19", dataset assert dataset["misc_info"] == "my cool bed", dataset assert dataset["file_ext"] == "bed", dataset assert dataset["created_from_basename"] == "4.bed" def test_fetch_zip_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder( "flat_zip") bed_test_data_path = self.test_data_resolver.get_filename("4.bed.zip") targets = [{ "destination": destination, "items_from": "archive", "src": "files", }] payload = { "history_id": history_id, # TODO: Shouldn't be needed :( "targets": json.dumps(targets), "__files": { "files_0|file_data": open(bed_test_data_path, 'rb') } } self.dataset_populator.fetch(payload) dataset = self.library_populator.get_library_contents_with_path( library["id"], "/4.bed") assert dataset["file_size"] == 61, dataset def test_fetch_single_url_to_folder(self): library, response = self._fetch_single_url_to_folder() dataset = self.library_populator.get_library_contents_with_path( library["id"], "/4.bed") assert dataset["file_size"] == 61, dataset def test_fetch_single_url_with_invalid_datatype(self): _, response = self._fetch_single_url_to_folder('xxx', assert_ok=False) self._assert_status_code_is(response, 400) assert response.json( )['err_msg'] == "Requested extension 'xxx' unknown, cannot upload dataset." def _fetch_single_url_to_folder(self, file_type='auto', assert_ok=True): history_id, library, destination = self._setup_fetch_to_folder( "single_url") items = [{ "src": "url", "url": FILE_URL, "MD5": FILE_MD5, "ext": file_type, }] targets = [{ "destination": destination, "items": items, }] payload = { "history_id": history_id, # TODO: Shouldn't be needed :( "targets": json.dumps(targets), "validate_hashes": True } return library, self.dataset_populator.fetch(payload, assert_ok=assert_ok) def test_legacy_upload_unknown_datatype(self): library = self.library_populator.new_private_library("ForLegacyUpload") folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] payload = { 'folder_id': folder_id, 'create_type': 'file', 'file_type': 'xxx', 'upload_option': 'upload_file', 'files_0|url_paste': FILE_URL, } create_response = self._post("libraries/%s/contents" % library['id'], payload) self._assert_status_code_is(create_response, 400) assert create_response.json( ) == "Requested extension 'xxx' unknown, cannot upload dataset." def test_fetch_failed_validation(self): # Exception handling is really rough here - we should be creating a dataset in error instead # of just failing the job like this. history_id, library, destination = self._setup_fetch_to_folder( "single_url") items = [{ "src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", "MD5": "37b59762b59fff860460522d271bc112", "name": "4.bed", }] targets = [{ "destination": destination, "items": items, }] payload = { "history_id": history_id, # TODO: Shouldn't be needed :( "targets": json.dumps(targets), "validate_hashes": True } tool_response = self.dataset_populator.fetch(payload, assert_ok=False) job = self.dataset_populator.check_run(tool_response) self.dataset_populator.wait_for_job(job["id"]) job = tool_response.json()["jobs"][0] details = self.dataset_populator.get_job_details(job["id"]).json() assert details["state"] == "ok", details dataset = self.library_populator.get_library_contents_with_path( library["id"], "/4.bed") assert dataset["state"] == "error", dataset def test_fetch_url_archive_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder( "single_url") targets = [{ "destination": destination, "items_from": "archive", "src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip", }] payload = { "history_id": history_id, # TODO: Shouldn't be needed :( "targets": json.dumps(targets), } self.dataset_populator.fetch(payload) dataset = self.library_populator.get_library_contents_with_path( library["id"], "/4.bed") assert dataset["file_size"] == 61, dataset @unittest.skip # reference URLs changed, checksums now invalid. def test_fetch_bagit_archive_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder( "bagit_archive") example_bag_path = self.test_data_resolver.get_filename( "example-bag.zip") targets = [{ "destination": destination, "items_from": "bagit_archive", "src": "files", }] payload = { "history_id": history_id, # TODO: Shouldn't be needed :( "targets": json.dumps(targets), "__files": { "files_0|file_data": open(example_bag_path) }, } self.dataset_populator.fetch(payload) dataset = self.library_populator.get_library_contents_with_path( library["id"], "/README.txt") assert dataset["file_size"] == 66, dataset dataset = self.library_populator.get_library_contents_with_path( library["id"], "/bdbag-profile.json") assert dataset["file_size"] == 723, dataset def _setup_fetch_to_folder(self, test_name): return self.library_populator.setup_fetch_to_folder(test_name) def test_create_dataset_in_folder(self): library = self.library_populator.new_private_library( "ForCreateDatasets") folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] history_id = self.dataset_populator.new_history() hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] payload = {'from_hda_id': hda_id} create_response = self._post("folders/%s/contents" % folder_id, payload) self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "name", "id") def test_create_dataset_in_subfolder(self): library = self.library_populator.new_private_library( "ForCreateDatasets") folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] subfolder_response = self._create_subfolder(folder_id) self._assert_status_code_is(folder_response, 200) print(subfolder_response.json()) subfolder_id = subfolder_response.json()['id'] history_id = self.dataset_populator.new_history() hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3 sub")['id'] payload = {'from_hda_id': hda_id} create_response = self._post("folders/%s/contents" % subfolder_id, payload) self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "name", "id") dataset_update_time = create_response.json()['update_time'] container_fetch_response = self.galaxy_interactor.get( "folders/%s/contents" % folder_id) container_update_time = container_fetch_response.json( )['folder_contents'][0]['update_time'] assert dataset_update_time == container_update_time, container_fetch_response def test_update_dataset_in_folder(self): ld = self._create_dataset_in_folder_in_library("ForUpdateDataset") data = { 'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build' } create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "name", "file_ext", "misc_info", "genome_build") def test_update_dataset_tags(self): ld = self._create_dataset_in_folder_in_library("ForTagtestDataset") data = {"tags": ["#Lancelot", "name:Holy Grail", "blue"]} create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "tags") assert create_response.json( )["tags"] == "name:Lancelot, name:HolyGrail, blue" def test_invalid_update_dataset_in_folder(self): ld = self._create_dataset_in_folder_in_library( "ForInvalidUpdateDataset") data = {'file_ext': 'nonexisting_type'} create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) self._assert_status_code_is(create_response, 400) assert 'This Galaxy does not recognize the datatype of:' in create_response.json( )['err_msg'] def test_detect_datatype_of_dataset_in_folder(self): ld = self._create_dataset_in_folder_in_library("ForDetectDataset") # Wait for metadata job to finish. time.sleep(2) data = {'file_ext': 'data'} create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "file_ext") assert create_response.json()["file_ext"] == "data" # Wait for metadata job to finish. time.sleep(2) data = {'file_ext': 'auto'} create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "file_ext") assert create_response.json()["file_ext"] == "txt" def test_ldda_collection_import_to_history(self): self._import_to_history(visible=True) def test_ldda_collection_import_to_history_hide_source(self): self._import_to_history(visible=False) def test_import_paired_collection(self): ld = self._create_dataset_in_folder_in_library( "ForHistoryImport").json() history_id = self.dataset_populator.new_history() url = "histories/%s/contents" % history_id collection_name = 'Paired-end data (from library)' payload = { 'name': collection_name, 'collection_type': 'list:paired', "type": "dataset_collection", 'element_identifiers': json.dumps([{ 'src': 'new_collection', 'name': 'pair1', 'collection_type': 'paired', 'element_identifiers': [{ 'name': 'forward', 'src': 'ldda', 'id': ld['id'] }, { 'name': 'reverse', 'src': 'ldda', 'id': ld['id'] }] }]) } new_collection = self._post(url, payload).json() assert new_collection['name'] == collection_name pair = new_collection['elements'][0] assert pair['element_identifier'] == 'pair1' assert pair['object']['elements'][0]['object'][ 'history_id'] == history_id def _import_to_history(self, visible=True): ld = self._create_dataset_in_folder_in_library( "ForHistoryImport").json() history_id = self.dataset_populator.new_history() url = "histories/%s/contents" % history_id collection_name = 'new_collection_name' element_identifer = 'new_element_identifier' payload = { "collection_type": "list", "history_content_type": "dataset_collection", "model_class": "HistoryDatasetCollectionAssociation", "history_id": history_id, "name": collection_name, "hide_source_items": not visible, "element_identifiers": json.dumps([{ "id": ld['id'], "name": element_identifer, "src": "ldda" }]), "type": "dataset_collection", "elements": [] } new_collection = self._post(url, payload).json() assert new_collection['name'] == collection_name assert new_collection['element_count'] == 1 element = new_collection['elements'][0] assert element['element_identifier'] == element_identifer assert element['object']['visible'] == visible def test_create_datasets_in_library_from_collection(self): library = self.library_populator.new_private_library( "ForCreateDatasetsFromCollection") folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] history_id = self.dataset_populator.new_history() hdca_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["xxx", "yyy"], direct_upload=True).json()["outputs"][0]["id"] payload = { 'from_hdca_id': hdca_id, 'create_type': 'file', 'folder_id': folder_id } create_response = self._post("libraries/%s/contents" % library['id'], payload) self._assert_status_code_is(create_response, 200) def test_create_datasets_in_folder_from_collection(self): library = self.library_populator.new_private_library( "ForCreateDatasetsFromCollection") history_id = self.dataset_populator.new_history() hdca_id = self.dataset_collection_populator.create_list_in_history( history_id, contents=["xxx", "yyy"], direct_upload=True).json()["outputs"][0]["id"] folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] payload = {'from_hdca_id': hdca_id} create_response = self._post("folders/%s/contents" % folder_id, payload) self._assert_status_code_is(create_response, 200) assert len(create_response.json()) == 2 # Also test that anything different from a flat dataset collection list # is refused hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history( history_id).json()["outputs"][0]['id'] payload = {'from_hdca_id': hdca_pair_id} create_response = self._post("folders/%s/contents" % folder_id, payload) self._assert_status_code_is(create_response, 501) assert create_response.json( )['err_msg'] == 'Cannot add nested collections to library. Please flatten your collection first.' def _create_folder(self, library): create_data = dict( folder_id=library["root_folder_id"], create_type="folder", name="New Folder", ) return self._post("libraries/%s/contents" % library["id"], data=create_data) def _create_subfolder(self, containing_folder_id): create_data = dict( description="new subfolder desc", name="New Subfolder", ) return self._post("folders/%s" % containing_folder_id, data=create_data) def _create_dataset_in_folder_in_library(self, library_name): library = self.library_populator.new_private_library(library_name) folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] history_id = self.dataset_populator.new_history() hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] payload = { 'from_hda_id': hda_id, 'create_type': 'file', 'folder_id': folder_id } ld = self._post("libraries/%s/contents" % folder_id, payload) return ld
class DatasetsApiTestCase(ApiTestCase): def setUp(self): super(DatasetsApiTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_index(self): index_response = self._get("datasets") self._assert_status_code_is(index_response, 200) def test_search_datasets(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] payload = {'limit': 1, 'offset': 0} index_response = self._get("datasets", payload).json() assert len(index_response) == 1 assert index_response[0]['id'] == hda_id hdca_id = self.dataset_collection_populator.create_list_in_history( self.history_id, contents=["1\n2\n3"]).json()['id'] payload = {'limit': 3, 'offset': 0} index_response = self._get("datasets", payload).json() assert len(index_response) == 3 assert index_response[0]['id'] == hdca_id assert index_response[0][ 'history_content_type'] == 'dataset_collection' assert index_response[2]['id'] == hda_id assert index_response[2]['history_content_type'] == 'dataset' payload = { 'limit': 2, 'offset': 0, 'q': ['history_content_type'], 'qv': ['dataset'] } index_response = self._get("datasets", payload).json() assert index_response[1]['id'] == hda_id def test_search_by_tag(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] update_payload = { 'tags': ['cool:new_tag', 'cool:another_tag'], } updated_hda = self._put( "histories/{history_id}/contents/{hda_id}".format( history_id=self.history_id, hda_id=hda_id), update_payload).json() assert 'cool:new_tag' in updated_hda['tags'] assert 'cool:another_tag' in updated_hda['tags'] payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag'], 'qv': ['dataset', 'cool:new_tag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 1 payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-contains'], 'qv': ['dataset', 'new_tag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 1 payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-contains'], 'qv': ['dataset', 'notag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 0 def test_search_by_tool_id(self): self.dataset_populator.new_dataset(self.history_id) payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset', 'upload1'] } assert len(self._get("datasets", payload).json()) == 1 payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset', 'uploadX'] } assert len(self._get("datasets", payload).json()) == 0 payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id-contains'], 'qv': ['dataset', 'pload1'] } assert len(self._get("datasets", payload).json()) == 1 self.dataset_collection_populator.create_list_in_history( self.history_id, name="search by tool id", contents=["1\n2\n3"]).json() self.dataset_populator.wait_for_history(self.history_id) payload = { 'limit': 10, 'offset': 0, 'history_id': self.history_id, 'q': ['name', 'tool_id'], 'qv': ['search by tool id', 'upload1'] } result = self._get("datasets", payload).json() assert result[0]['name'] == 'search by tool id', result payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset_collection', 'uploadX'] } result = self._get("datasets", payload).json() assert len(result) == 0 def test_invalid_search(self): payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-invalid_op'], 'qv': ['dataset', 'notag'] } index_response = self._get("datasets", payload) self._assert_status_code_is(index_response, 400) assert index_response.json()['err_msg'] == 'bad op in filter' def test_search_returns_only_accessible(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] with self._different_user(): payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type'], 'qv': ['dataset'] } index_response = self._get("datasets", payload).json() for item in index_response: assert hda_id != item['id'] def test_show(self): hda1 = self.dataset_populator.new_dataset(self.history_id) show_response = self._get("datasets/%s" % (hda1["id"])) self._assert_status_code_is(show_response, 200) self.__assert_matches_hda(hda1, show_response.json()) def __assert_matches_hda(self, input_hda, query_hda): self._assert_has_keys(query_hda, "id", "name") assert input_hda["name"] == query_hda["name"] assert input_hda["id"] == query_hda["id"] def test_display(self): contents = textwrap.dedent("""\ 1 2 3 4 A B C D 10 20 30 40 """) hda1 = self.dataset_populator.new_dataset(self.history_id, content=contents) self.dataset_populator.wait_for_history(self.history_id) display_response = self._get( "histories/%s/contents/%s/display" % (self.history_id, hda1["id"]), {'raw': 'True'}) self._assert_status_code_is(display_response, 200) assert display_response.text == contents def test_tag_change(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] payload = { 'item_id': hda_id, 'item_class': 'HistoryDatasetAssociation', 'item_tags': ['cool:tag_a', 'cool:tag_b', 'tag_c', 'name:tag_d', '#tag_e'], } self._put("tags", payload).json() updated_hda = self._get( "histories/{history_id}/contents/{hda_id}".format( history_id=self.history_id, hda_id=hda_id)).json() assert 'cool:tag_a' in updated_hda['tags'] assert 'cool:tag_b' in updated_hda['tags'] assert 'tag_c' in updated_hda['tags'] assert 'name:tag_d' in updated_hda['tags'] assert 'name:tag_e' in updated_hda['tags']
class ImportExportHistoryTestCase(ApiTestCase, BaseHistories): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) def test_import_export(self): history_name = "for_export_default" history_id = self.dataset_populator.setup_history_for_export_testing( history_name) imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2) def upload_job_check(job): assert job["tool_id"] == "upload1" def check_discarded(hda): assert hda["deleted"] assert hda["state"] == "discarded", hda assert hda["purged"] is True self._check_imported_dataset(history_id=imported_history_id, hid=1, job_checker=upload_job_check) self._check_imported_dataset(history_id=imported_history_id, hid=2, has_job=False, hda_checker=check_discarded, job_checker=upload_job_check) imported_content = self.dataset_populator.get_history_dataset_content( history_id=imported_history_id, hid=1, ) assert imported_content == "1 2 3\n" def test_import_1901_histories(self): f = open( self.test_data_resolver.get_filename( "exports/1901_two_datasets.tgz"), 'rb') import_data = dict(archive_source='', archive_file=f) self._import_history_and_wait(import_data, "API Test History", wait_on_history_length=2) def test_import_export_include_deleted(self): history_name = "for_export_include_deleted" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_populator.new_dataset(history_id, content="1 2 3") deleted_hda = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True) self.dataset_populator.delete_dataset(history_id, deleted_hda["id"]) imported_history_id = self._reimport_history( history_id, history_name, wait_on_history_length=2, export_kwds={"include_deleted": "True"}) self._assert_history_length(imported_history_id, 2) def upload_job_check(job): assert job["tool_id"] == "upload1" def check_deleted_not_purged(hda): assert hda["state"] == "ok", hda assert hda["deleted"] is True, hda assert hda["purged"] is False, hda self._check_imported_dataset(history_id=imported_history_id, hid=1, job_checker=upload_job_check) self._check_imported_dataset(history_id=imported_history_id, hid=2, hda_checker=check_deleted_not_purged, job_checker=upload_job_check) imported_content = self.dataset_populator.get_history_dataset_content( history_id=imported_history_id, hid=1, ) assert imported_content == "1 2 3\n" @skip_without_tool("job_properties") def test_import_export_failed_job(self): history_name = "for_export_include_failed_job" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_populator.run_tool('job_properties', inputs={'failbool': True}, history_id=history_id, assert_ok=False) self.dataset_populator.wait_for_history(history_id, assert_ok=False) imported_history_id = self._reimport_history( history_id, history_name, assert_ok=False, wait_on_history_length=4, export_kwds={"include_deleted": "True"}) self._assert_history_length(imported_history_id, 4) def check_failed(hda_or_job): print(hda_or_job) assert hda_or_job["state"] == "error", hda_or_job self.dataset_populator._summarize_history(imported_history_id) self._check_imported_dataset(history_id=imported_history_id, hid=1, assert_ok=False, hda_checker=check_failed, job_checker=check_failed) def test_import_metadata_regeneration(self): history_name = "for_import_metadata_regeneration" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_populator.new_dataset( history_id, content=open(self.test_data_resolver.get_filename("1.bam"), 'rb'), file_type='bam', wait=True) imported_history_id = self._reimport_history(history_id, history_name) self._assert_history_length(imported_history_id, 1) self._check_imported_dataset(history_id=imported_history_id, hid=1) import_bam_metadata = self.dataset_populator.get_history_dataset_details( history_id=imported_history_id, hid=1, ) # The cleanup() method of the __IMPORT_HISTORY__ job (which is executed # after the job has entered its final state): # - creates a new dataset with 'ok' state and adds it to the history # - starts a __SET_METADATA__ job to regenerate the dataset metadata, if # needed # We need to wait a bit for the creation of the __SET_METADATA__ job. time.sleep(1) self.dataset_populator.wait_for_history_jobs(imported_history_id, assert_ok=True) bai_metadata = import_bam_metadata["meta_files"][0] assert bai_metadata["file_type"] == "bam_index" api_url = bai_metadata["download_url"].split("api/", 1)[1] bai_response = self._get(api_url) self._assert_status_code_is(bai_response, 200) assert len(bai_response.content) > 4 def test_import_export_collection(self): history_name = "for_export_with_collections" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_collection_populator.create_list_in_history( history_id, contents=["Hello", "World"], direct_upload=True) imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3) self._assert_history_length(imported_history_id, 3) def check_elements(elements): assert len(elements) == 2 element0 = elements[0]["object"] element1 = elements[1]["object"] for element in [element0, element1]: assert not element["visible"] assert not element["deleted"] assert element["state"] == "ok" assert element0["hid"] == 2 assert element1["hid"] == 3 self._check_imported_collection(imported_history_id, hid=1, collection_type="list", elements_checker=check_elements) def test_import_export_nested_collection(self): history_name = "for_export_with_nested_collections" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_collection_populator.create_list_of_pairs_in_history( history_id) imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3) self._assert_history_length(imported_history_id, 3) def check_elements(elements): assert len(elements) == 1 element0 = elements[0]["object"] self._assert_has_keys(element0, "elements", "collection_type") child_elements = element0["elements"] assert len(child_elements) == 2 assert element0["collection_type"] == "paired" self._check_imported_collection(imported_history_id, hid=1, collection_type="list:paired", elements_checker=check_elements) def _reimport_history(self, history_id, history_name, wait_on_history_length=None, assert_ok=True, export_kwds=None): # Ensure the history is ready to go... export_kwds = export_kwds or {} self.dataset_populator.wait_for_history(history_id, assert_ok=assert_ok) return self.dataset_populator.reimport_history( history_id, history_name, wait_on_history_length=wait_on_history_length, export_kwds=export_kwds, url=self.url, api_key=self.galaxy_interactor.api_key) def _import_history_and_wait(self, import_data, history_name, wait_on_history_length=None): imported_history_id = self.dataset_populator.import_history_and_wait_for_name( import_data, history_name) if wait_on_history_length: self.dataset_populator.wait_on_history_length( imported_history_id, wait_on_history_length) return imported_history_id def _assert_history_length(self, history_id, n): contents_response = self._get(f"histories/{history_id}/contents") self._assert_status_code_is(contents_response, 200) contents = contents_response.json() assert len(contents) == n, contents def _check_imported_dataset(self, history_id, hid, assert_ok=True, has_job=True, hda_checker=None, job_checker=None): imported_dataset_metadata = self.dataset_populator.get_history_dataset_details( history_id=history_id, hid=hid, assert_ok=assert_ok, ) assert imported_dataset_metadata["history_content_type"] == "dataset" assert imported_dataset_metadata["history_id"] == history_id if hda_checker is not None: hda_checker(imported_dataset_metadata) assert "creating_job" in imported_dataset_metadata job_id = imported_dataset_metadata["creating_job"] if has_job: assert job_id job_details = self.dataset_populator.get_job_details(job_id, full=True) assert job_details.status_code == 200, job_details.content job = job_details.json() assert 'history_id' in job, job assert job['history_id'] == history_id, job if job_checker is not None: job_checker(job) def _check_imported_collection(self, history_id, hid, collection_type=None, elements_checker=None): imported_collection_metadata = self.dataset_populator.get_history_collection_details( history_id=history_id, hid=hid, ) assert imported_collection_metadata[ "history_content_type"] == "dataset_collection" assert imported_collection_metadata["history_id"] == history_id assert "collection_type" in imported_collection_metadata assert "elements" in imported_collection_metadata if collection_type is not None: assert imported_collection_metadata[ "collection_type"] == collection_type, imported_collection_metadata if elements_checker is not None: elements_checker(imported_collection_metadata["elements"])
class FolderContentsApiTestCase(ApiTestCase): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() self.library = self.library_populator.new_private_library("FolderContentsTestsLibrary") self.root_folder_id = self._create_folder_in_library("Test Folder Contents") def test_create_hda_with_ldda_message(self): hda_id = self._create_hda() ldda_message = "Test message" data = { "from_hda_id": hda_id, "ldda_message": ldda_message, } ldda = self._create_content_in_folder_with_payload(self.root_folder_id, data) self._assert_has_keys(ldda, "name", "id") def test_create_hdca_with_ldda_message(self): contents = ["dataset01", "dataset02"] hdca_id = self._create_hdca_with_contents(contents) ldda_message = "Test message" data = { "from_hdca_id": hdca_id, "ldda_message": ldda_message, } lddas = self._create_content_in_folder_with_payload(self.root_folder_id, data) assert len(contents) == len(lddas) def test_index(self): folder_id = self._create_folder_in_library("Test Folder Contents Index") self._create_dataset_in_folder(folder_id) response = self._get(f"folders/{folder_id}/contents") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == 1 def test_index_include_deleted(self): folder_name = "Test Folder Contents Index include deleted" folder_id = self._create_folder_in_library(folder_name) hda_id = self._create_dataset_in_folder(folder_id) self._delete_library_dataset(hda_id) response = self._get(f"folders/{folder_id}/contents") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == 0 include_deleted = True response = self._get(f"folders/{folder_id}/contents?include_deleted={include_deleted}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == 1 assert contents[0]["deleted"] is True def test_index_limit_offset(self): folder_name = "Test Folder Contents Index limit" folder_id = self._create_folder_in_library(folder_name) num_subfolders = 5 for index in range(num_subfolders): self._create_subfolder_in(folder_id, name=f"Folder_{index}") num_datasets = 5 for _ in range(num_datasets): self._create_dataset_in_folder(folder_id) total_items = num_datasets + num_subfolders response = self._get(f"folders/{folder_id}/contents") self._assert_status_code_is(response, 200) original_contents = response.json()["folder_contents"] assert len(original_contents) == total_items limit = 7 response = self._get(f"folders/{folder_id}/contents?limit={limit}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == limit offset = 3 response = self._get(f"folders/{folder_id}/contents?offset={offset}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == total_items - offset limit = 4 offset = 4 response = self._get(f"folders/{folder_id}/contents?limit={limit}&offset={offset}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == limit expected_query_result = original_contents[offset:offset + limit] for index in range(limit): assert contents[index]["id"] == expected_query_result[index]["id"] def test_index_search_text(self): folder_name = "Test Folder Contents Index search text" folder_id = self._create_folder_in_library(folder_name) dataset_names = ["AB", "BC", "ABC"] for name in dataset_names: self._create_dataset_in_folder(folder_id, name) subfolder_names = ["Folder_A", "Folder_C"] for name in subfolder_names: self._create_subfolder_in(folder_id, name) all_names = dataset_names + subfolder_names search_terms = ["A", "B", "C"] for search_text in search_terms: response = self._get(f"folders/{folder_id}/contents?search_text={search_text}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] matching_names = [name for name in all_names if search_text in name] assert len(contents) == len(matching_names) def test_index_permissions_include_deleted(self): folder_name = "Test Folder Contents Index permissions include deteleted" folder_id = self._create_folder_in_library(folder_name) num_subfolders = 5 subfolder_ids: List[str] = [] deleted_subfolder_ids: List[str] = [] for index in range(num_subfolders): id = self._create_subfolder_in(folder_id, name=f"Folder_{index}") subfolder_ids.append(id) for index, subfolder_id in enumerate(subfolder_ids): if index % 2 == 0: self._delete_subfolder(subfolder_id) deleted_subfolder_ids.append(subfolder_id) num_datasets = 5 datasets_ids: List[str] = [] deleted_datasets_ids: List[str] = [] for _ in range(num_datasets): id = self._create_dataset_in_folder(folder_id) datasets_ids.append(id) for index, ldda_id in enumerate(datasets_ids): if index % 2 == 0: self._delete_library_dataset(ldda_id) deleted_datasets_ids.append(ldda_id) num_total_contents = num_subfolders + num_datasets num_non_deleted = num_total_contents - len(deleted_subfolder_ids) - len(deleted_datasets_ids) # Verify deleted contents are not listed include_deleted = False response = self._get(f"folders/{folder_id}/contents?include_deleted={include_deleted}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == num_non_deleted include_deleted = True # Admins can see everything... response = self._get(f"folders/{folder_id}/contents?include_deleted={include_deleted}", admin=True) self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == num_total_contents # Owner can see everything too response = self._get(f"folders/{folder_id}/contents?include_deleted={include_deleted}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == num_total_contents # Users with access but no modify permission can't see deleted with self._different_user(): different_user_role_id = self.dataset_populator.user_private_role_id() self._allow_library_access_to_user_role(different_user_role_id) with self._different_user(): response = self._get(f"folders/{folder_id}/contents?include_deleted={include_deleted}") self._assert_status_code_is(response, 200) contents = response.json()["folder_contents"] assert len(contents) == num_non_deleted def _create_folder_in_library(self, name: str) -> Any: root_folder_id = self.library["root_folder_id"] return self._create_subfolder_in(root_folder_id, name) def _create_subfolder_in(self, folder_id: str, name: str) -> str: data = { "name": name, "description": f"The description of {name}", } create_response = self._post(f"folders/{folder_id}", data=data) self._assert_status_code_is(create_response, 200) folder = create_response.json() return folder["id"] def _create_dataset_in_folder(self, folder_id: str, name: Optional[str] = None) -> str: hda_id = self._create_hda(name) data = { "from_hda_id": hda_id, } ldda = self._create_content_in_folder_with_payload(folder_id, data) return ldda["id"] def _create_content_in_folder_with_payload(self, folder_id: str, payload) -> Any: create_response = self._post(f"folders/{folder_id}/contents", data=payload) self._assert_status_code_is(create_response, 200) return create_response.json() def _create_hda(self, name: Optional[str] = None) -> str: hda = self.dataset_populator.new_dataset(self.history_id, name=name) hda_id = hda["id"] return hda_id def _create_hdca_with_contents(self, contents: List[str]) -> str: hdca = self.dataset_collection_populator.create_list_in_history(self.history_id, contents=contents, direct_upload=True).json()["outputs"][0] hdca_id = hdca["id"] return hdca_id def _delete_library_dataset(self, ldda_id: str) -> None: delete_response = self._delete(f"libraries/datasets/{ldda_id}") self._assert_status_code_is(delete_response, 200) def _delete_subfolder(self, folder_id: str) -> None: delete_response = self._delete(f"folders/{folder_id}") self._assert_status_code_is(delete_response, 200) def _allow_library_access_to_user_role(self, role_id: str): library_id = self.library["id"] action = "set_permissions" data = { "access_ids[]": role_id, } response = self._post(f"libraries/{library_id}/permissions?action={action}", data=data, admin=True, json=True) self._assert_status_code_is(response, 200)
def setUp(self): super(ImportExportHistoryTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor)
class DatasetsApiTestCase(ApiTestCase): history_id: str def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_index(self): index_response = self._get("datasets") self._assert_status_code_is(index_response, 200) def test_search_datasets(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] payload = {'limit': 1, 'offset': 0} index_response = self._get("datasets", payload).json() assert len(index_response) == 1 assert index_response[0]['id'] == hda_id hdca_id = self.dataset_collection_populator.create_list_in_history( self.history_id, contents=["1\n2\n3"]).json()['id'] index_payload_1 = {'limit': 3, 'offset': 0} index_response = self._get("datasets", index_payload_1).json() assert len(index_response) == 3 assert index_response[0]['id'] == hdca_id assert index_response[0][ 'history_content_type'] == 'dataset_collection' assert index_response[2]['id'] == hda_id assert index_response[2]['history_content_type'] == 'dataset' index_payload_2 = { 'limit': 2, 'offset': 0, 'q': ['history_content_type'], 'qv': ['dataset'] } index_response = self._get("datasets", index_payload_2).json() assert index_response[1]['id'] == hda_id def test_search_by_tag(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] update_payload = { 'tags': ['cool:new_tag', 'cool:another_tag'], } updated_hda = self._put( f"histories/{self.history_id}/contents/{hda_id}", update_payload, json=True).json() assert 'cool:new_tag' in updated_hda['tags'] assert 'cool:another_tag' in updated_hda['tags'] payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag'], 'qv': ['dataset', 'cool:new_tag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 1 payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-contains'], 'qv': ['dataset', 'new_tag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 1 payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-contains'], 'qv': ['dataset', 'notag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 0 def test_search_by_tool_id(self): self.dataset_populator.new_dataset(self.history_id) payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset', 'upload1'] } assert len(self._get("datasets", payload).json()) == 1 payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset', 'uploadX'] } assert len(self._get("datasets", payload).json()) == 0 payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id-contains'], 'qv': ['dataset', 'pload1'] } assert len(self._get("datasets", payload).json()) == 1 self.dataset_collection_populator.create_list_in_history( self.history_id, name="search by tool id", contents=["1\n2\n3"]).json() self.dataset_populator.wait_for_history(self.history_id) payload = { 'limit': 10, 'offset': 0, 'history_id': self.history_id, 'q': ['name', 'tool_id'], 'qv': ['search by tool id', 'upload1'] } result = self._get("datasets", payload).json() assert result[0]['name'] == 'search by tool id', result payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset_collection', 'uploadX'] } result = self._get("datasets", payload).json() assert len(result) == 0 def test_invalid_search(self): payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-invalid_op'], 'qv': ['dataset', 'notag'] } index_response = self._get("datasets", payload) self._assert_status_code_is(index_response, 400) assert index_response.json()['err_msg'] == 'bad op in filter' def test_search_returns_only_accessible(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] with self._different_user(): payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type'], 'qv': ['dataset'] } index_response = self._get("datasets", payload).json() for item in index_response: assert hda_id != item['id'] def test_show(self): hda1 = self.dataset_populator.new_dataset(self.history_id) show_response = self._get(f"datasets/{hda1['id']}") self._assert_status_code_is(show_response, 200) self.__assert_matches_hda(hda1, show_response.json()) def test_show_permission_denied(self): hda = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.make_private(history_id=self.history_id, dataset_id=hda['id']) with self._different_user(): show_response = self._get(f"datasets/{hda['id']}") self._assert_status_code_is(show_response, 403) def test_admin_can_update_permissions(self): # Create private dataset hda = self.dataset_populator.new_dataset(self.history_id) dataset_id = hda['id'] self.dataset_populator.make_private(history_id=self.history_id, dataset_id=dataset_id) # Admin removes restrictions payload = {"action": "remove_restrictions"} update_response = self._put(f"datasets/{dataset_id}/permissions", payload, admin=True, json=True) self._assert_status_code_is_ok(update_response) # Other users can access the dataset with self._different_user(): show_response = self._get(f"datasets/{hda['id']}") self._assert_status_code_is_ok(show_response) def __assert_matches_hda(self, input_hda, query_hda): self._assert_has_keys(query_hda, "id", "name") assert input_hda["name"] == query_hda["name"] assert input_hda["id"] == query_hda["id"] def test_display(self): contents = textwrap.dedent("""\ 1 2 3 4 A B C D 10 20 30 40 """) hda1 = self.dataset_populator.new_dataset(self.history_id, content=contents) self.dataset_populator.wait_for_history(self.history_id) display_response = self._get( f"histories/{self.history_id}/contents/{hda1['id']}/display", {'raw': 'True'}) self._assert_status_code_is(display_response, 200) assert display_response.text == contents def test_tag_change(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] payload = { 'item_id': hda_id, 'item_class': 'HistoryDatasetAssociation', 'item_tags': ['cool:tag_a', 'cool:tag_b', 'tag_c', 'name:tag_d', '#tag_e'], } put_response = self._put("tags", data=payload, json=True) self._assert_status_code_is_ok(put_response) updated_hda = self._get( f"histories/{self.history_id}/contents/{hda_id}").json() assert 'cool:tag_a' in updated_hda['tags'] assert 'cool:tag_b' in updated_hda['tags'] assert 'tag_c' in updated_hda['tags'] assert 'name:tag_d' in updated_hda['tags'] assert 'name:tag_e' in updated_hda['tags'] @skip_without_tool("cat_data_and_sleep") def test_update_datatype(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] original_hda = self._get( f"histories/{self.history_id}/contents/{hda_id}").json() assert original_hda['extension'] == 'txt' assert original_hda['data_type'] == 'galaxy.datatypes.data.Text' assert 'scatterplot' not in [ viz['name'] for viz in original_hda['visualizations'] ] inputs = { 'input1': { 'src': 'hda', 'id': hda_id }, 'sleep_time': 10, } run_response = self.dataset_populator.run_tool_raw( "cat_data_and_sleep", inputs, self.history_id, ) queued_id = run_response.json()["outputs"][0]["id"] update_while_incomplete_response = self._put( # try updating datatype while used as output of a running job f"histories/{self.history_id}/contents/{queued_id}", data={'datatype': 'tabular'}, json=True) self._assert_status_code_is(update_while_incomplete_response, 400) self.dataset_populator.wait_for_history_jobs( self.history_id) # now wait for upload to complete successful_updated_hda_response = self._put( f"histories/{self.history_id}/contents/{hda_id}", data={ 'datatype': 'tabular' }, json=True).json() assert successful_updated_hda_response['extension'] == 'tabular' assert successful_updated_hda_response[ 'data_type'] == 'galaxy.datatypes.tabular.Tabular' assert 'scatterplot' in [ viz['name'] for viz in successful_updated_hda_response['visualizations'] ] invalidly_updated_hda_response = self._put( # try updating with invalid datatype f"histories/{self.history_id}/contents/{hda_id}", data={'datatype': 'invalid'}, json=True) self._assert_status_code_is(invalidly_updated_hda_response, 400) @skip_without_datatype("velvet") def test_composite_datatype_download(self): item = { "src": "composite", "ext": "velvet", "composite": { "items": [ { "src": "pasted", "paste_content": "sequences content" }, { "src": "pasted", "paste_content": "roadmaps content" }, { "src": "pasted", "paste_content": "log content" }, ] }, } output = self.dataset_populator.fetch_hda(self.history_id, item, wait=True) print(output) response = self._get( f"histories/{self.history_id}/contents/{output['id']}/display?to_ext=zip" ) self._assert_status_code_is(response, 200) archive = zipfile.ZipFile(BytesIO(response.content)) namelist = archive.namelist() assert len(namelist) == 4, f"Expected 3 elements in [{namelist}]"
class JobsApiTestCase(ApiTestCase, TestsTools): def setUp(self): super().setUp() self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) @uses_test_history(require_new=True) def test_index(self, history_id): # Create HDA to ensure at least one job exists... self.__history_with_new_dataset(history_id) jobs = self.__jobs_index() assert "upload1" in map(itemgetter("tool_id"), jobs) @uses_test_history(require_new=True) def test_system_details_admin_only(self, history_id): self.__history_with_new_dataset(history_id) jobs = self.__jobs_index(admin=False) job = jobs[0] self._assert_not_has_keys(job, "external_id") jobs = self.__jobs_index(admin=True) job = jobs[0] self._assert_has_keys(job, "command_line", "external_id") @uses_test_history(require_new=True) def test_admin_job_list(self, history_id): self.__history_with_new_dataset(history_id) jobs_response = self._get("jobs?view=admin_job_list", admin=False) assert jobs_response.status_code == 403 assert jobs_response.json( )['err_msg'] == 'Only admins can use the admin_job_list view' jobs = self._get("jobs?view=admin_job_list", admin=True).json() job = jobs[0] self._assert_has_keys(job, "command_line", "external_id", 'handler') @uses_test_history(require_new=True) def test_index_state_filter(self, history_id): # Initial number of ok jobs original_count = len(self.__uploads_with_state("ok")) # Run through dataset upload to ensure num uplaods at least greater # by 1. self.__history_with_ok_dataset(history_id) # Verify number of ok jobs is actually greater. count_increased = False for _ in range(10): new_count = len(self.__uploads_with_state("ok")) if original_count < new_count: count_increased = True break time.sleep(.1) if not count_increased: template = "Jobs in ok state did not increase (was %d, now %d)" message = template % (original_count, new_count) raise AssertionError(message) @uses_test_history(require_new=True) def test_index_date_filter(self, history_id): self.__history_with_new_dataset(history_id) two_weeks_ago = (datetime.datetime.utcnow() - datetime.timedelta(14)).isoformat() last_week = (datetime.datetime.utcnow() - datetime.timedelta(7)).isoformat() next_week = (datetime.datetime.utcnow() + datetime.timedelta(7)).isoformat() today = datetime.datetime.utcnow().isoformat() tomorrow = (datetime.datetime.utcnow() + datetime.timedelta(1)).isoformat() jobs = self.__jobs_index(data={ "date_range_min": today[0:10], "date_range_max": tomorrow[0:10] }) assert len(jobs) > 0 today_job_id = jobs[0]["id"] jobs = self.__jobs_index(data={ "date_range_min": two_weeks_ago, "date_range_max": last_week }) assert today_job_id not in map(itemgetter("id"), jobs) jobs = self.__jobs_index(data={ "date_range_min": last_week, "date_range_max": next_week }) assert today_job_id in map(itemgetter("id"), jobs) @uses_test_history(require_new=True) def test_index_history(self, history_id): self.__history_with_new_dataset(history_id) jobs = self.__jobs_index(data={"history_id": history_id}) assert len(jobs) > 0 with self.dataset_populator.test_history() as other_history_id: jobs = self.__jobs_index(data={"history_id": other_history_id}) assert len(jobs) == 0 @uses_test_history(require_new=True) def test_index_workflow_and_invocation_filter(self, history_id): workflow_simple = """ class: GalaxyWorkflow name: Simple Workflow inputs: input1: data outputs: wf_output_1: outputSource: first_cat/out_file1 steps: first_cat: tool_id: cat1 in: input1: input1 """ summary = self.workflow_populator.run_workflow( workflow_simple, history_id=history_id, test_data={"input1": "hello world"}) invocation_id = summary.invocation_id workflow_id = self._get( f"invocations/{invocation_id}").json()['workflow_id'] self.workflow_populator.wait_for_invocation(workflow_id, invocation_id) jobs1 = self.__jobs_index(data={"workflow_id": workflow_id}) assert len(jobs1) == 1 jobs2 = self.__jobs_index(data={"invocation_id": invocation_id}) assert len(jobs2) == 1 assert jobs1 == jobs2 @uses_test_history(require_new=True) def test_index_workflow_filter_implicit_jobs(self, history_id): workflow_id = self.workflow_populator.upload_yaml_workflow(""" class: GalaxyWorkflow inputs: input_datasets: collection steps: multi_data_optional: tool_id: multi_data_optional in: input1: input_datasets """) hdca_id = self.dataset_collection_populator.create_list_of_list_in_history( history_id).json() self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { '0': self.dataset_populator.ds_entry(hdca_id), } invocation_id = self.workflow_populator.invoke_workflow_and_wait( workflow_id, history_id=history_id, inputs=inputs, assert_ok=True) jobs1 = self.__jobs_index(data={"workflow_id": workflow_id}) jobs2 = self.__jobs_index(data={"invocation_id": invocation_id}) assert len(jobs1) == len(jobs2) == 1 second_invocation_id = self.workflow_populator.invoke_workflow_and_wait( workflow_id, history_id=history_id, inputs=inputs, assert_ok=True) workflow_jobs = self.__jobs_index(data={"workflow_id": workflow_id}) second_invocation_jobs = self.__jobs_index( data={"invocation_id": second_invocation_id}) assert len(workflow_jobs) == 2 assert len(second_invocation_jobs) == 1 @uses_test_history(require_new=True) def test_index_limit_and_offset_filter(self, history_id): self.__history_with_new_dataset(history_id) jobs = self.__jobs_index(data={"history_id": history_id}) assert len(jobs) > 0 length = len(jobs) jobs = self.__jobs_index(data={"history_id": history_id, "offset": 1}) assert len(jobs) == length - 1 jobs = self.__jobs_index(data={"history_id": history_id, "limit": 0}) assert len(jobs) == 0 @uses_test_history(require_new=True) def test_index_user_filter(self, history_id): test_user_email = "*****@*****.**" user = self._setup_user(test_user_email) with self._different_user(email=test_user_email): # User should be able to jobs for their own ID. jobs = self.__jobs_index(data={"user_id": user["id"]}) assert jobs == [] # Admin should be able to see jobs of another user. jobs = self.__jobs_index(data={"user_id": user["id"]}, admin=True) assert jobs == [] # Normal user should not be able to see jobs of another user. jobs_response = self._get("jobs", data={"user_id": user["id"]}) self._assert_status_code_is(jobs_response, 403) assert jobs_response.json() == { "err_msg": "Only admins can index the jobs of others", "err_code": 403006 } @uses_test_history(require_new=True) def test_index_multiple_states_filter(self, history_id): # Initial number of ok jobs original_count = len(self.__uploads_with_state("ok", "new")) # Run through dataset upload to ensure num uplaods at least greater # by 1. self.__history_with_ok_dataset(history_id) # Verify number of ok jobs is actually greater. new_count = len(self.__uploads_with_state("new", "ok")) assert original_count < new_count, new_count @uses_test_history(require_new=True) def test_show(self, history_id): job_properties_tool_run = self.dataset_populator.run_tool( tool_id="job_properties", inputs={}, history_id=history_id, ) first_job = self.__jobs_index()[0] self._assert_has_key(first_job, 'id', 'state', 'exit_code', 'update_time', 'create_time') job_id = job_properties_tool_run["jobs"][0]["id"] show_jobs_response = self.dataset_populator.get_job_details(job_id) self._assert_status_code_is(show_jobs_response, 200) job_details = show_jobs_response.json() self._assert_has_key(job_details, 'id', 'state', 'exit_code', 'update_time', 'create_time') show_jobs_response = self.dataset_populator.get_job_details(job_id, full=True) self._assert_status_code_is(show_jobs_response, 200) job_details = show_jobs_response.json() self._assert_has_key( job_details, "create_time", "exit_code", "id", "job_messages", "job_stderr", "job_stdout", "state", "stderr", "stdout", "tool_stderr", "tool_stdout", "update_time", ) self.dataset_populator.wait_for_job(job_id, assert_ok=True) show_jobs_response = self.dataset_populator.get_job_details(job_id, full=True) job_details = show_jobs_response.json() assert "The bool is not true\n" not in job_details["job_stdout"] assert "The bool is very not true\n" not in job_details["job_stderr"] assert job_details["tool_stdout"] == "The bool is not true\n" assert job_details["tool_stderr"] == "The bool is very not true\n" assert "The bool is not true\n" in job_details["stdout"] assert "The bool is very not true\n" in job_details["stderr"] @uses_test_history(require_new=True) def test_show_security(self, history_id): self.__history_with_new_dataset(history_id) jobs_response = self._get("jobs", data={"history_id": history_id}) job = jobs_response.json()[0] job_id = job["id"] job_lock_response = self._get("job_lock", admin=True) job_lock_response.raise_for_status() assert not job_lock_response.json()["active"] show_jobs_response = self._get(f"jobs/{job_id}", admin=False) self._assert_not_has_keys(show_jobs_response.json(), "external_id") # TODO: Re-activate test case when API accepts privacy settings # with self._different_user(): # show_jobs_response = self._get( "jobs/%s" % job_id, admin=False ) # self._assert_status_code_is( show_jobs_response, 200 ) show_jobs_response = self._get(f"jobs/{job_id}", admin=True) self._assert_has_keys(show_jobs_response.json(), "command_line", "external_id") def _run_detect_errors(self, history_id, inputs): payload = self.dataset_populator.run_tool_payload( tool_id='detect_errors_aggressive', inputs=inputs, history_id=history_id, ) return self._post("tools", data=payload).json() @skip_without_tool("detect_errors_aggressive") def test_unhide_on_error(self): with self.dataset_populator.test_history() as history_id: inputs = {'error_bool': 'true'} run_response = self._run_detect_errors(history_id=history_id, inputs=inputs) job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) job = self.dataset_populator.get_job_details(job_id).json() assert job['state'] == 'error' dataset = self.dataset_populator.get_history_dataset_details( history_id=history_id, dataset_id=run_response['outputs'][0]['id'], assert_ok=False) assert dataset['visible'] def _run_map_over_error(self, history_id): hdca1 = self.dataset_collection_populator.create_list_in_history( history_id, contents=[("sample1-1", "1 2 3")]).json() inputs = { 'error_bool': 'true', 'dataset': { 'batch': True, 'values': [{ 'src': 'hdca', 'id': hdca1['id'] }], } } return self._run_detect_errors(history_id=history_id, inputs=inputs) @skip_without_tool("detect_errors_aggressive") def test_no_unhide_on_error_if_mapped_over(self): with self.dataset_populator.test_history() as history_id: run_response = self._run_map_over_error(history_id) job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) job = self.dataset_populator.get_job_details(job_id).json() assert job['state'] == 'error' dataset = self.dataset_populator.get_history_dataset_details( history_id=history_id, dataset_id=run_response['outputs'][0]['id'], assert_ok=False) assert not dataset['visible'] def test_no_hide_on_rerun(self): with self.dataset_populator.test_history() as history_id: run_response = self._run_map_over_error(history_id) job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) failed_hdca = self.dataset_populator.get_history_collection_details( history_id=history_id, content_id=run_response['implicit_collections'][0]['id'], assert_ok=False, ) first_update_time = failed_hdca['update_time'] assert failed_hdca['visible'] rerun_params = self._get(f"jobs/{job_id}/build_for_rerun").json() inputs = rerun_params['state_inputs'] inputs['rerun_remap_job_id'] = job_id rerun_response = self._run_detect_errors(history_id=history_id, inputs=inputs) rerun_job_id = rerun_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(rerun_job_id) # Verify source hdca is still visible hdca = self.dataset_populator.get_history_collection_details( history_id=history_id, content_id=run_response['implicit_collections'][0]['id'], assert_ok=False, ) assert hdca['visible'] assert isoparse( hdca['update_time']) > (isoparse(first_update_time)) @skip_without_tool('empty_output') def test_common_problems(self): with self.dataset_populator.test_history() as history_id: empty_run_response = self.dataset_populator.run_tool( tool_id='empty_output', inputs={}, history_id=history_id, ) empty_hda = empty_run_response["outputs"][0] cat_empty_twice_run_response = self.dataset_populator.run_tool( tool_id='cat1', inputs={ 'input1': { 'src': 'hda', 'id': empty_hda['id'] }, 'queries_0|input2': { 'src': 'hda', 'id': empty_hda['id'] } }, history_id=history_id, ) empty_output_job = empty_run_response["jobs"][0] cat_empty_job = cat_empty_twice_run_response["jobs"][0] empty_output_common_problems_response = self._get( f"jobs/{empty_output_job['id']}/common_problems").json() cat_empty_common_problems_response = self._get( f"jobs/{cat_empty_job['id']}/common_problems").json() self._assert_has_keys(empty_output_common_problems_response, "has_empty_inputs", "has_duplicate_inputs") self._assert_has_keys(cat_empty_common_problems_response, "has_empty_inputs", "has_duplicate_inputs") assert not empty_output_common_problems_response["has_empty_inputs"] assert cat_empty_common_problems_response["has_empty_inputs"] assert not empty_output_common_problems_response[ "has_duplicate_inputs"] assert cat_empty_common_problems_response["has_duplicate_inputs"] @skip_without_tool('detect_errors_aggressive') def test_report_error(self): with self.dataset_populator.test_history() as history_id: self._run_error_report(history_id) @skip_without_tool('detect_errors_aggressive') def test_report_error_anon(self): with self._different_user(anon=True): history_id = self._get( urllib.parse.urljoin( self.url, "history/current_history_json")).json()['id'] self._run_error_report(history_id) def _run_error_report(self, history_id): payload = self.dataset_populator.run_tool_payload( tool_id='detect_errors_aggressive', inputs={'error_bool': 'true'}, history_id=history_id, ) run_response = self._post("tools", data=payload).json() job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) dataset_id = run_response['outputs'][0]['id'] response = self._post(f'jobs/{job_id}/error', data={'dataset_id': dataset_id}) assert response.status_code == 200, response.text @skip_without_tool('detect_errors_aggressive') def test_report_error_bootstrap_admin(self): with self.dataset_populator.test_history() as history_id: payload = self.dataset_populator.run_tool_payload( tool_id='detect_errors_aggressive', inputs={'error_bool': 'true'}, history_id=history_id, ) run_response = self._post("tools", data=payload, key=self.master_api_key) self._assert_status_code_is(run_response, 400) @skip_without_tool("create_2") @uses_test_history(require_new=True) def test_deleting_output_keep_running_until_all_deleted(self, history_id): job_state, outputs = self._setup_running_two_output_job( history_id, 120) self._hack_to_skip_test_if_state_ok(job_state) # Delete one of the two outputs and make sure the job is still running. self._raw_update_history_item(history_id, outputs[0]["id"], {"deleted": True}) self._hack_to_skip_test_if_state_ok(job_state) time.sleep(1) self._hack_to_skip_test_if_state_ok(job_state) state = job_state().json()["state"] assert state == "running", state # Delete the second output and make sure the job is cancelled. self._raw_update_history_item(history_id, outputs[1]["id"], {"deleted": True}) final_state = wait_on_state(job_state, assert_ok=False, timeout=15) assert final_state in ["deleting", "deleted"], final_state @skip_without_tool("create_2") @uses_test_history(require_new=True) def test_purging_output_keep_running_until_all_purged(self, history_id): job_state, outputs = self._setup_running_two_output_job( history_id, 120) # Pretty much right away after the job is running, these paths should be populated - # if they are grab them and make sure they are deleted at the end of the job. dataset_1 = self._get_history_item_as_admin(history_id, outputs[0]["id"]) dataset_2 = self._get_history_item_as_admin(history_id, outputs[1]["id"]) if "file_name" in dataset_1: output_dataset_paths = [ dataset_1["file_name"], dataset_2["file_name"] ] # This may or may not exist depending on if the test is local or not. output_dataset_paths_exist = os.path.exists( output_dataset_paths[0]) else: output_dataset_paths = [] output_dataset_paths_exist = False self._hack_to_skip_test_if_state_ok(job_state) current_state = job_state().json()["state"] assert current_state == "running", current_state # Purge one of the two outputs and make sure the job is still running. self._raw_update_history_item(history_id, outputs[0]["id"], {"purged": True}) time.sleep(1) self._hack_to_skip_test_if_state_ok(job_state) current_state = job_state().json()["state"] assert current_state == "running", current_state # Purge the second output and make sure the job is cancelled. self._raw_update_history_item(history_id, outputs[1]["id"], {"purged": True}) final_state = wait_on_state(job_state, assert_ok=False, timeout=15) assert final_state in ["deleting", "deleted"], final_state def paths_deleted(): if not os.path.exists( output_dataset_paths[0]) and not os.path.exists( output_dataset_paths[1]): return True if output_dataset_paths_exist: wait_on(paths_deleted, "path deletion") @skip_without_tool("create_2") @uses_test_history(require_new=True) def test_purging_output_cleaned_after_ok_run(self, history_id): job_state, outputs = self._setup_running_two_output_job(history_id, 10) # Pretty much right away after the job is running, these paths should be populated - # if they are grab them and make sure they are deleted at the end of the job. dataset_1 = self._get_history_item_as_admin(history_id, outputs[0]["id"]) dataset_2 = self._get_history_item_as_admin(history_id, outputs[1]["id"]) if "file_name" in dataset_1: output_dataset_paths = [ dataset_1["file_name"], dataset_2["file_name"] ] # This may or may not exist depending on if the test is local or not. output_dataset_paths_exist = os.path.exists( output_dataset_paths[0]) else: output_dataset_paths = [] output_dataset_paths_exist = False if not output_dataset_paths_exist: # Given this Galaxy configuration - there is nothing more to be tested here. # Consider throwing a skip instead. return # Purge one of the two outputs and wait for the job to complete. self._raw_update_history_item(history_id, outputs[0]["id"], {"purged": True}) wait_on_state(job_state, assert_ok=True) if output_dataset_paths_exist: time.sleep(.5) # Make sure the non-purged dataset is on disk and the purged one is not. assert os.path.exists(output_dataset_paths[1]) assert not os.path.exists(output_dataset_paths[0]) def _hack_to_skip_test_if_state_ok(self, job_state): from nose.plugins.skip import SkipTest if job_state().json()["state"] == "ok": message = "Job state switch from running to ok too quickly - the rest of the test requires the job to be in a running state. Skipping test." raise SkipTest(message) def _setup_running_two_output_job(self, history_id, sleep_time): payload = self.dataset_populator.run_tool_payload( tool_id='create_2', inputs=dict(sleep_time=sleep_time, ), history_id=history_id, ) run_response = self._post("tools", data=payload) run_response.raise_for_status() run_object = run_response.json() outputs = run_object["outputs"] jobs = run_object["jobs"] assert len(outputs) == 2 assert len(jobs) == 1 def job_state(): jobs_response = self._get(f"jobs/{jobs[0]['id']}") return jobs_response # Give job some time to get up and running. time.sleep(2) running_state = wait_on_state(job_state, skip_states=["queued", "new"], assert_ok=False, timeout=15) assert running_state == "running", running_state return job_state, outputs def _raw_update_history_item(self, history_id, item_id, data): update_url = self._api_url( f"histories/{history_id}/contents/{item_id}", use_key=True) update_response = requests.put(update_url, json=data) assert_status_code_is_ok(update_response) return update_response @skip_without_tool("cat_data_and_sleep") @uses_test_history(require_new=True) def test_resume_job(self, history_id): hda1 = self.dataset_populator.new_dataset( history_id, content="samp1\t10.0\nsamp2\t20.0\n") hda2 = self.dataset_populator.new_dataset( history_id, content="samp1\t30.0\nsamp2\t40.0\n") # Submit first job payload = self.dataset_populator.run_tool_payload( tool_id='cat_data_and_sleep', inputs={ 'sleep_time': 15, 'input1': { 'src': 'hda', 'id': hda2['id'] }, 'queries_0|input2': { 'src': 'hda', 'id': hda2['id'] } }, history_id=history_id, ) run_response = self._post("tools", data=payload).json() output = run_response["outputs"][0] # Submit second job that waits on job1 payload = self.dataset_populator.run_tool_payload( tool_id='cat1', inputs={ 'input1': { 'src': 'hda', 'id': hda1['id'] }, 'queries_0|input2': { 'src': 'hda', 'id': output['id'] } }, history_id=history_id, ) run_response = self._post("tools", data=payload).json() job_id = run_response['jobs'][0]['id'] output = run_response["outputs"][0] # Delete second jobs input while second job is waiting for first job delete_response = self._delete( f"histories/{history_id}/contents/{hda1['id']}") self._assert_status_code_is(delete_response, 200) self.dataset_populator.wait_for_history_jobs(history_id, assert_ok=False) dataset_details = self._get( f"histories/{history_id}/contents/{output['id']}").json() assert dataset_details['state'] == 'paused' # Undelete input dataset undelete_response = self._put( f"histories/{history_id}/contents/{hda1['id']}", data={'deleted': False}, json=True) self._assert_status_code_is(undelete_response, 200) resume_response = self._put(f"jobs/{job_id}/resume") self._assert_status_code_is(resume_response, 200) self.dataset_populator.wait_for_history_jobs(history_id, assert_ok=True) dataset_details = self._get( f"histories/{history_id}/contents/{output['id']}").json() assert dataset_details['state'] == 'ok' def _get_history_item_as_admin(self, history_id, item_id): response = self._get( f"histories/{history_id}/contents/{item_id}?view=detailed", admin=True) assert_status_code_is_ok(response) return response.json() @uses_test_history(require_new=True) def test_search(self, history_id): dataset_id = self.__history_with_ok_dataset(history_id) # We first copy the datasets, so that the update time is lower than the job creation time new_history_id = self.dataset_populator.new_history() copy_payload = { "content": dataset_id, "source": "hda", "type": "dataset" } copy_response = self._post(f"histories/{new_history_id}/contents", data=copy_payload, json=True) self._assert_status_code_is(copy_response, 200) inputs = json.dumps({'input1': {'src': 'hda', 'id': dataset_id}}) self._job_search(tool_id='cat1', history_id=history_id, inputs=inputs) # We test that a job can be found even if the dataset has been copied to another history new_dataset_id = copy_response.json()['id'] copied_inputs = json.dumps( {'input1': { 'src': 'hda', 'id': new_dataset_id }}) search_payload = self._search_payload(history_id=history_id, tool_id='cat1', inputs=copied_inputs) self._search(search_payload, expected_search_count=1) # Now we delete the original input HDA that was used -- we should still be able to find the job delete_respone = self._delete( f"histories/{history_id}/contents/{dataset_id}") self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=1) # Now we also delete the copy -- we shouldn't find a job delete_respone = self._delete( f"histories/{new_history_id}/contents/{new_dataset_id}") self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_handle_identifiers(self, history_id): # Test that input name and element identifier of a jobs' output must match for a job to be returned. dataset_id = self.__history_with_ok_dataset(history_id) inputs = json.dumps({'input1': {'src': 'hda', 'id': dataset_id}}) self._job_search(tool_id='identifier_single', history_id=history_id, inputs=inputs) dataset_details = self._get( f"histories/{history_id}/contents/{dataset_id}").json() dataset_details['name'] = 'Renamed Test Dataset' dataset_update_response = self._put( f"histories/{history_id}/contents/{dataset_id}", data=dict(name='Renamed Test Dataset'), json=True) self._assert_status_code_is(dataset_update_response, 200) assert dataset_update_response.json()['name'] == 'Renamed Test Dataset' search_payload = self._search_payload(history_id=history_id, tool_id='identifier_single', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_delete_outputs(self, history_id): dataset_id = self.__history_with_ok_dataset(history_id) inputs = json.dumps({'input1': {'src': 'hda', 'id': dataset_id}}) tool_response = self._job_search(tool_id='cat1', history_id=history_id, inputs=inputs) output_id = tool_response.json()['outputs'][0]['id'] delete_respone = self._delete( f"histories/{history_id}/contents/{output_id}") self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload(history_id=history_id, tool_id='cat1', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_with_hdca_list_input(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type='list', history_id=history_id) list_id_b = self.__history_with_ok_collection(collection_type='list', history_id=history_id) inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': list_id_a }, 'f2': { 'src': 'hdca', 'id': list_id_b }, }) tool_response = self._job_search(tool_id='multi_data_param', history_id=history_id, inputs=inputs) # We switch the inputs, this should not return a match inputs_switched = json.dumps({ 'f2': { 'src': 'hdca', 'id': list_id_a }, 'f1': { 'src': 'hdca', 'id': list_id_b }, }) search_payload = self._search_payload(history_id=history_id, tool_id='multi_data_param', inputs=inputs_switched) self._search(search_payload, expected_search_count=0) # We delete the ouput (this is a HDA, as multi_data_param reduces collections) # and use the correct input job definition, the job should not be found output_id = tool_response.json()['outputs'][0]['id'] delete_respone = self._delete( f"histories/{history_id}/contents/{output_id}") self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload(history_id=history_id, tool_id='multi_data_param', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_delete_hdca_output(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type='list', history_id=history_id) inputs = json.dumps({ 'input1': { 'src': 'hdca', 'id': list_id_a }, }) tool_response = self._job_search(tool_id='collection_creates_list', history_id=history_id, inputs=inputs) output_id = tool_response.json()['outputs'][0]['id'] # We delete a single tool output, no job should be returned delete_respone = self._delete( f"histories/{history_id}/contents/{output_id}") self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload( history_id=history_id, tool_id='collection_creates_list', inputs=inputs) self._search(search_payload, expected_search_count=0) tool_response = self._job_search(tool_id='collection_creates_list', history_id=history_id, inputs=inputs) output_collection_id = tool_response.json( )['output_collections'][0]['id'] # We delete a collection output, no job should be returned delete_respone = self._delete( f"histories/{history_id}/contents/dataset_collections/{output_collection_id}" ) self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload( history_id=history_id, tool_id='collection_creates_list', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_with_hdca_pair_input(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type='pair', history_id=history_id) inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': list_id_a }, 'f2': { 'src': 'hdca', 'id': list_id_a }, }) self._job_search(tool_id='multi_data_param', history_id=history_id, inputs=inputs) # We test that a job can be found even if the collection has been copied to another history new_history_id = self.dataset_populator.new_history() copy_payload = { "content": list_id_a, "source": "hdca", "type": "dataset_collection" } copy_response = self._post(f"histories/{new_history_id}/contents", data=copy_payload, json=True) self._assert_status_code_is(copy_response, 200) new_list_a = copy_response.json()['id'] copied_inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': new_list_a }, 'f2': { 'src': 'hdca', 'id': new_list_a }, }) search_payload = self._search_payload(history_id=new_history_id, tool_id='multi_data_param', inputs=copied_inputs) self._search(search_payload, expected_search_count=1) # Now we delete the original input HDCA that was used -- we should still be able to find the job delete_respone = self._delete( f"histories/{history_id}/contents/dataset_collections/{list_id_a}") self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=1) # Now we also delete the copy -- we shouldn't find a job delete_respone = self._delete( f"histories/{history_id}/contents/dataset_collections/{new_list_a}" ) self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_with_hdca_list_pair_input(self, history_id): list_id_a = self.__history_with_ok_collection( collection_type='list:pair', history_id=history_id) inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': list_id_a }, 'f2': { 'src': 'hdca', 'id': list_id_a }, }) self._job_search(tool_id='multi_data_param', history_id=history_id, inputs=inputs) @uses_test_history(require_new=True) def test_search_with_hdca_list_pair_collection_mapped_over_pair_input( self, history_id): list_id_a = self.__history_with_ok_collection( collection_type='list:pair', history_id=history_id) inputs = json.dumps({ 'f1': { 'batch': True, 'values': [{ 'src': 'hdca', 'id': list_id_a, 'map_over_type': 'paired' }] }, }) self._job_search(tool_id='collection_paired_test', history_id=history_id, inputs=inputs) def _get_simple_rerun_params(self, history_id, private=False): list_id_a = self.__history_with_ok_collection( collection_type='list:pair', history_id=history_id) inputs = { 'f1': { 'batch': True, 'values': [{ 'src': 'hdca', 'id': list_id_a, 'map_over_type': 'paired' }] } } run_response = self._run( history_id=history_id, tool_id="collection_paired_test", inputs=inputs, wait_for_job=True, assert_ok=True, ) rerun_params = self._get( f"jobs/{run_response['jobs'][0]['id']}/build_for_rerun").json() # Since we call rerun on the first (and only) job we should get the expanded input # which is a dataset collection element (and not the list:pair hdca that was used as input to the original # job). assert rerun_params['state_inputs']['f1']['values'][0]['src'] == 'dce' if private: hdca = self.dataset_populator.get_history_collection_details( history_id=history_id, content_id=list_id_a) for element in hdca['elements'][0]['object']['elements']: self.dataset_populator.make_private(history_id, element['object']['id']) return rerun_params @skip_without_tool("collection_paired_test") @uses_test_history(require_new=False) def test_job_build_for_rerun(self, history_id): rerun_params = self._get_simple_rerun_params(history_id) self._run( history_id=history_id, tool_id="collection_paired_test", inputs=rerun_params['state_inputs'], wait_for_job=True, assert_ok=True, ) @skip_without_tool("collection_paired_test") @uses_test_history(require_new=False) def test_dce_submission_security(self, history_id): rerun_params = self._get_simple_rerun_params(history_id, private=True) with self._different_user(): other_history_id = self.dataset_populator.new_history() response = self._run( history_id=other_history_id, tool_id="collection_paired_test", inputs=rerun_params['state_inputs'], wait_for_job=False, assert_ok=False, ) assert response.status_code == 403 @skip_without_tool("identifier_collection") @uses_test_history(require_new=False) def test_job_build_for_rerun_list_list(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type='list', history_id=history_id) list_id_b = self.__history_with_ok_collection(collection_type='list', history_id=history_id) list_list = self.dataset_collection_populator.create_nested_collection( history_id=history_id, collection_type='list:list', name='list list collection', collection=[list_id_a, list_id_b]).json() list_list_id = list_list['id'] first_element = list_list['elements'][0] assert first_element['element_type'] == 'dataset_collection' assert first_element['element_identifier'] == 'test0' assert first_element['model_class'] == 'DatasetCollectionElement' inputs = { 'input1': { 'batch': True, 'values': [{ 'src': 'hdca', 'id': list_list_id, 'map_over_type': 'list' }] } } run_response = self._run( history_id=history_id, tool_id="identifier_collection", inputs=inputs, wait_for_job=True, assert_ok=True, ) assert len(run_response['jobs']) == 2 rerun_params = self._get( f"jobs/{run_response['jobs'][0]['id']}/build_for_rerun").json() # Since we call rerun on the first (and only) job we should get the expanded input # which is a dataset collection element (and not the list:list hdca that was used as input to the original # job). assert rerun_params['state_inputs']['input1']['values'][0][ 'src'] == 'dce' rerun_response = self._run( history_id=history_id, tool_id="identifier_collection", inputs=rerun_params['state_inputs'], wait_for_job=True, assert_ok=True, ) assert len(rerun_response['jobs']) == 1 rerun_content = self.dataset_populator.get_history_dataset_content( history_id=history_id, dataset=rerun_response['outputs'][0]) run_content = self.dataset_populator.get_history_dataset_content( history_id=history_id, dataset=run_response['outputs'][0]) assert rerun_content == run_content def _job_search(self, tool_id, history_id, inputs): search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) empty_search_response = self._post("jobs/search", data=search_payload) self._assert_status_code_is(empty_search_response, 200) self.assertEqual(len(empty_search_response.json()), 0) tool_response = self._post("tools", data=search_payload) self.dataset_populator.wait_for_tool_run(history_id, run_response=tool_response) self._search(search_payload, expected_search_count=1) return tool_response def _search_payload(self, history_id, tool_id, inputs, state='ok'): search_payload = dict(tool_id=tool_id, inputs=inputs, history_id=history_id, state=state) return search_payload def _search(self, payload, expected_search_count=1): # in case job and history aren't updated at exactly the same # time give time to wait for _ in range(5): search_count = self._search_count(payload) if search_count == expected_search_count: break time.sleep(1) assert search_count == expected_search_count, "expected to find %d jobs, got %d jobs" % ( expected_search_count, search_count) return search_count def _search_count(self, search_payload): search_response = self._post("jobs/search", data=search_payload) self._assert_status_code_is(search_response, 200) search_json = search_response.json() return len(search_json) def __uploads_with_state(self, *states): jobs_response = self._get("jobs", data=dict(state=states)) self._assert_status_code_is(jobs_response, 200) jobs = jobs_response.json() assert not [j for j in jobs if not j['state'] in states] return [j for j in jobs if j['tool_id'] == 'upload1'] def __history_with_new_dataset(self, history_id): dataset_id = self.dataset_populator.new_dataset(history_id)["id"] return dataset_id def __history_with_ok_dataset(self, history_id): dataset_id = self.dataset_populator.new_dataset(history_id, wait=True)["id"] return dataset_id def __history_with_ok_collection(self, collection_type='list', history_id=None): if not history_id: history_id = self.dataset_populator.new_history() if collection_type == 'list': fetch_response = self.dataset_collection_populator.create_list_in_history( history_id, direct_upload=True).json() elif collection_type == 'pair': fetch_response = self.dataset_collection_populator.create_pair_in_history( history_id, direct_upload=True).json() elif collection_type == 'list:pair': fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history( history_id).json() self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) return fetch_response["outputs"][0]['id'] def __jobs_index(self, **kwds): jobs_response = self._get("jobs", **kwds) self._assert_status_code_is(jobs_response, 200) jobs = jobs_response.json() assert isinstance(jobs, list) return jobs
class DatasetsApiTestCase(ApiTestCase): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_index(self): index_response = self._get("datasets") self._assert_status_code_is(index_response, 200) def test_search_datasets(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] payload = {'limit': 1, 'offset': 0} index_response = self._get("datasets", payload).json() assert len(index_response) == 1 assert index_response[0]['id'] == hda_id hdca_id = self.dataset_collection_populator.create_list_in_history( self.history_id, contents=["1\n2\n3"]).json()['id'] payload = {'limit': 3, 'offset': 0} index_response = self._get("datasets", payload).json() assert len(index_response) == 3 assert index_response[0]['id'] == hdca_id assert index_response[0][ 'history_content_type'] == 'dataset_collection' assert index_response[2]['id'] == hda_id assert index_response[2]['history_content_type'] == 'dataset' payload = { 'limit': 2, 'offset': 0, 'q': ['history_content_type'], 'qv': ['dataset'] } index_response = self._get("datasets", payload).json() assert index_response[1]['id'] == hda_id def test_search_by_tag(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] update_payload = { 'tags': ['cool:new_tag', 'cool:another_tag'], } updated_hda = self._put( f"histories/{self.history_id}/contents/{hda_id}", update_payload).json() assert 'cool:new_tag' in updated_hda['tags'] assert 'cool:another_tag' in updated_hda['tags'] payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag'], 'qv': ['dataset', 'cool:new_tag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 1 payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-contains'], 'qv': ['dataset', 'new_tag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 1 payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-contains'], 'qv': ['dataset', 'notag'] } index_response = self._get("datasets", payload).json() assert len(index_response) == 0 def test_search_by_tool_id(self): self.dataset_populator.new_dataset(self.history_id) payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset', 'upload1'] } assert len(self._get("datasets", payload).json()) == 1 payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset', 'uploadX'] } assert len(self._get("datasets", payload).json()) == 0 payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id-contains'], 'qv': ['dataset', 'pload1'] } assert len(self._get("datasets", payload).json()) == 1 self.dataset_collection_populator.create_list_in_history( self.history_id, name="search by tool id", contents=["1\n2\n3"]).json() self.dataset_populator.wait_for_history(self.history_id) payload = { 'limit': 10, 'offset': 0, 'history_id': self.history_id, 'q': ['name', 'tool_id'], 'qv': ['search by tool id', 'upload1'] } result = self._get("datasets", payload).json() assert result[0]['name'] == 'search by tool id', result payload = { 'limit': 1, 'offset': 0, 'q': ['history_content_type', 'tool_id'], 'qv': ['dataset_collection', 'uploadX'] } result = self._get("datasets", payload).json() assert len(result) == 0 def test_invalid_search(self): payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag-invalid_op'], 'qv': ['dataset', 'notag'] } index_response = self._get("datasets", payload) self._assert_status_code_is(index_response, 400) assert index_response.json()['err_msg'] == 'bad op in filter' def test_search_returns_only_accessible(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] with self._different_user(): payload = { 'limit': 10, 'offset': 0, 'q': ['history_content_type'], 'qv': ['dataset'] } index_response = self._get("datasets", payload).json() for item in index_response: assert hda_id != item['id'] def test_show(self): hda1 = self.dataset_populator.new_dataset(self.history_id) show_response = self._get("datasets/%s" % (hda1["id"])) self._assert_status_code_is(show_response, 200) self.__assert_matches_hda(hda1, show_response.json()) def __assert_matches_hda(self, input_hda, query_hda): self._assert_has_keys(query_hda, "id", "name") assert input_hda["name"] == query_hda["name"] assert input_hda["id"] == query_hda["id"] def test_display(self): contents = textwrap.dedent("""\ 1 2 3 4 A B C D 10 20 30 40 """) hda1 = self.dataset_populator.new_dataset(self.history_id, content=contents) self.dataset_populator.wait_for_history(self.history_id) display_response = self._get( "histories/{}/contents/{}/display".format(self.history_id, hda1["id"]), {'raw': 'True'}) self._assert_status_code_is(display_response, 200) assert display_response.text == contents def test_tag_change(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] payload = { 'item_id': hda_id, 'item_class': 'HistoryDatasetAssociation', 'item_tags': ['cool:tag_a', 'cool:tag_b', 'tag_c', 'name:tag_d', '#tag_e'], } self._put("tags", payload).json() updated_hda = self._get( f"histories/{self.history_id}/contents/{hda_id}").json() assert 'cool:tag_a' in updated_hda['tags'] assert 'cool:tag_b' in updated_hda['tags'] assert 'tag_c' in updated_hda['tags'] assert 'name:tag_d' in updated_hda['tags'] assert 'name:tag_e' in updated_hda['tags'] @skip_without_tool("cat_data_and_sleep") def test_update_datatype(self): hda_id = self.dataset_populator.new_dataset(self.history_id)['id'] original_hda = self._get( f"histories/{self.history_id}/contents/{hda_id}").json() assert original_hda['extension'] == 'txt' assert original_hda['data_type'] == 'galaxy.datatypes.data.Text' assert 'scatterplot' not in [ viz['name'] for viz in original_hda['visualizations'] ] inputs = { 'input1': { 'src': 'hda', 'id': hda_id }, 'sleep_time': 10, } run_response = self.dataset_populator.run_tool( "cat_data_and_sleep", inputs, self.history_id, assert_ok=False, ) queued_id = run_response.json()["outputs"][0]["id"] update_while_incomplete_response = self._put( # try updating datatype while used as output of a running job f"histories/{self.history_id}/contents/{queued_id}", {'datatype': 'tabular'}) self._assert_status_code_is(update_while_incomplete_response, 400) self.dataset_populator.wait_for_history_jobs( self.history_id) # now wait for upload to complete successful_updated_hda_response = self._put( f"histories/{self.history_id}/contents/{hda_id}", { 'datatype': 'tabular' }).json() assert successful_updated_hda_response['extension'] == 'tabular' assert successful_updated_hda_response[ 'data_type'] == 'galaxy.datatypes.tabular.Tabular' assert 'scatterplot' in [ viz['name'] for viz in successful_updated_hda_response['visualizations'] ] invalidly_updated_hda_response = self._put( # try updating with invalid datatype f"histories/{self.history_id}/contents/{hda_id}", {'datatype': 'invalid'}) self._assert_status_code_is(invalidly_updated_hda_response, 400)
def setUp(self): super(DatasetsApiTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.history_id = self.dataset_populator.new_history()
def _set_up_populators(self): self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
class HistoriesApiTestCase(ApiTestCase, BaseHistories): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) def test_create_history(self): # Create a history. create_response = self._create_history("TestHistory1") created_id = create_response["id"] # Make sure new history appears in index of user's histories. index_response = self._get("histories").json() indexed_history = [h for h in index_response if h["id"] == created_id][0] self.assertEqual(indexed_history["name"], "TestHistory1") def test_create_history_json(self): name = "TestHistoryJson" post_data = dict(name=name) create_response = self._post("histories", data=post_data, json=True).json() self._assert_has_keys(create_response, "name", "id") self.assertEqual(create_response["name"], name) return create_response def test_show_history(self): history_id = self._create_history("TestHistoryForShow")["id"] show_response = self._show(history_id) self._assert_has_key( show_response, 'id', 'name', 'annotation', 'size', 'contents_url', 'state', 'state_details', 'state_ids' ) state_details = show_response["state_details"] state_ids = show_response["state_ids"] states = [ 'discarded', 'empty', 'error', 'failed_metadata', 'new', 'ok', 'paused', 'queued', 'running', 'setting_metadata', 'upload' ] assert isinstance(state_details, dict) assert isinstance(state_ids, dict) self._assert_has_keys(state_details, *states) self._assert_has_keys(state_ids, *states) def test_show_most_recently_used(self): history_id = self._create_history("TestHistoryRecent")["id"] show_response = self._get("histories/most_recently_used").json() assert show_response["id"] == history_id def test_index_order(self): slightly_older_history_id = self._create_history("TestHistorySlightlyOlder")["id"] newer_history_id = self._create_history("TestHistoryNewer")["id"] index_response = self._get("histories").json() assert index_response[0]["id"] == newer_history_id assert index_response[1]["id"] == slightly_older_history_id def test_delete(self): # Setup a history and ensure it is in the index history_id = self._create_history("TestHistoryForDelete")["id"] index_response = self._get("histories").json() assert index_response[0]["id"] == history_id show_response = self._show(history_id) assert not show_response["deleted"] # Delete the history self._delete(f"histories/{history_id}") # Check can view it - but it is deleted show_response = self._show(history_id) assert show_response["deleted"] # Verify it is dropped from history index index_response = self._get("histories").json() assert len(index_response) == 0 or index_response[0]["id"] != history_id # Add deleted filter to index to view it index_response = self._get("histories", {"deleted": "true"}).json() assert index_response[0]["id"] == history_id def test_purge(self): history_id = self._create_history("TestHistoryForPurge")["id"] data = {'purge': True} self._delete(f"histories/{history_id}", data=data, json=True) show_response = self._show(history_id) assert show_response["deleted"] assert show_response["purged"] def test_undelete(self): history_id = self._create_history("TestHistoryForDeleteAndUndelete")["id"] self._delete(f"histories/{history_id}") self._post(f"histories/deleted/{history_id}/undelete") show_response = self._show(history_id) assert not show_response["deleted"] def test_update(self): history_id = self._create_history("TestHistoryForUpdating")["id"] self._update(history_id, {"name": "New Name"}) show_response = self._show(history_id) assert show_response["name"] == "New Name" unicode_name = '桜ゲノム' self._update(history_id, {"name": unicode_name}) show_response = self._show(history_id) assert show_response["name"] == unicode_name, show_response quoted_name = "'MooCow'" self._update(history_id, {"name": quoted_name}) show_response = self._show(history_id) assert show_response["name"] == quoted_name self._update(history_id, {"deleted": True}) show_response = self._show(history_id) assert show_response["deleted"], show_response self._update(history_id, {"deleted": False}) show_response = self._show(history_id) assert not show_response["deleted"] self._update(history_id, {"published": True}) show_response = self._show(history_id) assert show_response["published"] self._update(history_id, {"genome_build": "hg18"}) show_response = self._show(history_id) assert show_response["genome_build"] == "hg18" self._update(history_id, {"annotation": "The annotation is cool"}) show_response = self._show(history_id) assert show_response["annotation"] == "The annotation is cool" self._update(history_id, {"annotation": unicode_name}) show_response = self._show(history_id) assert show_response["annotation"] == unicode_name, show_response self._update(history_id, {"annotation": quoted_name}) show_response = self._show(history_id) assert show_response["annotation"] == quoted_name def test_update_invalid_attribute(self): history_id = self._create_history("TestHistoryForInvalidUpdating")["id"] put_response = self._update(history_id, {"invalidkey": "moo"}) assert "invalidkey" not in put_response.json() def test_update_invalid_types(self): history_id = self._create_history("TestHistoryForUpdatingInvalidTypes")["id"] for str_key in ["name", "annotation"]: assert self._update(history_id, {str_key: False}).status_code == 400 for bool_key in ['deleted', 'importable', 'published']: assert self._update(history_id, {bool_key: "a string"}).status_code == 400 assert self._update(history_id, {"tags": "a simple string"}).status_code == 400 assert self._update(history_id, {"tags": [True]}).status_code == 400 def test_invalid_keys(self): invalid_history_id = "1234123412341234" assert self._get(f"histories/{invalid_history_id}").status_code == 400 assert self._update(invalid_history_id, {"name": "new name"}).status_code == 400 assert self._delete(f"histories/{invalid_history_id}").status_code == 400 assert self._post(f"histories/deleted/{invalid_history_id}/undelete").status_code == 400 def test_create_anonymous_fails(self): post_data = dict(name="CannotCreate") create_response = self._post("histories", data=post_data, anon=True) self._assert_status_code_is(create_response, 403) def test_create_without_session_fails(self): post_data = dict(name="SessionNeeded") # Using admin=True will boostrap an Admin user without session create_response = self._post("histories", data=post_data, admin=True, json=True) self._assert_status_code_is(create_response, 400) def test_create_tag(self): post_data = dict(name="TestHistoryForTag") history_id = self._post("histories", data=post_data, json=True).json()["id"] tag_data = dict(value="awesometagvalue") tag_url = f"histories/{history_id}/tags/awesometagname" tag_create_response = self._post(tag_url, data=tag_data, json=True) self._assert_status_code_is(tag_create_response, 200) def test_copy_history(self): history_id = self.dataset_populator.new_history() fetch_response = self.dataset_collection_populator.create_list_in_history(history_id, contents=["Hello", "World"], direct_upload=True) dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response.json()) copied_history_response = self.dataset_populator.copy_history(history_id) copied_history_response.raise_for_status() copied_history = copied_history_response.json() copied_collection = self.dataset_populator.get_history_collection_details(history_id=copied_history['id'], history_content_type="dataset_collection") assert dataset_collection['name'] == copied_collection['name'] assert dataset_collection['id'] != copied_collection['id'] assert len(dataset_collection['elements']) == len(copied_collection['elements']) == 2 source_element = dataset_collection['elements'][0] copied_element = copied_collection['elements'][0] assert source_element['element_identifier'] == copied_element['element_identifier'] == 'data0' assert source_element['id'] != copied_element['id'] source_hda = source_element['object'] copied_hda = copied_element['object'] assert source_hda['name'] == copied_hda['name'] == 'data0' assert source_hda['id'] != copied_hda['id'] assert source_hda['history_id'] != copied_hda['history_id'] assert source_hda['hid'] == copied_hda['hid'] == 2
class HistoryContentsApiTestCase(ApiTestCase): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_index_hda_summary(self): hda1 = self.dataset_populator.new_dataset(self.history_id) contents_response = self._get("histories/%s/contents" % self.history_id) hda_summary = self.__check_for_hda(contents_response, hda1) assert "display_types" not in hda_summary # Quick summary, not full details def test_make_private_and_public(self): hda1 = self._wait_for_new_hda() update_url = "histories/{}/contents/{}/permissions".format( self.history_id, hda1["id"]) role_id = self.dataset_populator.user_private_role_id() # Give manage permission to the user. payload = { "access": [], "manage": [role_id], } update_response = self._update_permissions(update_url, payload, admin=True) self._assert_status_code_is(update_response, 200) self._assert_other_user_can_access(hda1["id"]) # Then we restrict access. payload = { "action": "make_private", } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) self._assert_other_user_cannot_access(hda1["id"]) # Then we restrict access. payload = { "action": "remove_restrictions", } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) self._assert_other_user_can_access(hda1["id"]) def test_set_permissions_add_admin_history_contents(self): self._verify_dataset_permissions("history_contents") def test_set_permissions_add_admin_datasets(self): self._verify_dataset_permissions("dataset") def _verify_dataset_permissions(self, api_endpoint): hda1 = self._wait_for_new_hda() hda_id = hda1["id"] if api_endpoint == "history_contents": update_url = f"histories/{self.history_id}/contents/{hda_id}/permissions" else: update_url = "datasets/%s/permissions" % hda_id role_id = self.dataset_populator.user_private_role_id() payload = { "access": [role_id], "manage": [role_id], } # Other users cannot modify permissions. with self._different_user(): update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 403) # First the details render for another user. self._assert_other_user_can_access(hda_id) # Then we restrict access. update_response = self._update_permissions(update_url, payload, admin=True) self._assert_status_code_is(update_response, 200) # Finally the details don't render. self._assert_other_user_cannot_access(hda_id) # But they do for the original user. contents_response = self._get( f"histories/{self.history_id}/contents/{hda_id}").json() assert "name" in contents_response update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) payload = { "access": [role_id], "manage": [role_id], } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) self._assert_other_user_cannot_access(hda_id) user_id = self.dataset_populator.user_id() with self._different_user(): different_user_id = self.dataset_populator.user_id() combined_user_role = self.dataset_populator.create_role( [user_id, different_user_id], description="role for testing permissions") payload = { "access": [combined_user_role["id"]], "manage": [role_id], } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) # Now other user can see dataset again with access permission. self._assert_other_user_can_access(hda_id) # access doesn't imply management though... with self._different_user(): update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 403) def _assert_other_user_cannot_access(self, history_content_id): with self._different_user(): contents_response = self.dataset_populator.get_history_dataset_details_raw( history_id=self.history_id, dataset_id=history_content_id) assert contents_response.status_code == 403 def _assert_other_user_can_access(self, history_content_id): with self._different_user(): contents_response = self.dataset_populator.get_history_dataset_details_raw( history_id=self.history_id, dataset_id=history_content_id) contents_response.raise_for_status() assert "name" in contents_response.json() def test_index_hda_all_details(self): hda1 = self.dataset_populator.new_dataset(self.history_id) contents_response = self._get("histories/%s/contents?details=all" % self.history_id) hda_details = self.__check_for_hda(contents_response, hda1) self.__assert_hda_has_full_details(hda_details) def test_index_hda_detail_by_id(self): hda1 = self.dataset_populator.new_dataset(self.history_id) contents_response = self._get( "histories/{}/contents?details={}".format(self.history_id, hda1["id"])) hda_details = self.__check_for_hda(contents_response, hda1) self.__assert_hda_has_full_details(hda_details) def test_show_hda(self): hda1 = self.dataset_populator.new_dataset(self.history_id) show_response = self.__show(hda1) self._assert_status_code_is(show_response, 200) self.__assert_matches_hda(hda1, show_response.json()) def test_hda_copy(self): hda1 = self.dataset_populator.new_dataset(self.history_id) create_data = dict( source='hda', content=hda1["id"], ) second_history_id = self.dataset_populator.new_history() assert self.__count_contents(second_history_id) == 0 create_response = self._post( "histories/%s/contents" % second_history_id, create_data) self._assert_status_code_is(create_response, 200) assert self.__count_contents(second_history_id) == 1 def test_library_copy(self): ld = self.library_populator.new_library_dataset("lda_test_library") create_data = dict( source='library', content=ld["id"], ) assert self.__count_contents(self.history_id) == 0 create_response = self._post("histories/%s/contents" % self.history_id, create_data) self._assert_status_code_is(create_response, 200) assert self.__count_contents(self.history_id) == 1 def test_update(self): hda1 = self._wait_for_new_hda() assert str(hda1["deleted"]).lower() == "false" update_response = self._raw_update(hda1["id"], dict(deleted=True)) self._assert_status_code_is(update_response, 200) show_response = self.__show(hda1) assert str(show_response.json()["deleted"]).lower() == "true" update_response = self._raw_update(hda1["id"], dict(name="Updated Name")) assert self.__show(hda1).json()["name"] == "Updated Name" update_response = self._raw_update(hda1["id"], dict(name="Updated Name")) assert self.__show(hda1).json()["name"] == "Updated Name" unicode_name = 'ржевский сапоги' update_response = self._raw_update(hda1["id"], dict(name=unicode_name)) updated_hda = self.__show(hda1).json() assert updated_hda["name"] == unicode_name, updated_hda quoted_name = '"Mooo"' update_response = self._raw_update(hda1["id"], dict(name=quoted_name)) updated_hda = self.__show(hda1).json() assert updated_hda["name"] == quoted_name, quoted_name data = { "dataset_id": hda1["id"], "name": "moocow", "dbkey": "?", "annotation": None, "info": "my info is", "operation": "attributes" } update_response = self._set_edit_update(data) # No key or anything supplied, expect a permission problem. # A bit questionable but I think this is a 400 instead of a 403 so that # we don't distinguish between this is a valid ID you don't have access to # and this is an invalid ID. assert update_response.status_code == 400, update_response.content def test_update_batch(self): hda1 = self._wait_for_new_hda() assert str(hda1["deleted"]).lower() == "false" payload = dict(items=[{ "history_content_type": "dataset", "id": hda1["id"] }], deleted=True) update_response = self._raw_update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] def test_update_type_failures(self): hda1 = self._wait_for_new_hda() update_response = self._raw_update(hda1["id"], dict(deleted='not valid')) self._assert_status_code_is(update_response, 400) def _wait_for_new_hda(self): hda1 = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.wait_for_history(self.history_id) return hda1 def _set_edit_update(self, json): set_edit_url = "%s/dataset/set_edit" % self.url update_response = put(set_edit_url, json=json) return update_response def _raw_update(self, item_id, data, admin=False, history_id=None): history_id = history_id or self.history_id key_param = "use_admin_key" if admin else "use_key" update_url = self._api_url( f"histories/{history_id}/contents/{item_id}", **{key_param: True}) update_response = put(update_url, json=data) return update_response def _update_permissions(self, url, data, admin=False): key_param = "use_admin_key" if admin else "use_key" update_url = self._api_url(url, **{key_param: True}) update_response = put(update_url, json=data) return update_response def _raw_update_batch(self, data): update_url = self._api_url("histories/%s/contents" % (self.history_id), use_key=True) update_response = put(update_url, json=data) return update_response def test_delete(self): hda1 = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.wait_for_history(self.history_id) assert str(self.__show(hda1).json()["deleted"]).lower() == "false" delete_response = self._delete("histories/{}/contents/{}".format( self.history_id, hda1["id"])) assert delete_response.status_code < 300 # Something in the 200s :). assert str(self.__show(hda1).json()["deleted"]).lower() == "true" def test_purge(self): hda1 = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.wait_for_history(self.history_id) assert str(self.__show(hda1).json()["deleted"]).lower() == "false" assert str(self.__show(hda1).json()["purged"]).lower() == "false" data = {'purge': True} delete_response = self._delete("histories/{}/contents/{}".format( self.history_id, hda1["id"]), data=data) assert delete_response.status_code < 300 # Something in the 200s :). assert str(self.__show(hda1).json()["deleted"]).lower() == "true" assert str(self.__show(hda1).json()["purged"]).lower() == "true" def test_dataset_collection_creation_on_contents(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, type="dataset_collection") endpoint = "histories/%s/contents" % self.history_id self._check_pair_creation(endpoint, payload) def test_dataset_collection_creation_on_typed_contents(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, ) endpoint = "histories/%s/contents/dataset_collections" % self.history_id self._check_pair_creation(endpoint, payload) def test_dataset_collection_create_from_exisiting_datasets_with_new_tags( self): with self.dataset_populator.test_history() as history_id: hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] hda2_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] update_response = self._raw_update(hda2_id, dict(tags=['existing:tag']), history_id=history_id).json() assert update_response['tags'] == ['existing:tag'] creation_payload = { 'collection_type': 'list', 'history_id': history_id, 'element_identifiers': json.dumps([{ 'id': hda_id, 'src': 'hda', 'name': 'element_id1', 'tags': ['my_new_tag'] }, { 'id': hda2_id, 'src': 'hda', 'name': 'element_id2', 'tags': ['another_new_tag'] }]), 'type': 'dataset_collection', 'copy_elements': True } r = self._post("histories/%s/contents" % self.history_id, creation_payload).json() assert r['elements'][0]['object'][ 'id'] != hda_id, "HDA has not been copied" assert len(r['elements'][0]['object']['tags']) == 1 assert r['elements'][0]['object']['tags'][0] == 'my_new_tag' assert len(r['elements'][1]['object'] ['tags']) == 2, r['elements'][1]['object']['tags'] original_hda = self.dataset_populator.get_history_dataset_details( history_id=history_id, dataset_id=hda_id) assert len(original_hda['tags']) == 0, original_hda['tags'] def _check_pair_creation(self, endpoint, payload): pre_collection_count = self.__count_contents(type="dataset_collection") pre_dataset_count = self.__count_contents(type="dataset") pre_combined_count = self.__count_contents( type="dataset,dataset_collection") dataset_collection_response = self._post(endpoint, payload) dataset_collection = self.__check_create_collection_response( dataset_collection_response) post_collection_count = self.__count_contents( type="dataset_collection") post_dataset_count = self.__count_contents(type="dataset") post_combined_count = self.__count_contents( type="dataset,dataset_collection") # Test filtering types with index. assert pre_collection_count == 0 assert post_collection_count == 1 assert post_combined_count == pre_dataset_count + 1 assert post_combined_count == pre_combined_count + 1 assert pre_dataset_count == post_dataset_count # Test show dataset colleciton. collection_url = "histories/{}/contents/dataset_collections/{}".format( self.history_id, dataset_collection["id"]) show_response = self._get(collection_url) self._assert_status_code_is(show_response, 200) dataset_collection = show_response.json() self._assert_has_keys(dataset_collection, "url", "name", "deleted") assert not dataset_collection["deleted"] delete_response = delete(self._api_url(collection_url, use_key=True)) self._assert_status_code_is(delete_response, 200) show_response = self._get(collection_url) dataset_collection = show_response.json() assert dataset_collection["deleted"] @skip_without_tool("collection_creates_list") def test_jobs_summary_simple_hdca(self): create_response = self.dataset_collection_populator.create_list_in_history( self.history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]) hdca_id = create_response.json()["id"] run = self.dataset_populator.run_collection_creates_list( self.history_id, hdca_id) collections = run['output_collections'] collection = collections[0] jobs_summary_url = "histories/{}/contents/dataset_collections/{}/jobs_summary".format( self.history_id, collection["id"]) jobs_summary_response = self._get(jobs_summary_url) self._assert_status_code_is(jobs_summary_response, 200) jobs_summary = jobs_summary_response.json() self._assert_has_keys(jobs_summary, "populated_state", "states") @skip_without_tool("cat1") def test_jobs_summary_implicit_hdca(self): create_response = self.dataset_collection_populator.create_pair_in_history( self.history_id, contents=["123", "456"]) hdca_id = create_response.json()["id"] inputs = { "input1": { 'batch': True, 'values': [{ 'src': 'hdca', 'id': hdca_id }] }, } run = self.dataset_populator.run_tool("cat1", inputs=inputs, history_id=self.history_id) self.dataset_populator.wait_for_history_jobs(self.history_id) collections = run['implicit_collections'] collection = collections[0] jobs_summary_url = "histories/{}/contents/dataset_collections/{}/jobs_summary".format( self.history_id, collection["id"]) jobs_summary_response = self._get(jobs_summary_url) self._assert_status_code_is(jobs_summary_response, 200) jobs_summary = jobs_summary_response.json() self._assert_has_keys(jobs_summary, "populated_state", "states") states = jobs_summary["states"] assert states.get("ok") == 2, states def test_dataset_collection_hide_originals(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, type="dataset_collection") payload["hide_source_items"] = True dataset_collection_response = self._post( "histories/%s/contents" % self.history_id, payload) self.__check_create_collection_response(dataset_collection_response) contents_response = self._get("histories/%s/contents" % self.history_id) datasets = [ d for d in contents_response.json() if d["history_content_type"] == "dataset" and d["hid"] in [1, 2] ] # Assert two datasets in source were hidden. assert len(datasets) == 2 assert not datasets[0]["visible"] assert not datasets[1]["visible"] def test_update_dataset_collection(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, type="dataset_collection") dataset_collection_response = self._post( "histories/%s/contents" % self.history_id, payload) self._assert_status_code_is(dataset_collection_response, 200) hdca = dataset_collection_response.json() update_url = self._api_url( "histories/{}/contents/dataset_collections/{}".format( self.history_id, hdca["id"]), use_key=True) # Awkward json.dumps required here because of https://trello.com/c/CQwmCeG6 body = json.dumps(dict(name="newnameforpair")) update_response = put(update_url, data=body) self._assert_status_code_is(update_response, 200) show_response = self.__show(hdca) assert str(show_response.json()["name"]) == "newnameforpair" def test_hdca_copy(self): hdca = self.dataset_collection_populator.create_pair_in_history( self.history_id).json() hdca_id = hdca["id"] second_history_id = self.dataset_populator.new_history() create_data = dict( source='hdca', content=hdca_id, ) assert len( self._get("histories/%s/contents/dataset_collections" % second_history_id).json()) == 0 create_response = self._post( "histories/%s/contents/dataset_collections" % second_history_id, create_data) self.__check_create_collection_response(create_response) contents = self._get("histories/%s/contents/dataset_collections" % second_history_id).json() assert len(contents) == 1 new_forward, _ = self.__get_paired_response_elements(contents[0]) self._assert_has_keys(new_forward, "history_id") assert new_forward["history_id"] == self.history_id def test_hdca_copy_with_new_dbkey(self): hdca = self.dataset_collection_populator.create_pair_in_history( self.history_id).json() hdca_id = hdca["id"] assert hdca["elements"][0]["object"]["metadata_dbkey"] == "?" assert hdca["elements"][0]["object"]["genome_build"] == "?" create_data = {'source': 'hdca', 'content': hdca_id, 'dbkey': 'hg19'} create_response = self._post( f"histories/{self.history_id}/contents/dataset_collections", create_data) collection = self.__check_create_collection_response(create_response) new_forward = collection['elements'][0]['object'] assert new_forward["metadata_dbkey"] == "hg19" assert new_forward["genome_build"] == "hg19" def test_hdca_copy_and_elements(self): hdca = self.dataset_collection_populator.create_pair_in_history( self.history_id).json() hdca_id = hdca["id"] second_history_id = self.dataset_populator.new_history() create_data = dict( source='hdca', content=hdca_id, copy_elements=True, ) assert len( self._get("histories/%s/contents/dataset_collections" % second_history_id).json()) == 0 create_response = self._post( "histories/%s/contents/dataset_collections" % second_history_id, create_data) self.__check_create_collection_response(create_response) contents = self._get("histories/%s/contents/dataset_collections" % second_history_id).json() assert len(contents) == 1 new_forward, _ = self.__get_paired_response_elements(contents[0]) self._assert_has_keys(new_forward, "history_id") assert new_forward["history_id"] == second_history_id def __get_paired_response_elements(self, contents): hdca = self.__show(contents).json() self._assert_has_keys(hdca, "name", "deleted", "visible", "elements") elements = hdca["elements"] assert len(elements) == 2 element0 = elements[0] element1 = elements[1] self._assert_has_keys(element0, "object") self._assert_has_keys(element1, "object") return element0["object"], element1["object"] def test_hdca_from_library_datasets(self): ld = self.library_populator.new_library_dataset("el1") ldda_id = ld["ldda_id"] element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}] history_id = self.dataset_populator.new_history() create_data = dict( history_id=history_id, type="dataset_collection", name="Test From Library", element_identifiers=json.dumps(element_identifiers), collection_type="list", ) create_response = self._post( "histories/%s/contents/dataset_collections" % history_id, create_data) hdca = self.__check_create_collection_response(create_response) elements = hdca["elements"] assert len(elements) == 1 hda = elements[0]["object"] assert hda["hda_ldda"] == "hda" assert hda["history_content_type"] == "dataset" assert hda["copied_from_ldda_id"] == ldda_id assert hda['history_id'] == history_id def test_hdca_from_inaccessible_library_datasets(self): library, library_dataset = self.library_populator.new_library_dataset_in_private_library( "HDCACreateInaccesibleLibrary") ldda_id = library_dataset["id"] element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}] create_data = dict( history_id=self.history_id, type="dataset_collection", name="Test From Library", element_identifiers=json.dumps(element_identifiers), collection_type="list", ) with self._different_user(): second_history_id = self.dataset_populator.new_history() create_response = self._post( "histories/%s/contents/dataset_collections" % second_history_id, create_data) self._assert_status_code_is(create_response, 403) def __check_create_collection_response(self, response): self._assert_status_code_is(response, 200) dataset_collection = response.json() self._assert_has_keys(dataset_collection, "url", "name", "deleted", "visible", "elements") return dataset_collection def __show(self, contents): show_response = self._get("histories/{}/contents/{}s/{}".format( self.history_id, contents["history_content_type"], contents["id"])) return show_response def __count_contents(self, history_id=None, **kwds): if history_id is None: history_id = self.history_id contents_response = self._get("histories/%s/contents" % history_id, kwds) return len(contents_response.json()) def __assert_hda_has_full_details(self, hda_details): self._assert_has_keys(hda_details, "display_types", "display_apps") def __check_for_hda(self, contents_response, hda): self._assert_status_code_is(contents_response, 200) contents = contents_response.json() assert len(contents) == 1 hda_summary = contents[0] self.__assert_matches_hda(hda, hda_summary) return hda_summary def __assert_matches_hda(self, input_hda, query_hda): self._assert_has_keys(query_hda, "id", "name") assert input_hda["name"] == query_hda["name"] assert input_hda["id"] == query_hda["id"] def test_job_state_summary_field(self): create_response = self.dataset_collection_populator.create_pair_in_history( self.history_id, contents=["123", "456"]) self._assert_status_code_is(create_response, 200) contents_response = self._get( "histories/%s/contents?v=dev&keys=job_state_summary&view=summary" % self.history_id) self._assert_status_code_is(contents_response, 200) contents = contents_response.json() for c in filter( lambda c: c['history_content_type'] == 'dataset_collection', contents): assert isinstance(c, dict) assert 'job_state_summary' in c assert isinstance(c['job_state_summary'], dict) def _get_content(self, history_id, update_time): return self._get( f"/api/histories/{history_id}/contents/near/100/100?update_time-ge={update_time}" ).json() def test_history_contents_near_with_update_time(self): with self.dataset_populator.test_history() as history_id: first_time = datetime.utcnow().isoformat() assert len(self._get_content(history_id, update_time=first_time)) == 0 self.dataset_collection_populator.create_list_in_history( history_id=history_id) assert len(self._get_content( history_id, update_time=first_time)) == 4 # 3 datasets self.dataset_populator.wait_for_history(history_id) all_datasets_finished = first_time = datetime.utcnow().isoformat() assert len( self._get_content(history_id, update_time=all_datasets_finished)) == 0 @skip_without_tool('cat_data_and_sleep') def test_history_contents_near_with_update_time_implicit_collection(self): with self.dataset_populator.test_history() as history_id: hdca_id = self.dataset_collection_populator.create_list_in_history( history_id=history_id).json()['id'] self.dataset_populator.wait_for_history(history_id) inputs = { "input1": { 'batch': True, 'values': [{ "src": "hdca", "id": hdca_id }] }, "sleep_time": 2, } response = self.dataset_populator.run_tool( "cat_data_and_sleep", inputs, history_id, assert_ok=False, ).json() update_time = datetime.utcnow().isoformat() collection_id = response['implicit_collections'][0]['id'] for _ in range(20): time.sleep(1) update = self._get_content(history_id, update_time=update_time) if any(c for c in update if c['history_content_type'] == 'dataset_collection' and c['job_state_summary']['ok'] == 3): return raise Exception( f"History content update time query did not include final update for implicit collection {collection_id}" ) @skip_without_tool('collection_creates_dynamic_nested') def test_history_contents_near_with_update_time_explicit_collection(self): with self.dataset_populator.test_history() as history_id: inputs = {'foo': 'bar', 'sleep_time': 2} response = self.dataset_populator.run_tool( "collection_creates_dynamic_nested", inputs, history_id, assert_ok=False, ).json() update_time = datetime.utcnow().isoformat() collection_id = response['output_collections'][0]['id'] for _ in range(20): time.sleep(1) update = self._get_content(history_id, update_time=update_time) if any(c for c in update if c['history_content_type'] == 'dataset_collection' and c['populated_state'] == 'ok'): return raise Exception( f"History content update time query did not include populated_state update for dynamic nested collection {collection_id}" )
class DatasetCollectionApiTestCase(ApiTestCase): history_id: str def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_create_pair_from_history(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, instance_type="history", ) create_response = self._post("dataset_collections", payload, json=True) dataset_collection = self._check_create_response(create_response) returned_datasets = dataset_collection["elements"] assert len(returned_datasets) == 2, dataset_collection def test_create_list_from_history(self): element_identifiers = self.dataset_collection_populator.list_identifiers( self.history_id) payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=element_identifiers, collection_type="list", ) create_response = self._post("dataset_collections", payload, json=True) dataset_collection = self._check_create_response(create_response) returned_datasets = dataset_collection["elements"] assert len(returned_datasets) == 3, dataset_collection def test_create_list_of_existing_pairs(self): pair_payload = self.dataset_collection_populator.create_pair_payload( self.history_id, instance_type="history", ) pair_create_response = self._post("dataset_collections", pair_payload, json=True) dataset_collection = self._check_create_response(pair_create_response) hdca_id = dataset_collection["id"] element_identifiers = [dict(name="test1", src="hdca", id=hdca_id)] payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=element_identifiers, collection_type="list", ) create_response = self._post("dataset_collections", payload, json=True) dataset_collection = self._check_create_response(create_response) returned_collections = dataset_collection["elements"] assert len(returned_collections) == 1, dataset_collection def test_create_list_of_new_pairs(self): identifiers = self.dataset_collection_populator.nested_collection_identifiers( self.history_id, "list:paired") payload = dict( collection_type="list:paired", instance_type="history", history_id=self.history_id, name="a nested collection", element_identifiers=identifiers, ) create_response = self._post("dataset_collections", payload, json=True) dataset_collection = self._check_create_response(create_response) assert dataset_collection["collection_type"] == "list:paired" assert dataset_collection["name"] == "a nested collection" returned_collections = dataset_collection["elements"] assert len(returned_collections) == 1, dataset_collection pair_1_element = returned_collections[0] self._assert_has_keys(pair_1_element, "element_identifier", "element_index", "object") assert pair_1_element[ "element_identifier"] == "test_level_1", pair_1_element assert pair_1_element["element_index"] == 0, pair_1_element pair_1_object = pair_1_element["object"] self._assert_has_keys(pair_1_object, "collection_type", "elements", "element_count") self.assertEqual(pair_1_object["collection_type"], "paired") self.assertEqual(pair_1_object["populated"], True) pair_elements = pair_1_object["elements"] assert len(pair_elements) == 2 pair_1_element_1 = pair_elements[0] assert pair_1_element_1["element_index"] == 0 def test_list_download(self): fetch_response = self.dataset_collection_populator.create_list_in_history( self.history_id, direct_upload=True).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 3, dataset_collection create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 3, f"Expected 3 elements in [{namelist}]" collection_name = dataset_collection['name'] for element, zip_path in zip(returned_dce, namelist): assert f"{collection_name}/{element['element_identifier']}.{element['object']['file_ext']}" == zip_path def test_pair_download(self): fetch_response = self.dataset_collection_populator.create_pair_in_history( self.history_id, direct_upload=True).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 2, dataset_collection hdca_id = dataset_collection['id'] create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=hdca_id) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 2, f"Expected 2 elements in [{namelist}]" collection_name = dataset_collection['name'] for element, zip_path in zip(returned_dce, namelist): assert f"{collection_name}/{element['element_identifier']}.{element['object']['file_ext']}" == zip_path def test_list_pair_download(self): fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history( self.history_id).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection list_collection_name = dataset_collection['name'] pair = returned_dce[0] create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 2, f"Expected 2 elements in [{namelist}]" pair_collection_name = pair['element_identifier'] for element, zip_path in zip(pair['object']['elements'], namelist): assert f"{list_collection_name}/{pair_collection_name}/{element['element_identifier']}.{element['object']['file_ext']}" == zip_path def test_list_list_download(self): dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history( self.history_id).json() self.dataset_collection_populator.wait_for_dataset_collection( dataset_collection, assert_ok=True) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 3, f"Expected 3 elements in [{namelist}]" def test_list_list_list_download(self): dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history( self.history_id, collection_type='list:list:list').json() self.dataset_collection_populator.wait_for_dataset_collection( dataset_collection, assert_ok=True) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 3, f"Expected 3 elements in [{namelist}]" def test_hda_security(self): element_identifiers = self.dataset_collection_populator.pair_identifiers( self.history_id) self.dataset_populator.make_private(self.history_id, element_identifiers[0]["id"]) with self._different_user(): history_id = self.dataset_populator.new_history() payload = dict( instance_type="history", history_id=history_id, element_identifiers=element_identifiers, collection_type="paired", ) create_response = self._post("dataset_collections", payload, json=True) self._assert_status_code_is(create_response, 403) def test_enforces_unique_names(self): element_identifiers = self.dataset_collection_populator.list_identifiers( self.history_id) element_identifiers[2]["name"] = element_identifiers[0]["name"] payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=element_identifiers, collection_type="list", ) create_response = self._post("dataset_collections", payload, json=True) self._assert_status_code_is(create_response, 400) def test_upload_collection(self): elements = [{ "src": "files", "dbkey": "hg19", "info": "my cool bed", "tags": ["name:data1", "group:condition:treated", "machine:illumina"] }] targets = [{ "destination": { "type": "hdca" }, "elements": elements, "collection_type": "list", "name": "Test upload", "tags": ["name:collection1"] }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() self.assertEqual(hdca["name"], "Test upload") hdca_tags = hdca["tags"] assert len(hdca_tags) == 1 assert "name:collection1" in hdca_tags assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "4.bed" dataset0 = element0["object"] assert dataset0["file_size"] == 61 dataset_tags = dataset0["tags"] assert len(dataset_tags) == 3, dataset0 def test_upload_nested(self): elements = [{ "name": "samp1", "elements": [{ "src": "files", "dbkey": "hg19", "info": "my cool bed" }] }] targets = [{ "destination": { "type": "hdca" }, "elements": elements, "collection_type": "list:list", "name": "Test upload", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() self.assertEqual(hdca["name"], "Test upload") assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "samp1" @skip_if_github_down def test_upload_collection_from_url(self): elements = [{ "src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", "info": "my cool bed" }] targets = [{ "destination": { "type": "hdca" }, "elements": elements, "collection_type": "list", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "4.bed" assert element0["object"]["file_size"] == 61 @skip_if_github_down def test_upload_collection_failed_expansion_url(self): targets = [{ "destination": { "type": "hdca" }, "elements_from": "bagit", "collection_type": "list", "src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload, assert_ok=False, wait=True) hdca = self._assert_one_collection_created_in_history() assert hdca["populated"] is False assert "bagit.txt" in hdca["populated_state_message"], hdca def _assert_one_collection_created_in_history(self): contents_response = self._get( f"histories/{self.history_id}/contents/dataset_collections") self._assert_status_code_is(contents_response, 200) contents = contents_response.json() assert len(contents) == 1 hdca = contents[0] assert hdca["history_content_type"] == "dataset_collection" hdca_id = hdca["id"] collection_response = self._get( f"histories/{self.history_id}/contents/dataset_collections/{hdca_id}" ) self._assert_status_code_is(collection_response, 200) return collection_response.json() def _check_create_response(self, create_response): self._assert_status_code_is(create_response, 200) dataset_collection = create_response.json() self._assert_has_keys(dataset_collection, "elements", "url", "name", "collection_type", "element_count") return dataset_collection def _download_dataset_collection(self, history_id, hdca_id): return self._get( f"histories/{history_id}/contents/dataset_collections/{hdca_id}/download" ) def test_collection_contents_security(self): # request contents on an hdca that doesn't belong to user hdca, contents_url = self._create_collection_contents_pair() with self._different_user(): contents_response = self._get(contents_url) self._assert_status_code_is(contents_response, 403) def test_collection_contents_invalid_collection(self): # request an invalid collection from a valid hdca, should get 404 hdca, contents_url = self._create_collection_contents_pair() response = self._get(contents_url) self._assert_status_code_is(response, 200) fake_collection_id = '5d7db0757a2eb7ef' fake_contents_url = f"/api/dataset_collections/{hdca['id']}/contents/{fake_collection_id}" error_response = self._get(fake_contents_url) assert_object_id_error(error_response) def test_show_dataset_collection(self): fetch_response = self.dataset_collection_populator.create_list_in_history( self.history_id, direct_upload=True).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 3, dataset_collection hdca_id = dataset_collection['id'] dataset_collection_url = f"/api/dataset_collections/{hdca_id}" dataset_collection = self._get(dataset_collection_url).json() assert dataset_collection['id'] == hdca_id assert dataset_collection['collection_type'] == 'list' def test_show_dataset_collection_contents(self): # Get contents_url from history contents, use it to show the first level # of collection contents in the created HDCA, then use it again to drill # down into the nested collection contents hdca = self.dataset_collection_populator.create_list_of_list_in_history( self.history_id).json() root_contents_url = self._get_contents_url_for_hdca(hdca) # check root contents for this collection root_contents = self._get(root_contents_url).json() assert len(root_contents) == len(hdca['elements']) self._compare_collection_contents_elements(root_contents, hdca['elements']) # drill down, retrieve nested collection contents assert 'object' in root_contents[0] assert 'contents_url' in root_contents[0]['object'] drill_contents_url = root_contents[0]['object']['contents_url'] drill_contents = self._get(drill_contents_url).json() assert len(drill_contents) == len( hdca['elements'][0]['object']['elements']) self._compare_collection_contents_elements( drill_contents, hdca['elements'][0]['object']['elements']) def test_collection_contents_limit_offset(self): # check limit/offset params for collection contents endpoint hdca, root_contents_url = self._create_collection_contents_pair() # check limit limited_contents = self._get(f"{root_contents_url}?limit=1").json() assert len(limited_contents) == 1 assert limited_contents[0]['element_index'] == 0 # check offset offset_contents = self._get(f"{root_contents_url}?offset=1").json() assert len(offset_contents) == 1 assert offset_contents[0]['element_index'] == 1 def test_get_suitable_converters_single_datatype(self): response = self.dataset_collection_populator.upload_collection( self.history_id, "list:paired", elements=[{ "name": "test0", "elements": [ { "src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "bed" }, { "src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "bed" }, ] }, { "name": "test1", "elements": [ { "src": "pasted", "paste_content": "789\n", "name": "forward", "ext": "bed" }, { "src": "pasted", "paste_content": "0ab\n", "name": "reverse", "ext": "bed" }, ] }]) self._assert_status_code_is(response, 200) hdca_list_id = response.json()["outputs"][0]["id"] converters = self._get("dataset_collections/" + hdca_list_id + "/suitable_converters") expected = [ # This list is subject to change, but it's unlikely we'll be removing converters 'CONVERTER_bed_to_fli_0', 'CONVERTER_bed_gff_or_vcf_to_bigwig_0', 'CONVERTER_bed_to_gff_0', 'CONVERTER_interval_to_bgzip_0', 'tabular_to_csv', 'CONVERTER_interval_to_bed6_0', 'CONVERTER_interval_to_bedstrict_0', 'CONVERTER_interval_to_tabix_0', 'CONVERTER_interval_to_bed12_0' ] actual = [] for converter in converters.json(): actual.append(converter["tool_id"]) missing_expected_converters = set(expected) - set(actual) assert not missing_expected_converters, f"Expected converter(s) {', '.join(missing_expected_converters)} missing from response" def test_get_suitable_converters_different_datatypes_matches(self): response = self.dataset_collection_populator.upload_collection( self.history_id, "list:paired", elements=[{ "name": "test0", "elements": [ { "src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "bed" }, { "src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "bed" }, ] }, { "name": "test1", "elements": [ { "src": "pasted", "paste_content": "789\n", "name": "forward", "ext": "tabular" }, { "src": "pasted", "paste_content": "0ab\n", "name": "reverse", "ext": "tabular" }, ] }]) self._assert_status_code_is(response, 200) hdca_list_id = response.json()["outputs"][0]["id"] converters = self._get("dataset_collections/" + hdca_list_id + "/suitable_converters") expected = 'tabular_to_csv' actual = [] for converter in converters.json(): actual.append(converter["tool_id"]) assert expected in actual def test_get_suitable_converters_different_datatypes_no_matches(self): response = self.dataset_collection_populator.upload_collection( self.history_id, "list:paired", elements=[{ "name": "test0", "elements": [ { "src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "bed" }, { "src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "bed" }, ] }, { "name": "test1", "elements": [ { "src": "pasted", "paste_content": "789\n", "name": "forward", "ext": "fasta" }, { "src": "pasted", "paste_content": "0ab\n", "name": "reverse", "ext": "fasta" }, ] }]) self._assert_status_code_is(response, 200) hdca_list_id = response.json()["outputs"][0]["id"] converters = self._get("dataset_collections/" + hdca_list_id + "/suitable_converters") actual: List[str] = [] for converter in converters.json(): actual.append(converter["tool_id"]) assert actual == [] def test_collection_tools_tag_propagation(self): elements = [{"src": "files", "tags": ["name:element_tag"]}] targets = [{ "destination": { "type": "hdca" }, "elements": elements, "collection_type": "list", "name": "Test collection", "tags": ["name:collection_tag"] }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } hdca_id = self.dataset_populator.fetch( payload).json()['output_collections'][0]['id'] inputs = { "input": { "batch": False, "src": "hdca", "id": hdca_id }, } payload = self.dataset_populator.run_tool_payload( tool_id='__FILTER_FAILED_DATASETS__', inputs=inputs, history_id=self.history_id, input_format='legacy', ) response = self._post("tools", payload).json() self.dataset_populator.wait_for_history(self.history_id, assert_ok=False) output_collection = response["output_collections"][0] # collection should not inherit tags from input collection elements, only parent collection assert output_collection['tags'] == ["name:collection_tag"] element = output_collection['elements'][0] # new element hda should have tags copied from old hda assert element['object']['tags'] == ['name:element_tag'] def _compare_collection_contents_elements(self, contents_elements, hdca_elements): # compare collection api results to existing hdca element contents fields = [ 'element_identifier', 'element_index', 'element_type', 'id', 'model_class' ] for (content_element, hdca_element) in zip(contents_elements, hdca_elements): for f in fields: assert content_element[f] == hdca_element[f] def _create_collection_contents_pair(self): # Create a simple collection, return hdca and contents_url payload = self.dataset_collection_populator.create_pair_payload( self.history_id, instance_type="history") create_response = self._post("dataset_collections", payload, json=True) hdca = self._check_create_response(create_response) root_contents_url = self._get_contents_url_for_hdca(hdca) return hdca, root_contents_url def _get_contents_url_for_hdca(self, hdca): # look up the history contents using optional serialization key history_contents_url = f"histories/{self.history_id}/contents?v=dev&view=summary&keys=contents_url" json = self._get(history_contents_url).json() # filter out the collection we just made id = hdca.id # make sure the contents_url appears def find_hdca(c): return c['history_content_type'] == 'dataset_collection' and c[ 'id'] == hdca['id'] matches = list(filter(find_hdca, json)) assert len(matches) == 1 assert 'contents_url' in matches[0] return matches[0]['contents_url']
class DatasetCollectionApiTestCase(ApiTestCase): def setUp(self): super(DatasetCollectionApiTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_create_pair_from_history(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, instance_type="history", ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) returned_datasets = dataset_collection["elements"] assert len(returned_datasets) == 2, dataset_collection def test_create_list_from_history(self): element_identifiers = self.dataset_collection_populator.list_identifiers( self.history_id) payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=json.dumps(element_identifiers), collection_type="list", ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) returned_datasets = dataset_collection["elements"] assert len(returned_datasets) == 3, dataset_collection def test_create_list_of_existing_pairs(self): pair_payload = self.dataset_collection_populator.create_pair_payload( self.history_id, instance_type="history", ) pair_create_response = self._post("dataset_collections", pair_payload) dataset_collection = self._check_create_response(pair_create_response) hdca_id = dataset_collection["id"] element_identifiers = [dict(name="test1", src="hdca", id=hdca_id)] payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=json.dumps(element_identifiers), collection_type="list", ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) returned_collections = dataset_collection["elements"] assert len(returned_collections) == 1, dataset_collection def test_create_list_of_new_pairs(self): identifiers = self.dataset_collection_populator.nested_collection_identifiers( self.history_id, "list:paired") payload = dict( collection_type="list:paired", instance_type="history", history_id=self.history_id, name="a nested collection", element_identifiers=json.dumps(identifiers), ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) assert dataset_collection["collection_type"] == "list:paired" assert dataset_collection["name"] == "a nested collection" returned_collections = dataset_collection["elements"] assert len(returned_collections) == 1, dataset_collection pair_1_element = returned_collections[0] self._assert_has_keys(pair_1_element, "element_identifier", "element_index", "object") assert pair_1_element[ "element_identifier"] == "test_level_1", pair_1_element assert pair_1_element["element_index"] == 0, pair_1_element pair_1_object = pair_1_element["object"] self._assert_has_keys(pair_1_object, "collection_type", "elements", "element_count") self.assertEqual(pair_1_object["collection_type"], "paired") self.assertEqual(pair_1_object["populated"], True) pair_elements = pair_1_object["elements"] assert len(pair_elements) == 2 pair_1_element_1 = pair_elements[0] assert pair_1_element_1["element_index"] == 0 def test_list_download(self): fetch_response = self.dataset_collection_populator.create_list_in_history( self.history_id, direct_upload=True).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 3, dataset_collection create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) tar_contents = tarfile.open(fileobj=BytesIO(create_response.content)) namelist = tar_contents.getnames() assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist collection_name = dataset_collection['name'] for element, zip_path in zip(returned_dce, namelist): assert "%s/%s.%s" % (collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path def test_pair_download(self): fetch_response = self.dataset_collection_populator.create_pair_in_history( self.history_id, direct_upload=True).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 2, dataset_collection hdca_id = dataset_collection['id'] create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=hdca_id) self._assert_status_code_is(create_response, 200) tar_contents = tarfile.open(fileobj=BytesIO(create_response.content)) namelist = tar_contents.getnames() assert len(namelist) == 2, "Expected 2 elements in [%s]" % namelist collection_name = dataset_collection['name'] for element, zip_path in zip(returned_dce, namelist): assert "%s/%s.%s" % (collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path def test_list_pair_download(self): fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history( self.history_id).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection list_collection_name = dataset_collection['name'] pair = returned_dce[0] create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) tar_contents = tarfile.open(fileobj=BytesIO(create_response.content)) namelist = tar_contents.getnames() assert len(namelist) == 2, "Expected 2 elements in [%s]" % namelist pair_collection_name = pair['element_identifier'] for element, zip_path in zip(pair['object']['elements'], namelist): assert "%s/%s/%s.%s" % (list_collection_name, pair_collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path def test_list_list_download(self): dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history( self.history_id).json() self.dataset_collection_populator.wait_for_dataset_collection( dataset_collection, assert_ok=True) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) tar_contents = tarfile.open(fileobj=BytesIO(create_response.content)) namelist = tar_contents.getnames() assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist def test_list_list_list_download(self): dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history( self.history_id, collection_type='list:list:list').json() self.dataset_collection_populator.wait_for_dataset_collection( dataset_collection, assert_ok=True) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection create_response = self._download_dataset_collection( history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) tar_contents = tarfile.open(fileobj=BytesIO(create_response.content)) namelist = tar_contents.getnames() assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist def test_hda_security(self): element_identifiers = self.dataset_collection_populator.pair_identifiers( self.history_id) self.dataset_populator.make_private(self.history_id, element_identifiers[0]["id"]) with self._different_user(): history_id = self.dataset_populator.new_history() payload = dict( instance_type="history", history_id=history_id, element_identifiers=json.dumps(element_identifiers), collection_type="paired", ) create_response = self._post("dataset_collections", payload) self._assert_status_code_is(create_response, 403) def test_enforces_unique_names(self): element_identifiers = self.dataset_collection_populator.list_identifiers( self.history_id) element_identifiers[2]["name"] = element_identifiers[0]["name"] payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=json.dumps(element_identifiers), collection_type="list", ) create_response = self._post("dataset_collections", payload) self._assert_status_code_is(create_response, 400) def test_upload_collection(self): elements = [{ "src": "files", "dbkey": "hg19", "info": "my cool bed", "tags": ["name:data1", "group:condition:treated", "machine:illumina"] }] targets = [{ "destination": { "type": "hdca" }, "elements": elements, "collection_type": "list", "name": "Test upload", "tags": ["name:collection1"] }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() self.assertEqual(hdca["name"], "Test upload") hdca_tags = hdca["tags"] assert len(hdca_tags) == 1 assert "name:collection1" in hdca_tags assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "4.bed" dataset0 = element0["object"] assert dataset0["file_size"] == 61 dataset_tags = dataset0["tags"] assert len(dataset_tags) == 3, dataset0 def test_upload_nested(self): elements = [{ "name": "samp1", "elements": [{ "src": "files", "dbkey": "hg19", "info": "my cool bed" }] }] targets = [{ "destination": { "type": "hdca" }, "elements": elements, "collection_type": "list:list", "name": "Test upload", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() self.assertEqual(hdca["name"], "Test upload") assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "samp1" @skip_if_github_down def test_upload_collection_from_url(self): elements = [{ "src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", "info": "my cool bed" }] targets = [{ "destination": { "type": "hdca" }, "elements": elements, "collection_type": "list", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": { "files_0|file_data": open(self.test_data_resolver.get_filename("4.bed")) }, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "4.bed" assert element0["object"]["file_size"] == 61 def _assert_one_collection_created_in_history(self): contents_response = self._get( "histories/%s/contents/dataset_collections" % self.history_id) self._assert_status_code_is(contents_response, 200) contents = contents_response.json() assert len(contents) == 1 hdca = contents[0] assert hdca["history_content_type"] == "dataset_collection" hdca_id = hdca["id"] collection_response = self._get( "histories/%s/contents/dataset_collections/%s" % (self.history_id, hdca_id)) self._assert_status_code_is(collection_response, 200) return collection_response.json() def _check_create_response(self, create_response): self._assert_status_code_is(create_response, 200) dataset_collection = create_response.json() self._assert_has_keys(dataset_collection, "elements", "url", "name", "collection_type", "element_count") return dataset_collection def _download_dataset_collection(self, history_id, hdca_id): return self._get( "histories/%s/contents/dataset_collections/%s/download" % (history_id, hdca_id))
def setUp(self): super(LibrariesApiTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor)
class HistoryContentsApiNearTestCase(ApiTestCase): """ Test the /api/histories/{history_id}/contents/{direction}/{hid}/{limit} endpoint. """ NEAR = DirectionOptions.near BEFORE = DirectionOptions.before AFTER = DirectionOptions.after def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) def _create_list_in_history(self, history_id, n=2): # Creates list of size n*4 (n collections with 3 items each) for _ in range(n): self.dataset_collection_populator.create_list_in_history( history_id=history_id) def _get_content(self, history_id, direction, *, hid, limit=1000): return self._get( f"/api/histories/{history_id}/contents/{direction}/{hid}/{limit}" ).json() def test_returned_hid_sequence_in_base_case(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=1) assert len(result) == 8 assert result[0]['hid'] == 8 assert result[1]['hid'] == 7 assert result[2]['hid'] == 6 assert result[3]['hid'] == 5 assert result[4]['hid'] == 4 assert result[5]['hid'] == 3 assert result[6]['hid'] == 2 assert result[7]['hid'] == 1 def test_near_even_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=5, limit=3) assert len(result) == 3 assert result[0]['hid'] == 6 # hid + 1 assert result[1]['hid'] == 5 # hid assert result[2]['hid'] == 4 # hid - 1 def test_near_odd_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=5, limit=4) assert len(result) == 4 assert result[0]['hid'] == 7 # hid + 2 assert result[1]['hid'] == 6 # hid + 1 assert result[2]['hid'] == 5 # hid assert result[3]['hid'] == 4 # hid - 1 def test_near_less_than_before_limit(self): # n before < limit // 2 with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=1, limit=3) assert len(result) == 2 assert result[0]['hid'] == 2 # hid + 1 assert result[1]['hid'] == 1 # hid (there's nothing before hid=1) def test_near_less_than_after_limit(self): # n after < limit // 2 + 1 with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.NEAR, hid=8, limit=3) assert len(result) == 2 assert result[0]['hid'] == 8 # hid (there's nothing after hid=8) assert result[1]['hid'] == 7 # hid - 1 def test_near_less_than_before_and_after_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id, n=1) result = self._get_content(history_id, self.NEAR, hid=2, limit=10) assert len(result) == 4 assert result[0]['hid'] == 4 # hid + 2 (can't go after hid=4) assert result[1]['hid'] == 3 # hid + 1 assert result[2]['hid'] == 2 # hid assert result[3]['hid'] == 1 # hid - 1 (can't go before hid=1) def test_before(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.BEFORE, hid=5, limit=3) assert len(result) == 3 assert result[0]['hid'] == 4 # hid - 1 assert result[1]['hid'] == 3 # hid - 2 assert result[2]['hid'] == 2 # hid - 3 def test_before_less_than_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.BEFORE, hid=2, limit=3) assert len(result) == 1 assert result[0]['hid'] == 1 # hid - 1 def test_after(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.AFTER, hid=5, limit=2) assert len(result) == 2 assert result[0][ 'hid'] == 7 # hid + 2 (hid + 3 not included: tests reversed order) assert result[1]['hid'] == 6 # hid + 1 def test_after_less_than_limit(self): with self.dataset_populator.test_history() as history_id: self._create_list_in_history(history_id) result = self._get_content(history_id, self.AFTER, hid=7, limit=3) assert len(result) == 1 assert result[0]['hid'] == 8 # hid + 1
class DatasetCollectionApiTestCase(ApiTestCase): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_create_pair_from_history(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, instance_type="history", ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) returned_datasets = dataset_collection["elements"] assert len(returned_datasets) == 2, dataset_collection def test_create_list_from_history(self): element_identifiers = self.dataset_collection_populator.list_identifiers(self.history_id) payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=json.dumps(element_identifiers), collection_type="list", ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) returned_datasets = dataset_collection["elements"] assert len(returned_datasets) == 3, dataset_collection def test_create_list_of_existing_pairs(self): pair_payload = self.dataset_collection_populator.create_pair_payload( self.history_id, instance_type="history", ) pair_create_response = self._post("dataset_collections", pair_payload) dataset_collection = self._check_create_response(pair_create_response) hdca_id = dataset_collection["id"] element_identifiers = [ dict(name="test1", src="hdca", id=hdca_id) ] payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=json.dumps(element_identifiers), collection_type="list", ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) returned_collections = dataset_collection["elements"] assert len(returned_collections) == 1, dataset_collection def test_create_list_of_new_pairs(self): identifiers = self.dataset_collection_populator.nested_collection_identifiers(self.history_id, "list:paired") payload = dict( collection_type="list:paired", instance_type="history", history_id=self.history_id, name="a nested collection", element_identifiers=json.dumps(identifiers), ) create_response = self._post("dataset_collections", payload) dataset_collection = self._check_create_response(create_response) assert dataset_collection["collection_type"] == "list:paired" assert dataset_collection["name"] == "a nested collection" returned_collections = dataset_collection["elements"] assert len(returned_collections) == 1, dataset_collection pair_1_element = returned_collections[0] self._assert_has_keys(pair_1_element, "element_identifier", "element_index", "object") assert pair_1_element["element_identifier"] == "test_level_1", pair_1_element assert pair_1_element["element_index"] == 0, pair_1_element pair_1_object = pair_1_element["object"] self._assert_has_keys(pair_1_object, "collection_type", "elements", "element_count") self.assertEqual(pair_1_object["collection_type"], "paired") self.assertEqual(pair_1_object["populated"], True) pair_elements = pair_1_object["elements"] assert len(pair_elements) == 2 pair_1_element_1 = pair_elements[0] assert pair_1_element_1["element_index"] == 0 def test_list_download(self): fetch_response = self.dataset_collection_populator.create_list_in_history(self.history_id, direct_upload=True).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 3, dataset_collection create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist collection_name = dataset_collection['name'] for element, zip_path in zip(returned_dce, namelist): assert f"{collection_name}/{element['element_identifier']}.{element['object']['file_ext']}" == zip_path def test_pair_download(self): fetch_response = self.dataset_collection_populator.create_pair_in_history(self.history_id, direct_upload=True).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 2, dataset_collection hdca_id = dataset_collection['id'] create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=hdca_id) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 2, "Expected 2 elements in [%s]" % namelist collection_name = dataset_collection['name'] for element, zip_path in zip(returned_dce, namelist): assert "{}/{}.{}".format(collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path def test_list_pair_download(self): fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history(self.history_id).json() dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection list_collection_name = dataset_collection['name'] pair = returned_dce[0] create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 2, "Expected 2 elements in [%s]" % namelist pair_collection_name = pair['element_identifier'] for element, zip_path in zip(pair['object']['elements'], namelist): assert "{}/{}/{}.{}".format(list_collection_name, pair_collection_name, element['element_identifier'], element['object']['file_ext']) == zip_path def test_list_list_download(self): dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(self.history_id).json() self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist def test_list_list_list_download(self): dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(self.history_id, collection_type='list:list:list').json() self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True) returned_dce = dataset_collection["elements"] assert len(returned_dce) == 1, dataset_collection create_response = self._download_dataset_collection(history_id=self.history_id, hdca_id=dataset_collection['id']) self._assert_status_code_is(create_response, 200) archive = zipfile.ZipFile(BytesIO(create_response.content)) namelist = archive.namelist() assert len(namelist) == 3, "Expected 3 elements in [%s]" % namelist def test_hda_security(self): element_identifiers = self.dataset_collection_populator.pair_identifiers(self.history_id) self.dataset_populator.make_private(self.history_id, element_identifiers[0]["id"]) with self._different_user(): history_id = self.dataset_populator.new_history() payload = dict( instance_type="history", history_id=history_id, element_identifiers=json.dumps(element_identifiers), collection_type="paired", ) create_response = self._post("dataset_collections", payload) self._assert_status_code_is(create_response, 403) def test_enforces_unique_names(self): element_identifiers = self.dataset_collection_populator.list_identifiers(self.history_id) element_identifiers[2]["name"] = element_identifiers[0]["name"] payload = dict( instance_type="history", history_id=self.history_id, element_identifiers=json.dumps(element_identifiers), collection_type="list", ) create_response = self._post("dataset_collections", payload) self._assert_status_code_is(create_response, 400) def test_upload_collection(self): elements = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "tags": ["name:data1", "group:condition:treated", "machine:illumina"]}] targets = [{ "destination": {"type": "hdca"}, "elements": elements, "collection_type": "list", "name": "Test upload", "tags": ["name:collection1"] }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))}, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() self.assertEqual(hdca["name"], "Test upload") hdca_tags = hdca["tags"] assert len(hdca_tags) == 1 assert "name:collection1" in hdca_tags assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "4.bed" dataset0 = element0["object"] assert dataset0["file_size"] == 61 dataset_tags = dataset0["tags"] assert len(dataset_tags) == 3, dataset0 def test_upload_nested(self): elements = [{"name": "samp1", "elements": [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}]}] targets = [{ "destination": {"type": "hdca"}, "elements": elements, "collection_type": "list:list", "name": "Test upload", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))}, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() self.assertEqual(hdca["name"], "Test upload") assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "samp1" @skip_if_github_down def test_upload_collection_from_url(self): elements = [{"src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", "info": "my cool bed"}] targets = [{ "destination": {"type": "hdca"}, "elements": elements, "collection_type": "list", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))}, } self.dataset_populator.fetch(payload) hdca = self._assert_one_collection_created_in_history() assert len(hdca["elements"]) == 1, hdca element0 = hdca["elements"][0] assert element0["element_identifier"] == "4.bed" assert element0["object"]["file_size"] == 61 @skip_if_github_down def test_upload_collection_failed_expansion_url(self): targets = [{ "destination": {"type": "hdca"}, "elements_from": "bagit", "collection_type": "list", "src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", }] payload = { "history_id": self.history_id, "targets": json.dumps(targets), "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))}, } self.dataset_populator.fetch(payload, assert_ok=False, wait=True) hdca = self._assert_one_collection_created_in_history() assert hdca["populated"] is False assert "bagit.txt" in hdca["populated_state_message"], hdca def _assert_one_collection_created_in_history(self): contents_response = self._get("histories/%s/contents/dataset_collections" % self.history_id) self._assert_status_code_is(contents_response, 200) contents = contents_response.json() assert len(contents) == 1 hdca = contents[0] assert hdca["history_content_type"] == "dataset_collection" hdca_id = hdca["id"] collection_response = self._get(f"histories/{self.history_id}/contents/dataset_collections/{hdca_id}") self._assert_status_code_is(collection_response, 200) return collection_response.json() def _check_create_response(self, create_response): self._assert_status_code_is(create_response, 200) dataset_collection = create_response.json() self._assert_has_keys(dataset_collection, "elements", "url", "name", "collection_type", "element_count") return dataset_collection def _download_dataset_collection(self, history_id, hdca_id): return self._get(f"histories/{history_id}/contents/dataset_collections/{hdca_id}/download") def test_collection_contents_security(self): # request contents on an hdca that doesn't belong to user hdca, contents_url = self._create_collection_contents_pair() with self._different_user(): contents_response = self._get(contents_url) self._assert_status_code_is(contents_response, 403) def test_collection_contents_invalid_collection(self): # request an invalid collection from a valid hdca, should get 404 hdca, contents_url = self._create_collection_contents_pair() response = self._get(contents_url) self._assert_status_code_is(response, 200) fake_collection_id = '5d7db0757a2eb7ef' fake_contents_url = '/api/dataset_collections/{}/contents/{}'.format(hdca['id'], fake_collection_id) error_response = self._get(fake_contents_url) assert_object_id_error(error_response) def test_show_dataset_collection_contents(self): # Get contents_url from history contents, use it to show the first level # of collection contents in the created HDCA, then use it again to drill # down into the nested collection contents hdca = self.dataset_collection_populator.create_list_of_list_in_history(self.history_id).json() root_contents_url = self._get_contents_url_for_hdca(hdca) # check root contents for this collection root_contents = self._get(root_contents_url).json() assert len(root_contents) == len(hdca['elements']) self._compare_collection_contents_elements(root_contents, hdca['elements']) # drill down, retrieve nested collection contents assert 'object' in root_contents[0] assert 'contents_url' in root_contents[0]['object'] drill_contents_url = root_contents[0]['object']['contents_url'] drill_contents = self._get(drill_contents_url).json() assert len(drill_contents) == len(hdca['elements'][0]['object']['elements']) self._compare_collection_contents_elements(drill_contents, hdca['elements'][0]['object']['elements']) def test_collection_contents_limit_offset(self): # check limit/offset params for collection contents endpoint hdca, root_contents_url = self._create_collection_contents_pair() # check limit limited_contents = self._get(root_contents_url + '?limit=1').json() assert len(limited_contents) == 1 assert limited_contents[0]['element_index'] == 0 # check offset offset_contents = self._get(root_contents_url + '?offset=1').json() assert len(offset_contents) == 1 assert offset_contents[0]['element_index'] == 1 def _compare_collection_contents_elements(self, contents_elements, hdca_elements): # compare collection api results to existing hdca element contents fields = ['element_identifier', 'element_index', 'element_type', 'id', 'model_class'] for (content_element, hdca_element) in zip(contents_elements, hdca_elements): for f in fields: assert content_element[f] == hdca_element[f] def _create_collection_contents_pair(self): # Create a simple collection, return hdca and contents_url payload = self.dataset_collection_populator.create_pair_payload(self.history_id, instance_type="history") create_response = self._post("dataset_collections", payload) hdca = self._check_create_response(create_response) root_contents_url = self._get_contents_url_for_hdca(hdca) return hdca, root_contents_url def _get_contents_url_for_hdca(self, hdca): # look up the history contents using optional serialization key history_contents_url = "histories/%s/contents?v=dev&view=summary&keys=contents_url" % (self.history_id) json = self._get(history_contents_url).json() # filter out the collection we just made id = hdca.id # make sure the contents_url appears def find_hdca(c): return c['history_content_type'] == 'dataset_collection' and c['id'] == hdca['id'] matches = list(filter(find_hdca, json)) assert len(matches) == 1 assert 'contents_url' in matches[0] return matches[0]['contents_url']
class HistoryContentsApiTestCase(ApiTestCase): def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) self.library_populator = LibraryPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() def test_index_hda_summary(self): hda1 = self.dataset_populator.new_dataset(self.history_id) contents_response = self._get(f"histories/{self.history_id}/contents") hda_summary = self.__check_for_hda(contents_response, hda1) assert "display_types" not in hda_summary # Quick summary, not full details def test_make_private_and_public(self): hda1 = self._wait_for_new_hda() update_url = f"histories/{self.history_id}/contents/{hda1['id']}/permissions" role_id = self.dataset_populator.user_private_role_id() # Give manage permission to the user. payload = { "access": [], "manage": [role_id], } update_response = self._update_permissions(update_url, payload, admin=True) self._assert_status_code_is(update_response, 200) self._assert_other_user_can_access(hda1["id"]) # Then we restrict access. payload = { "action": "make_private", } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) self._assert_other_user_cannot_access(hda1["id"]) # Then we restrict access. payload = { "action": "remove_restrictions", } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) self._assert_other_user_can_access(hda1["id"]) def test_set_permissions_add_admin_history_contents(self): self._verify_dataset_permissions("history_contents") def test_set_permissions_add_admin_datasets(self): self._verify_dataset_permissions("dataset") def _verify_dataset_permissions(self, api_endpoint): hda1 = self._wait_for_new_hda() hda_id = hda1["id"] if api_endpoint == "history_contents": update_url = f"histories/{self.history_id}/contents/{hda_id}/permissions" else: update_url = f"datasets/{hda_id}/permissions" role_id = self.dataset_populator.user_private_role_id() payload = { "access": [role_id], "manage": [role_id], } # Other users cannot modify permissions. with self._different_user(): update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 403) # First the details render for another user. self._assert_other_user_can_access(hda_id) # Then we restrict access. update_response = self._update_permissions(update_url, payload, admin=True) self._assert_status_code_is(update_response, 200) # Finally the details don't render. self._assert_other_user_cannot_access(hda_id) # But they do for the original user. contents_response = self._get( f"histories/{self.history_id}/contents/{hda_id}").json() assert "name" in contents_response update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) payload = { "access": [role_id], "manage": [role_id], } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) self._assert_other_user_cannot_access(hda_id) user_id = self.dataset_populator.user_id() with self._different_user(): different_user_id = self.dataset_populator.user_id() combined_user_role = self.dataset_populator.create_role( [user_id, different_user_id], description="role for testing permissions") payload = { "access": [combined_user_role["id"]], "manage": [role_id], } update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 200) # Now other user can see dataset again with access permission. self._assert_other_user_can_access(hda_id) # access doesn't imply management though... with self._different_user(): update_response = self._update_permissions(update_url, payload) self._assert_status_code_is(update_response, 403) def _assert_other_user_cannot_access(self, history_content_id): with self._different_user(): contents_response = self.dataset_populator.get_history_dataset_details_raw( history_id=self.history_id, dataset_id=history_content_id) assert contents_response.status_code == 403 def _assert_other_user_can_access(self, history_content_id): with self._different_user(): contents_response = self.dataset_populator.get_history_dataset_details_raw( history_id=self.history_id, dataset_id=history_content_id) contents_response.raise_for_status() assert "name" in contents_response.json() def test_index_hda_all_details(self): hda1 = self.dataset_populator.new_dataset(self.history_id) contents_response = self._get( f"histories/{self.history_id}/contents?details=all") hda_details = self.__check_for_hda(contents_response, hda1) self.__assert_hda_has_full_details(hda_details) def test_index_hda_detail_by_id(self): hda1 = self.dataset_populator.new_dataset(self.history_id) contents_response = self._get( f"histories/{self.history_id}/contents?details={hda1['id']}") hda_details = self.__check_for_hda(contents_response, hda1) self.__assert_hda_has_full_details(hda_details) def test_show_hda(self): hda1 = self.dataset_populator.new_dataset(self.history_id) show_response = self.__show(hda1) self._assert_status_code_is(show_response, 200) self.__assert_matches_hda(hda1, show_response.json()) def _create_copy(self): hda1 = self.dataset_populator.new_dataset(self.history_id) create_data = dict( source='hda', content=hda1["id"], ) second_history_id = self.dataset_populator.new_history() assert self.__count_contents(second_history_id) == 0 create_response = self._post(f"histories/{second_history_id}/contents", create_data, json=True) self._assert_status_code_is(create_response, 200) return create_response.json() def test_hda_copy(self): response = self._create_copy() assert self.__count_contents(response['history_id']) == 1 def test_inheritance_chain(self): response = self._create_copy() inheritance_chain_response = self._get( f"datasets/{response['id']}/inheritance_chain") self._assert_status_code_is_ok(inheritance_chain_response) inheritance_chain = inheritance_chain_response.json() assert len(inheritance_chain) == 1 def test_library_copy(self): ld = self.library_populator.new_library_dataset("lda_test_library") create_data = dict( source='library', content=ld["id"], ) assert self.__count_contents(self.history_id) == 0 create_response = self._post(f"histories/{self.history_id}/contents", create_data, json=True) self._assert_status_code_is(create_response, 200) assert self.__count_contents(self.history_id) == 1 def test_update(self): hda1 = self._wait_for_new_hda() assert str(hda1["deleted"]).lower() == "false" update_response = self._update(hda1["id"], dict(deleted=True)) self._assert_status_code_is(update_response, 200) show_response = self.__show(hda1) assert str(show_response.json()["deleted"]).lower() == "true" update_response = self._update(hda1["id"], dict(name="Updated Name")) assert self.__show(hda1).json()["name"] == "Updated Name" update_response = self._update(hda1["id"], dict(name="Updated Name")) assert self.__show(hda1).json()["name"] == "Updated Name" unicode_name = 'ржевский сапоги' update_response = self._update(hda1["id"], dict(name=unicode_name)) updated_hda = self.__show(hda1).json() assert updated_hda["name"] == unicode_name, updated_hda quoted_name = '"Mooo"' update_response = self._update(hda1["id"], dict(name=quoted_name)) updated_hda = self.__show(hda1).json() assert updated_hda["name"] == quoted_name, quoted_name data = { "dataset_id": hda1["id"], "name": "moocow", "dbkey": "?", "annotation": None, "info": "my info is", "operation": "attributes" } update_response = self._set_edit_update(data) # No key or anything supplied, expect a permission problem. # A bit questionable but I think this is a 400 instead of a 403 so that # we don't distinguish between this is a valid ID you don't have access to # and this is an invalid ID. assert update_response.status_code == 400, update_response.content def test_update_batch(self): hda1 = self._wait_for_new_hda() assert str(hda1["deleted"]).lower() == "false" assert str(hda1["visible"]).lower() == "true" # update deleted flag => true payload = dict(items=[{ "history_content_type": "dataset", "id": hda1["id"] }], deleted=True) update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is True assert objects[0]["visible"] is True # update visibility flag => false payload = dict(items=[{ "history_content_type": "dataset", "id": hda1["id"] }], visible=False) update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is True assert objects[0]["visible"] is False # update both flags payload = dict(items=[{ "history_content_type": "dataset", "id": hda1["id"] }], deleted=False, visible=True) update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is False assert objects[0]["visible"] is True def test_update_batch_collections(self): hdca = self._create_pair_collection() assert hdca["deleted"] is False assert hdca["visible"] is True # update deleted flag => true payload = dict(items=[{ "history_content_type": "dataset_collection", "id": hdca["id"] }], deleted=True) update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is True assert objects[0]["visible"] is True # update visibility flag => false payload = dict(items=[{ "history_content_type": "dataset_collection", "id": hdca["id"] }], visible=False) update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is True assert objects[0]["visible"] is False # update both flags payload = dict(items=[{ "history_content_type": "dataset_collection", "id": hdca["id"] }], deleted=False, visible=True) update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is False assert objects[0]["visible"] is True def test_update_type_failures(self): hda1 = self._wait_for_new_hda() update_response = self._update(hda1["id"], dict(deleted='not valid')) self._assert_status_code_is(update_response, 400) def _wait_for_new_hda(self): hda1 = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.wait_for_history(self.history_id) return hda1 def _set_edit_update(self, data): update_response = self._put(urllib.parse.urljoin( self.url, "dataset/set_edit"), data=data, json=True) return update_response def _update(self, item_id, data, admin=False, history_id=None): history_id = history_id or self.history_id update_response = self._put( f"histories/{history_id}/contents/{item_id}", data=data, json=True, admin=admin) return update_response def _update_permissions(self, url, data, admin=False): update_response = self._put(url, data=data, json=True, admin=admin) return update_response def _update_batch(self, data): update_response = self._put(f"histories/{self.history_id}/contents", data=data, json=True) return update_response def test_delete(self): hda1 = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.wait_for_history(self.history_id) assert str(self.__show(hda1).json()["deleted"]).lower() == "false" delete_response = self._delete( f"histories/{self.history_id}/contents/{hda1['id']}") assert delete_response.status_code < 300 # Something in the 200s :). assert str(self.__show(hda1).json()["deleted"]).lower() == "true" def test_delete_anon(self): with self._different_user(anon=True): history_id = self._get( urllib.parse.urljoin( self.url, "history/current_history_json")).json()['id'] hda1 = self.dataset_populator.new_dataset(history_id) self.dataset_populator.wait_for_history(history_id) assert str(self.__show(hda1).json()["deleted"]).lower() == "false" delete_response = self._delete( f"histories/{history_id}/contents/{hda1['id']}") assert delete_response.status_code < 300 # Something in the 200s :). assert str(self.__show(hda1).json()["deleted"]).lower() == "true" def test_delete_permission_denied(self): hda1 = self.dataset_populator.new_dataset(self.history_id) with self._different_user(anon=True): delete_response = self._delete( f"histories/{self.history_id}/contents/{hda1['id']}") assert delete_response.status_code == 403 assert delete_response.json( )['err_msg'] == 'HistoryDatasetAssociation is not owned by user' def test_purge(self): hda1 = self.dataset_populator.new_dataset(self.history_id) self.dataset_populator.wait_for_history(self.history_id) assert str(self.__show(hda1).json()["deleted"]).lower() == "false" assert str(self.__show(hda1).json()["purged"]).lower() == "false" data = {'purge': True} delete_response = self._delete( f"histories/{self.history_id}/contents/{hda1['id']}", data=data, json=True) assert delete_response.status_code < 300 # Something in the 200s :). assert str(self.__show(hda1).json()["deleted"]).lower() == "true" assert str(self.__show(hda1).json()["purged"]).lower() == "true" def test_dataset_collection_creation_on_contents(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, type="dataset_collection") endpoint = f"histories/{self.history_id}/contents" self._check_pair_creation(endpoint, payload) def test_dataset_collection_creation_on_typed_contents(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, ) endpoint = f"histories/{self.history_id}/contents/dataset_collections" self._check_pair_creation(endpoint, payload) def test_dataset_collection_create_from_exisiting_datasets_with_new_tags( self): with self.dataset_populator.test_history() as history_id: hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] hda2_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] update_response = self._update(hda2_id, dict(tags=['existing:tag']), history_id=history_id).json() assert update_response['tags'] == ['existing:tag'] creation_payload = { 'collection_type': 'list', 'history_id': history_id, 'element_identifiers': [{ 'id': hda_id, 'src': 'hda', 'name': 'element_id1', 'tags': ['my_new_tag'] }, { 'id': hda2_id, 'src': 'hda', 'name': 'element_id2', 'tags': ['another_new_tag'] }], 'type': 'dataset_collection', 'copy_elements': True } r = self._post(f"histories/{self.history_id}/contents", creation_payload, json=True).json() assert r['elements'][0]['object'][ 'id'] != hda_id, "HDA has not been copied" assert len(r['elements'][0]['object']['tags']) == 1 assert r['elements'][0]['object']['tags'][0] == 'my_new_tag' assert len(r['elements'][1]['object'] ['tags']) == 2, r['elements'][1]['object']['tags'] original_hda = self.dataset_populator.get_history_dataset_details( history_id=history_id, dataset_id=hda_id) assert len(original_hda['tags']) == 0, original_hda['tags'] def _check_pair_creation(self, endpoint, payload): pre_collection_count = self.__count_contents(type="dataset_collection") pre_dataset_count = self.__count_contents(type="dataset") pre_combined_count = self.__count_contents( type="dataset,dataset_collection") dataset_collection_response = self._post(endpoint, payload, json=True) dataset_collection = self.__check_create_collection_response( dataset_collection_response) post_collection_count = self.__count_contents( type="dataset_collection") post_dataset_count = self.__count_contents(type="dataset") post_combined_count = self.__count_contents( type="dataset,dataset_collection") # Test filtering types with index. assert pre_collection_count == 0 assert post_collection_count == 1 assert post_combined_count == pre_dataset_count + 1 assert post_combined_count == pre_combined_count + 1 assert pre_dataset_count == post_dataset_count # Test show dataset colleciton. collection_url = f"histories/{self.history_id}/contents/dataset_collections/{dataset_collection['id']}" show_response = self._get(collection_url) self._assert_status_code_is(show_response, 200) dataset_collection = show_response.json() self._assert_has_keys(dataset_collection, "url", "name", "deleted") assert not dataset_collection["deleted"] delete_response = self._delete(collection_url) self._assert_status_code_is(delete_response, 200) show_response = self._get(collection_url) dataset_collection = show_response.json() assert dataset_collection["deleted"] @skip_without_tool("collection_creates_list") def test_jobs_summary_simple_hdca(self): create_response = self.dataset_collection_populator.create_list_in_history( self.history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]) hdca_id = create_response.json()["id"] run = self.dataset_populator.run_collection_creates_list( self.history_id, hdca_id) collections = run['output_collections'] collection = collections[0] jobs_summary_url = f"histories/{self.history_id}/contents/dataset_collections/{collection['id']}/jobs_summary" jobs_summary_response = self._get(jobs_summary_url) self._assert_status_code_is(jobs_summary_response, 200) jobs_summary = jobs_summary_response.json() self._assert_has_keys(jobs_summary, "populated_state", "states") @skip_without_tool("cat1") def test_jobs_summary_implicit_hdca(self): create_response = self.dataset_collection_populator.create_pair_in_history( self.history_id, contents=["123", "456"]) hdca_id = create_response.json()["id"] inputs = { "input1": { 'batch': True, 'values': [{ 'src': 'hdca', 'id': hdca_id }] }, } run = self.dataset_populator.run_tool("cat1", inputs=inputs, history_id=self.history_id) self.dataset_populator.wait_for_history_jobs(self.history_id) collections = run['implicit_collections'] collection = collections[0] jobs_summary_url = f"histories/{self.history_id}/contents/dataset_collections/{collection['id']}/jobs_summary" jobs_summary_response = self._get(jobs_summary_url) self._assert_status_code_is(jobs_summary_response, 200) jobs_summary = jobs_summary_response.json() self._assert_has_keys(jobs_summary, "populated_state", "states") states = jobs_summary["states"] assert states.get("ok") == 2, states def test_dataset_collection_hide_originals(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, type="dataset_collection") payload["hide_source_items"] = True dataset_collection_response = self._post( f"histories/{self.history_id}/contents", payload, json=True) self.__check_create_collection_response(dataset_collection_response) contents_response = self._get(f"histories/{self.history_id}/contents") datasets = [ d for d in contents_response.json() if d["history_content_type"] == "dataset" and d["hid"] in [1, 2] ] # Assert two datasets in source were hidden. assert len(datasets) == 2 assert not datasets[0]["visible"] assert not datasets[1]["visible"] def test_update_dataset_collection(self): hdca = self._create_pair_collection() body = dict(name="newnameforpair") update_response = self._put( f"histories/{self.history_id}/contents/dataset_collections/{hdca['id']}", data=body, json=True) self._assert_status_code_is(update_response, 200) show_response = self.__show(hdca) assert str(show_response.json()["name"]) == "newnameforpair" def test_update_batch_dataset_collection(self): hdca = self._create_pair_collection() body = { "items": [{ "history_content_type": "dataset_collection", "id": hdca["id"] }], "name": "newnameforpair" } update_response = self._put(f"histories/{self.history_id}/contents", data=body, json=True) self._assert_status_code_is(update_response, 200) show_response = self.__show(hdca) assert str(show_response.json()["name"]) == "newnameforpair" def _create_pair_collection(self): payload = self.dataset_collection_populator.create_pair_payload( self.history_id, type="dataset_collection") dataset_collection_response = self._post( f"histories/{self.history_id}/contents", payload, json=True) self._assert_status_code_is(dataset_collection_response, 200) hdca = dataset_collection_response.json() return hdca def test_hdca_copy(self): hdca = self.dataset_collection_populator.create_pair_in_history( self.history_id).json() hdca_id = hdca["id"] second_history_id = self.dataset_populator.new_history() create_data = dict( source='hdca', content=hdca_id, ) assert len( self._get( f"histories/{second_history_id}/contents/dataset_collections"). json()) == 0 create_response = self._post( f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) self.__check_create_collection_response(create_response) contents = self._get( f"histories/{second_history_id}/contents/dataset_collections" ).json() assert len(contents) == 1 new_forward, _ = self.__get_paired_response_elements(contents[0]) self._assert_has_keys(new_forward, "history_id") assert new_forward["history_id"] == self.history_id def test_hdca_copy_with_new_dbkey(self): hdca = self.dataset_collection_populator.create_pair_in_history( self.history_id).json() hdca_id = hdca["id"] assert hdca["elements"][0]["object"]["metadata_dbkey"] == "?" assert hdca["elements"][0]["object"]["genome_build"] == "?" create_data = {'source': 'hdca', 'content': hdca_id, 'dbkey': 'hg19'} create_response = self._post( f"histories/{self.history_id}/contents/dataset_collections", create_data, json=True) collection = self.__check_create_collection_response(create_response) new_forward = collection['elements'][0]['object'] assert new_forward["metadata_dbkey"] == "hg19" assert new_forward["genome_build"] == "hg19" def test_hdca_copy_and_elements(self): hdca = self.dataset_collection_populator.create_pair_in_history( self.history_id).json() hdca_id = hdca["id"] second_history_id = self.dataset_populator.new_history() create_data = dict( source='hdca', content=hdca_id, copy_elements=True, ) assert len( self._get( f"histories/{second_history_id}/contents/dataset_collections"). json()) == 0 create_response = self._post( f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) self.__check_create_collection_response(create_response) contents = self._get( f"histories/{second_history_id}/contents/dataset_collections" ).json() assert len(contents) == 1 new_forward, _ = self.__get_paired_response_elements(contents[0]) self._assert_has_keys(new_forward, "history_id") assert new_forward["history_id"] == second_history_id def __get_paired_response_elements(self, contents): hdca = self.__show(contents).json() self._assert_has_keys(hdca, "name", "deleted", "visible", "elements") elements = hdca["elements"] assert len(elements) == 2 element0 = elements[0] element1 = elements[1] self._assert_has_keys(element0, "object") self._assert_has_keys(element1, "object") return element0["object"], element1["object"] def test_hdca_from_library_datasets(self): ld = self.library_populator.new_library_dataset("el1") ldda_id = ld["ldda_id"] element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}] history_id = self.dataset_populator.new_history() create_data = dict( history_id=history_id, type="dataset_collection", name="Test From Library", element_identifiers=element_identifiers, collection_type="list", ) create_response = self._post( f"histories/{history_id}/contents/dataset_collections", create_data, json=True) hdca = self.__check_create_collection_response(create_response) elements = hdca["elements"] assert len(elements) == 1 hda = elements[0]["object"] assert hda["hda_ldda"] == "hda" assert hda["history_content_type"] == "dataset" assert hda["copied_from_ldda_id"] == ldda_id assert hda['history_id'] == history_id def test_hdca_from_inaccessible_library_datasets(self): library, library_dataset = self.library_populator.new_library_dataset_in_private_library( "HDCACreateInaccesibleLibrary") ldda_id = library_dataset["id"] element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}] create_data = dict( history_id=self.history_id, type="dataset_collection", name="Test From Library", element_identifiers=element_identifiers, collection_type="list", ) with self._different_user(): second_history_id = self.dataset_populator.new_history() create_response = self._post( f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) self._assert_status_code_is(create_response, 403) def __check_create_collection_response(self, response): self._assert_status_code_is(response, 200) dataset_collection = response.json() self._assert_has_keys(dataset_collection, "url", "name", "deleted", "visible", "elements") return dataset_collection def __show(self, contents): show_response = self._get( f"histories/{self.history_id}/contents/{contents['history_content_type']}s/{contents['id']}" ) return show_response def __count_contents(self, history_id=None, **kwds): if history_id is None: history_id = self.history_id contents_response = self._get(f"histories/{history_id}/contents", kwds) return len(contents_response.json()) def __assert_hda_has_full_details(self, hda_details): self._assert_has_keys(hda_details, "display_types", "display_apps") def __check_for_hda(self, contents_response, hda): self._assert_status_code_is(contents_response, 200) contents = contents_response.json() assert len(contents) == 1 hda_summary = contents[0] self.__assert_matches_hda(hda, hda_summary) return hda_summary def __assert_matches_hda(self, input_hda, query_hda): self._assert_has_keys(query_hda, "id", "name") assert input_hda["name"] == query_hda["name"] assert input_hda["id"] == query_hda["id"] def test_job_state_summary_field(self): create_response = self.dataset_collection_populator.create_pair_in_history( self.history_id, contents=["123", "456"]) self._assert_status_code_is(create_response, 200) contents_response = self._get( f"histories/{self.history_id}/contents?v=dev&keys=job_state_summary&view=summary" ) self._assert_status_code_is(contents_response, 200) contents = contents_response.json() for c in filter( lambda c: c['history_content_type'] == 'dataset_collection', contents): assert isinstance(c, dict) assert 'job_state_summary' in c assert isinstance(c['job_state_summary'], dict) def _get_content(self, history_id, update_time): return self._get( f"/api/histories/{history_id}/contents/near/100/100?update_time-gt={update_time}" ).json() def test_history_contents_near_with_update_time(self): with self.dataset_populator.test_history() as history_id: first_time = datetime.utcnow().isoformat() assert len(self._get_content(history_id, update_time=first_time)) == 0 self.dataset_collection_populator.create_list_in_history( history_id=history_id) assert len(self._get_content( history_id, update_time=first_time)) == 4 # 3 datasets self.dataset_populator.wait_for_history(history_id) all_datasets_finished = first_time = datetime.utcnow().isoformat() assert len( self._get_content(history_id, update_time=all_datasets_finished)) == 0 def test_history_contents_near_with_since(self): with self.dataset_populator.test_history() as history_id: original_history = self._get(f"/api/histories/{history_id}").json() original_history_stamp = original_history['update_time'] # check empty contents, with no since flag, should return an empty 200 result history_contents = self._get( f"/api/histories/{history_id}/contents/near/100/100") assert history_contents.status_code == 200 assert len(history_contents.json()) == 0 # adding a since parameter, should return a 204 if history has not changed at all history_contents = self._get( f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}" ) assert history_contents.status_code == 204 # add some stuff self.dataset_collection_populator.create_list_in_history( history_id=history_id) self.dataset_populator.wait_for_history(history_id) # check to make sure the added stuff is there changed_history_contents = self._get( f"/api/histories/{history_id}/contents/near/100/100") assert changed_history_contents.status_code == 200 assert len(changed_history_contents.json()) == 4 # check to make sure the history date has actually changed due to changing the contents changed_history = self._get(f"/api/histories/{history_id}").json() changed_history_stamp = changed_history['update_time'] assert original_history_stamp != changed_history_stamp # a repeated contents request with since=original_history_stamp should now return data # because we have added datasets and the update_time should have been changed changed_content = self._get( f"/api/histories/{history_id}/contents/near/100/100?since={original_history_stamp}" ) assert changed_content.status_code == 200 assert len(changed_content.json()) == 4 def test_history_contents_near_since_with_standard_iso8601_date(self): with self.dataset_populator.test_history() as history_id: original_history = self._get(f"/api/histories/{history_id}").json() original_history_stamp = original_history['update_time'] # this is the standard date format that javascript will emit using .toISOString(), it # should be the expected date format for any modern api # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString # checking to make sure that the same exact history.update_time returns a "not changed" # result after date parsing valid_iso8601_date = original_history_stamp + 'Z' encoded_valid_date = urllib.parse.quote_plus(valid_iso8601_date) history_contents = self._get( f"/api/histories/{history_id}/contents/near/100/100?since={encoded_valid_date}" ) assert history_contents.status_code == 204 # test parsing for other standard is08601 formats sample_formats = [ '2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '2002-10-10T12:00:00-05:00' ] for date_str in sample_formats: encoded_date = urllib.parse.quote_plus( date_str) # handles pluses, minuses history_contents = self._get( f"/api/histories/{history_id}/contents/near/100/100?since={encoded_date}" ) self._assert_status_code_is_ok(history_contents) @skip_without_tool('cat_data_and_sleep') def test_history_contents_near_with_update_time_implicit_collection(self): with self.dataset_populator.test_history() as history_id: hdca_id = self.dataset_collection_populator.create_list_in_history( history_id=history_id).json()['id'] self.dataset_populator.wait_for_history(history_id) inputs = { "input1": { 'batch': True, 'values': [{ "src": "hdca", "id": hdca_id }] }, "sleep_time": 2, } response = self.dataset_populator.run_tool( "cat_data_and_sleep", inputs, history_id, ) update_time = datetime.utcnow().isoformat() collection_id = response['implicit_collections'][0]['id'] for _ in range(20): time.sleep(1) update = self._get_content(history_id, update_time=update_time) if any(c for c in update if c['history_content_type'] == 'dataset_collection' and c['job_state_summary']['ok'] == 3): return raise Exception( f"History content update time query did not include final update for implicit collection {collection_id}" ) @skip_without_tool('collection_creates_dynamic_nested') def test_history_contents_near_with_update_time_explicit_collection(self): with self.dataset_populator.test_history() as history_id: inputs = {'foo': 'bar', 'sleep_time': 2} response = self.dataset_populator.run_tool( "collection_creates_dynamic_nested", inputs, history_id, ) update_time = datetime.utcnow().isoformat() collection_id = response['output_collections'][0]['id'] for _ in range(20): time.sleep(1) update = self._get_content(history_id, update_time=update_time) if any(c for c in update if c['history_content_type'] == 'dataset_collection' and c['populated_state'] == 'ok'): return raise Exception( f"History content update time query did not include populated_state update for dynamic nested collection {collection_id}" ) def test_index_filter_by_type(self): history_id = self.dataset_populator.new_history() self.dataset_populator.new_dataset(history_id) self.dataset_collection_populator.create_list_in_history( history_id=history_id) contents_response = self._get( f"histories/{history_id}/contents").json() num_items = len(contents_response) expected_num_collections = 1 expected_num_datasets = num_items - expected_num_collections contents_response = self._get( f"histories/{history_id}/contents?types=dataset").json() assert len(contents_response) == expected_num_datasets contents_response = self._get( f"histories/{history_id}/contents?types=dataset_collection").json( ) assert len(contents_response) == expected_num_collections def test_elements_datatypes_field(self): history_id = self.dataset_populator.new_history() collection_name = "homogeneous" expected_datatypes = ["txt"] elements = [ # List with all elements of txt datatype (homogeneous) { "name": "test1", "src": "pasted", "paste_content": "abc", "ext": "txt" }, { "name": "test2", "src": "pasted", "paste_content": "abc", "ext": "txt" }, ] self._upload_collection_list_with_elements(history_id, collection_name, elements) self._assert_collection_has_expected_elements_datatypes( history_id, collection_name, expected_datatypes) collection_name = "heterogeneous" expected_datatypes = ["txt", "tabular"] elements = [ # List with txt and tabular datatype (heterogeneous) { "name": "test2", "src": "pasted", "paste_content": "abc", "ext": "txt" }, { "name": "test3", "src": "pasted", "paste_content": "a,b,c\n", "ext": "tabular" }, ] self._upload_collection_list_with_elements(history_id, collection_name, elements) self._assert_collection_has_expected_elements_datatypes( history_id, collection_name, expected_datatypes) def _upload_collection_list_with_elements(self, history_id: str, collection_name: str, elements: List[Any]): create_homogeneous_response = self.dataset_collection_populator.upload_collection( history_id, "list", elements=elements, name=collection_name) self._assert_status_code_is_ok(create_homogeneous_response) def _assert_collection_has_expected_elements_datatypes( self, history_id, collection_name, expected_datatypes): contents_response = self._get( f"histories/{history_id}/contents?v=dev&view=betawebclient&q=name-eq&qv={collection_name}" ) self._assert_status_code_is(contents_response, 200) collection = contents_response.json()[0] self.assertCountEqual(collection["elements_datatypes"], expected_datatypes)
class MaximumWorkflowJobsPerSchedulingIterationTestCase( integration_util.IntegrationTestCase): framework_tool_and_types = True def setUp(self): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) @classmethod def handle_galaxy_config_kwds(cls, config): config["maximum_workflow_jobs_per_scheduling_iteration"] = 1 def test_collection_explicit_and_implicit(self): workflow_id = self.workflow_populator.upload_yaml_workflow(""" class: GalaxyWorkflow steps: - type: input_collection - tool_id: collection_creates_pair state: input1: $link: 0 - tool_id: collection_paired_test state: f1: $link: 1/paired_output - tool_id: cat_list state: input1: $link: 2/out1 """) with self.dataset_populator.test_history() as history_id: hdca1 = self.dataset_collection_populator.create_list_in_history( history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]).json() self.dataset_populator.wait_for_history(history_id, assert_ok=True) inputs = { '0': { "src": "hdca", "id": hdca1["id"] }, } invocation_id = self.workflow_populator.invoke_workflow( history_id, workflow_id, inputs) self.workflow_populator.wait_for_workflow(history_id, workflow_id, invocation_id) self.dataset_populator.wait_for_history(history_id, assert_ok=True) self.assertEqual( "a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0)) def test_scheduling_rounds(self): with self.dataset_populator.test_history() as history_id: invocation_response = self.workflow_populator.run_workflow( """ class: GalaxyWorkflow inputs: input1: data text_input: text steps: first_cat: tool_id: cat1 in: input1: input1 second_cat: tool_id: cat1 in: input1: first_cat/out_file1 collection_creates_dynamic_list_of_pairs: tool_id: collection_creates_dynamic_list_of_pairs in: file: second_cat/out_file1 count_multi_file: tool_id: count_multi_file in: input1: collection_creates_dynamic_list_of_pairs/list_output outputs: wf_output_1: outputSource: collection_creates_dynamic_list_of_pairs/list_output """, test_data=""" input1: value: 1.fasta type: File name: fasta1 text_input: foo """, history_id=history_id) invocation = self._get("/invocations/{}".format( invocation_response.invocation_id)).json() assert 'wf_output_1' in invocation['output_collections']
class JobsApiTestCase(ApiTestCase): def setUp(self): super(JobsApiTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) @uses_test_history(require_new=True) def test_index(self, history_id): # Create HDA to ensure at least one job exists... self.__history_with_new_dataset(history_id) jobs = self.__jobs_index() assert "upload1" in map(itemgetter("tool_id"), jobs) @uses_test_history(require_new=True) def test_system_details_admin_only(self, history_id): self.__history_with_new_dataset(history_id) jobs = self.__jobs_index(admin=False) job = jobs[0] self._assert_not_has_keys(job, "command_line", "external_id") jobs = self.__jobs_index(admin=True) job = jobs[0] self._assert_has_keys(job, "command_line", "external_id") @uses_test_history(require_new=True) def test_index_state_filter(self, history_id): # Initial number of ok jobs original_count = len(self.__uploads_with_state("ok")) # Run through dataset upload to ensure num uplaods at least greater # by 1. self.__history_with_ok_dataset(history_id) # Verify number of ok jobs is actually greater. count_increased = False for _ in range(10): new_count = len(self.__uploads_with_state("ok")) if original_count < new_count: count_increased = True break time.sleep(.1) if not count_increased: template = "Jobs in ok state did not increase (was %d, now %d)" message = template % (original_count, new_count) raise AssertionError(message) @uses_test_history(require_new=True) def test_index_date_filter(self, history_id): self.__history_with_new_dataset(history_id) two_weeks_ago = (datetime.datetime.utcnow() - datetime.timedelta(7)).isoformat() last_week = (datetime.datetime.utcnow() - datetime.timedelta(7)).isoformat() next_week = (datetime.datetime.utcnow() + datetime.timedelta(7)).isoformat() today = datetime.datetime.utcnow().isoformat() tomorrow = (datetime.datetime.utcnow() + datetime.timedelta(1)).isoformat() jobs = self.__jobs_index(data={ "date_range_min": today[0:10], "date_range_max": tomorrow[0:10] }) assert len(jobs) > 0 today_job_id = jobs[0]["id"] jobs = self.__jobs_index(data={ "date_range_min": two_weeks_ago, "date_range_max": last_week }) assert today_job_id not in map(itemgetter("id"), jobs) jobs = self.__jobs_index(data={ "date_range_min": last_week, "date_range_max": next_week }) assert today_job_id in map(itemgetter("id"), jobs) @uses_test_history(require_new=True) def test_index_history(self, history_id): self.__history_with_new_dataset(history_id) jobs = self.__jobs_index(data={"history_id": history_id}) assert len(jobs) > 0 with self.dataset_populator.test_history() as other_history_id: jobs = self.__jobs_index(data={"history_id": other_history_id}) assert len(jobs) == 0 @uses_test_history(require_new=True) def test_index_multiple_states_filter(self, history_id): # Initial number of ok jobs original_count = len(self.__uploads_with_state("ok", "new")) # Run through dataset upload to ensure num uplaods at least greater # by 1. self.__history_with_ok_dataset(history_id) # Verify number of ok jobs is actually greater. new_count = len(self.__uploads_with_state("new", "ok")) assert original_count < new_count, new_count @uses_test_history(require_new=True) def test_show(self, history_id): # Create HDA to ensure at least one job exists... self.__history_with_new_dataset(history_id) jobs_response = self._get("jobs") first_job = jobs_response.json()[0] self._assert_has_key(first_job, 'id', 'state', 'exit_code', 'update_time', 'create_time') job_id = first_job["id"] show_jobs_response = self._get("jobs/%s" % job_id) self._assert_status_code_is(show_jobs_response, 200) job_details = show_jobs_response.json() self._assert_has_key(job_details, 'id', 'state', 'exit_code', 'update_time', 'create_time') show_jobs_response = self._get("jobs/%s" % job_id, {"full": True}) self._assert_status_code_is(show_jobs_response, 200) job_details = show_jobs_response.json() self._assert_has_key(job_details, 'id', 'state', 'exit_code', 'update_time', 'create_time', 'stdout', 'stderr', 'job_messages') @uses_test_history(require_new=True) def test_show_security(self, history_id): self.__history_with_new_dataset(history_id) jobs_response = self._get("jobs", data={"history_id": history_id}) job = jobs_response.json()[0] job_id = job["id"] show_jobs_response = self._get("jobs/%s" % job_id, admin=False) self._assert_not_has_keys(show_jobs_response.json(), "command_line", "external_id") # TODO: Re-activate test case when API accepts privacy settings # with self._different_user(): # show_jobs_response = self._get( "jobs/%s" % job_id, admin=False ) # self._assert_status_code_is( show_jobs_response, 200 ) show_jobs_response = self._get("jobs/%s" % job_id, admin=True) self._assert_has_keys(show_jobs_response.json(), "command_line", "external_id") def _run_detect_errors(self, history_id, inputs): payload = self.dataset_populator.run_tool_payload( tool_id='detect_errors_aggressive', inputs=inputs, history_id=history_id, ) return self._post("tools", data=payload).json() @skip_without_tool("detect_errors_aggressive") def test_unhide_on_error(self): with self.dataset_populator.test_history() as history_id: inputs = {'error_bool': 'true'} run_response = self._run_detect_errors(history_id=history_id, inputs=inputs) job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) job = self.dataset_populator.get_job_details(job_id).json() assert job['state'] == 'error' dataset = self.dataset_populator.get_history_dataset_details( history_id=history_id, dataset_id=run_response['outputs'][0]['id'], assert_ok=False) assert dataset['visible'] @skip_without_tool("detect_errors_aggressive") def test_no_unhide_on_error_if_mapped_over(self): with self.dataset_populator.test_history() as history_id: hdca1 = self.dataset_collection_populator.create_list_in_history( history_id, contents=[("sample1-1", "1 2 3")]).json() inputs = { 'error_bool': 'true', 'dataset': { 'batch': True, 'values': [{ 'src': 'hdca', 'id': hdca1['id'] }], } } run_response = self._run_detect_errors(history_id=history_id, inputs=inputs) job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) job = self.dataset_populator.get_job_details(job_id).json() assert job['state'] == 'error' dataset = self.dataset_populator.get_history_dataset_details( history_id=history_id, dataset_id=run_response['outputs'][0]['id'], assert_ok=False) assert not dataset['visible'] @skip_without_tool('empty_output') def test_common_problems(self): with self.dataset_populator.test_history() as history_id: empty_run_response = self.dataset_populator.run_tool( tool_id='empty_output', inputs={}, history_id=history_id, ) empty_hda = empty_run_response["outputs"][0] cat_empty_twice_run_response = self.dataset_populator.run_tool( tool_id='cat1', inputs={ 'input1': { 'src': 'hda', 'id': empty_hda['id'] }, 'queries_0|input2': { 'src': 'hda', 'id': empty_hda['id'] } }, history_id=history_id, ) empty_output_job = empty_run_response["jobs"][0] cat_empty_job = cat_empty_twice_run_response["jobs"][0] empty_output_common_problems_response = self._get( 'jobs/%s/common_problems' % empty_output_job["id"]).json() cat_empty_common_problems_response = self._get( 'jobs/%s/common_problems' % cat_empty_job["id"]).json() self._assert_has_keys(empty_output_common_problems_response, "has_empty_inputs", "has_duplicate_inputs") self._assert_has_keys(cat_empty_common_problems_response, "has_empty_inputs", "has_duplicate_inputs") assert not empty_output_common_problems_response["has_empty_inputs"] assert cat_empty_common_problems_response["has_empty_inputs"] assert not empty_output_common_problems_response[ "has_duplicate_inputs"] assert cat_empty_common_problems_response["has_duplicate_inputs"] @skip_without_tool('detect_errors_aggressive') def test_report_error(self): with self.dataset_populator.test_history() as history_id: payload = self.dataset_populator.run_tool_payload( tool_id='detect_errors_aggressive', inputs={'error_bool': 'true'}, history_id=history_id, ) run_response = self._post("tools", data=payload).json() job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) dataset_id = run_response['outputs'][0]['id'] response = self._post('jobs/%s/error' % job_id, data={'dataset_id': dataset_id}) assert response.status_code == 200, response.text @skip_without_tool('detect_errors_aggressive') def test_report_error_anon(self): # Need to get a cookie and use that for anonymous tool runs cookies = requests.get(self.url).cookies payload = json.dumps({ "tool_id": "detect_errors_aggressive", "inputs": { "error_bool": "true" } }) run_response = requests.post("%s/tools" % self.galaxy_interactor.api_url, data=payload, cookies=cookies).json() job_id = run_response['jobs'][0]["id"] dataset_id = run_response['outputs'][0]['id'] response = requests.post('%s/jobs/%s/error' % (self.galaxy_interactor.api_url, job_id), data={ 'email': '*****@*****.**', 'dataset_id': dataset_id }, cookies=cookies) assert response.status_code == 200, response.text @uses_test_history(require_new=True) def test_deleting_output_keep_running_until_all_deleted(self, history_id): job_state, outputs = self._setup_running_two_output_job( history_id, 120) self._hack_to_skip_test_if_state_ok(job_state) # Delete one of the two outputs and make sure the job is still running. self._raw_update_history_item(history_id, outputs[0]["id"], {"deleted": True}) self._hack_to_skip_test_if_state_ok(job_state) time.sleep(1) self._hack_to_skip_test_if_state_ok(job_state) state = job_state().json()["state"] assert state == "running", state # Delete the second output and make sure the job is cancelled. self._raw_update_history_item(history_id, outputs[1]["id"], {"deleted": True}) final_state = wait_on_state(job_state, assert_ok=False, timeout=15) assert final_state in ["deleted_new", "deleted"], final_state @uses_test_history(require_new=True) def test_purging_output_keep_running_until_all_purged(self, history_id): job_state, outputs = self._setup_running_two_output_job( history_id, 120) # Pretty much right away after the job is running, these paths should be populated - # if they are grab them and make sure they are deleted at the end of the job. dataset_1 = self._get_history_item_as_admin(history_id, outputs[0]["id"]) dataset_2 = self._get_history_item_as_admin(history_id, outputs[1]["id"]) if "file_name" in dataset_1: output_dataset_paths = [ dataset_1["file_name"], dataset_2["file_name"] ] # This may or may not exist depending on if the test is local or not. output_dataset_paths_exist = os.path.exists( output_dataset_paths[0]) else: output_dataset_paths = [] output_dataset_paths_exist = False self._hack_to_skip_test_if_state_ok(job_state) current_state = job_state().json()["state"] assert current_state == "running", current_state # Purge one of the two outputs and make sure the job is still running. self._raw_update_history_item(history_id, outputs[0]["id"], {"purged": True}) time.sleep(1) self._hack_to_skip_test_if_state_ok(job_state) current_state = job_state().json()["state"] assert current_state == "running", current_state # Purge the second output and make sure the job is cancelled. self._raw_update_history_item(history_id, outputs[1]["id"], {"purged": True}) final_state = wait_on_state(job_state, assert_ok=False, timeout=15) assert final_state in ["deleted_new", "deleted"], final_state def paths_deleted(): if not os.path.exists( output_dataset_paths[0]) and not os.path.exists( output_dataset_paths[1]): return True if output_dataset_paths_exist: wait_on(paths_deleted, "path deletion") @uses_test_history(require_new=True) def test_purging_output_cleaned_after_ok_run(self, history_id): job_state, outputs = self._setup_running_two_output_job(history_id, 10) # Pretty much right away after the job is running, these paths should be populated - # if they are grab them and make sure they are deleted at the end of the job. dataset_1 = self._get_history_item_as_admin(history_id, outputs[0]["id"]) dataset_2 = self._get_history_item_as_admin(history_id, outputs[1]["id"]) if "file_name" in dataset_1: output_dataset_paths = [ dataset_1["file_name"], dataset_2["file_name"] ] # This may or may not exist depending on if the test is local or not. output_dataset_paths_exist = os.path.exists( output_dataset_paths[0]) else: output_dataset_paths = [] output_dataset_paths_exist = False if not output_dataset_paths_exist: # Given this Galaxy configuration - there is nothing more to be tested here. # Consider throwing a skip instead. return # Purge one of the two outputs and wait for the job to complete. self._raw_update_history_item(history_id, outputs[0]["id"], {"purged": True}) wait_on_state(job_state, assert_ok=True) if output_dataset_paths_exist: time.sleep(.5) # Make sure the non-purged dataset is on disk and the purged one is not. assert os.path.exists(output_dataset_paths[1]) assert not os.path.exists(output_dataset_paths[0]) def _hack_to_skip_test_if_state_ok(self, job_state): from nose.plugins.skip import SkipTest if job_state().json()["state"] == "ok": message = "Job state switch from running to ok too quickly - the rest of the test requires the job to be in a running state. Skipping test." raise SkipTest(message) def _setup_running_two_output_job(self, history_id, sleep_time): payload = self.dataset_populator.run_tool_payload( tool_id='create_2', inputs=dict(sleep_time=sleep_time, ), history_id=history_id, ) run_response = self._post("tools", data=payload).json() outputs = run_response["outputs"] jobs = run_response["jobs"] assert len(outputs) == 2 assert len(jobs) == 1 def job_state(): jobs_response = self._get("jobs/%s" % jobs[0]["id"]) return jobs_response # Give job some time to get up and running. time.sleep(2) running_state = wait_on_state(job_state, skip_states=["queued", "new"], assert_ok=False, timeout=15) assert running_state == "running", running_state def job_state(): jobs_response = self._get("jobs/%s" % jobs[0]["id"]) return jobs_response return job_state, outputs def _raw_update_history_item(self, history_id, item_id, data): update_url = self._api_url("histories/%s/contents/%s" % (history_id, item_id), use_key=True) update_response = requests.put(update_url, json=data) assert_status_code_is_ok(update_response) return update_response @skip_without_tool("cat_data_and_sleep") @uses_test_history(require_new=True) def test_resume_job(self, history_id): hda1 = self.dataset_populator.new_dataset( history_id, content="samp1\t10.0\nsamp2\t20.0\n") hda2 = self.dataset_populator.new_dataset( history_id, content="samp1\t30.0\nsamp2\t40.0\n") # Submit first job payload = self.dataset_populator.run_tool_payload( tool_id='cat_data_and_sleep', inputs={ 'sleep_time': 15, 'input1': { 'src': 'hda', 'id': hda2['id'] }, 'queries_0|input2': { 'src': 'hda', 'id': hda2['id'] } }, history_id=history_id, ) run_response = self._post("tools", data=payload).json() output = run_response["outputs"][0] # Submit second job that waits on job1 payload = self.dataset_populator.run_tool_payload( tool_id='cat1', inputs={ 'input1': { 'src': 'hda', 'id': hda1['id'] }, 'queries_0|input2': { 'src': 'hda', 'id': output['id'] } }, history_id=history_id, ) run_response = self._post("tools", data=payload).json() job_id = run_response['jobs'][0]['id'] output = run_response["outputs"][0] # Delete second jobs input while second job is waiting for first job delete_response = self._delete("histories/%s/contents/%s" % (history_id, hda1['id'])) self._assert_status_code_is(delete_response, 200) self.dataset_populator.wait_for_history_jobs(history_id, assert_ok=False) dataset_details = self._get("histories/%s/contents/%s" % (history_id, output['id'])).json() assert dataset_details['state'] == 'paused' # Undelete input dataset undelete_response = self._put("histories/%s/contents/%s" % (history_id, hda1['id']), data=json.dumps({'deleted': False})) self._assert_status_code_is(undelete_response, 200) resume_response = self._put("jobs/%s/resume" % job_id) self._assert_status_code_is(resume_response, 200) self.dataset_populator.wait_for_history_jobs(history_id, assert_ok=True) dataset_details = self._get("histories/%s/contents/%s" % (history_id, output['id'])).json() assert dataset_details['state'] == 'ok' def _get_history_item_as_admin(self, history_id, item_id): response = self._get("histories/%s/contents/%s?view=detailed" % (history_id, item_id), admin=True) assert_status_code_is_ok(response) return response.json() @uses_test_history(require_new=True) def test_search(self, history_id): dataset_id = self.__history_with_ok_dataset(history_id) # We first copy the datasets, so that the update time is lower than the job creation time new_history_id = self.dataset_populator.new_history() copy_payload = { "content": dataset_id, "source": "hda", "type": "dataset" } copy_response = self._post("histories/%s/contents" % new_history_id, data=copy_payload) self._assert_status_code_is(copy_response, 200) inputs = json.dumps({'input1': {'src': 'hda', 'id': dataset_id}}) self._job_search(tool_id='cat1', history_id=history_id, inputs=inputs) # We test that a job can be found even if the dataset has been copied to another history new_dataset_id = copy_response.json()['id'] copied_inputs = json.dumps( {'input1': { 'src': 'hda', 'id': new_dataset_id }}) search_payload = self._search_payload(history_id=history_id, tool_id='cat1', inputs=copied_inputs) self._search(search_payload, expected_search_count=1) # Now we delete the original input HDA that was used -- we should still be able to find the job delete_respone = self._delete("histories/%s/contents/%s" % (history_id, dataset_id)) self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=1) # Now we also delete the copy -- we shouldn't find a job delete_respone = self._delete("histories/%s/contents/%s" % (new_history_id, new_dataset_id)) self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_handle_identifiers(self, history_id): # Test that input name and element identifier of a jobs' output must match for a job to be returned. dataset_id = self.__history_with_ok_dataset(history_id) inputs = json.dumps({'input1': {'src': 'hda', 'id': dataset_id}}) self._job_search(tool_id='identifier_single', history_id=history_id, inputs=inputs) dataset_details = self._get("histories/%s/contents/%s" % (history_id, dataset_id)).json() dataset_details['name'] = 'Renamed Test Dataset' dataset_update_response = self._put( "histories/%s/contents/%s" % (history_id, dataset_id), data=dict(name='Renamed Test Dataset')) self._assert_status_code_is(dataset_update_response, 200) assert dataset_update_response.json()['name'] == 'Renamed Test Dataset' search_payload = self._search_payload(history_id=history_id, tool_id='identifier_single', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_delete_outputs(self, history_id): dataset_id = self.__history_with_ok_dataset(history_id) inputs = json.dumps({'input1': {'src': 'hda', 'id': dataset_id}}) tool_response = self._job_search(tool_id='cat1', history_id=history_id, inputs=inputs) output_id = tool_response.json()['outputs'][0]['id'] delete_respone = self._delete("histories/%s/contents/%s" % (history_id, output_id)) self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload(history_id=history_id, tool_id='cat1', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_with_hdca_list_input(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type='list', history_id=history_id) list_id_b = self.__history_with_ok_collection(collection_type='list', history_id=history_id) inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': list_id_a }, 'f2': { 'src': 'hdca', 'id': list_id_b }, }) tool_response = self._job_search(tool_id='multi_data_param', history_id=history_id, inputs=inputs) # We switch the inputs, this should not return a match inputs_switched = json.dumps({ 'f2': { 'src': 'hdca', 'id': list_id_a }, 'f1': { 'src': 'hdca', 'id': list_id_b }, }) search_payload = self._search_payload(history_id=history_id, tool_id='multi_data_param', inputs=inputs_switched) self._search(search_payload, expected_search_count=0) # We delete the ouput (this is a HDA, as multi_data_param reduces collections) # and use the correct input job definition, the job should not be found output_id = tool_response.json()['outputs'][0]['id'] delete_respone = self._delete("histories/%s/contents/%s" % (history_id, output_id)) self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload(history_id=history_id, tool_id='multi_data_param', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_delete_hdca_output(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type='list', history_id=history_id) inputs = json.dumps({ 'input1': { 'src': 'hdca', 'id': list_id_a }, }) tool_response = self._job_search(tool_id='collection_creates_list', history_id=history_id, inputs=inputs) output_id = tool_response.json()['outputs'][0]['id'] # We delete a single tool output, no job should be returned delete_respone = self._delete("histories/%s/contents/%s" % (history_id, output_id)) self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload( history_id=history_id, tool_id='collection_creates_list', inputs=inputs) self._search(search_payload, expected_search_count=0) tool_response = self._job_search(tool_id='collection_creates_list', history_id=history_id, inputs=inputs) output_collection_id = tool_response.json( )['output_collections'][0]['id'] # We delete a collection output, no job should be returned delete_respone = self._delete( "histories/%s/contents/dataset_collections/%s" % (history_id, output_collection_id)) self._assert_status_code_is(delete_respone, 200) search_payload = self._search_payload( history_id=history_id, tool_id='collection_creates_list', inputs=inputs) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_with_hdca_pair_input(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type='pair', history_id=history_id) inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': list_id_a }, 'f2': { 'src': 'hdca', 'id': list_id_a }, }) self._job_search(tool_id='multi_data_param', history_id=history_id, inputs=inputs) # We test that a job can be found even if the collection has been copied to another history new_history_id = self.dataset_populator.new_history() copy_payload = { "content": list_id_a, "source": "hdca", "type": "dataset_collection" } copy_response = self._post("histories/%s/contents" % new_history_id, data=copy_payload) self._assert_status_code_is(copy_response, 200) new_list_a = copy_response.json()['id'] copied_inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': new_list_a }, 'f2': { 'src': 'hdca', 'id': new_list_a }, }) search_payload = self._search_payload(history_id=new_history_id, tool_id='multi_data_param', inputs=copied_inputs) self._search(search_payload, expected_search_count=1) # Now we delete the original input HDCA that was used -- we should still be able to find the job delete_respone = self._delete( "histories/%s/contents/dataset_collections/%s" % (history_id, list_id_a)) self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=1) # Now we also delete the copy -- we shouldn't find a job delete_respone = self._delete( "histories/%s/contents/dataset_collections/%s" % (history_id, new_list_a)) self._assert_status_code_is(delete_respone, 200) self._search(search_payload, expected_search_count=0) @uses_test_history(require_new=True) def test_search_with_hdca_list_pair_input(self, history_id): list_id_a = self.__history_with_ok_collection( collection_type='list:pair', history_id=history_id) inputs = json.dumps({ 'f1': { 'src': 'hdca', 'id': list_id_a }, 'f2': { 'src': 'hdca', 'id': list_id_a }, }) self._job_search(tool_id='multi_data_param', history_id=history_id, inputs=inputs) def _job_search(self, tool_id, history_id, inputs): search_payload = self._search_payload(history_id=history_id, tool_id=tool_id, inputs=inputs) empty_search_response = self._post("jobs/search", data=search_payload) self._assert_status_code_is(empty_search_response, 200) self.assertEqual(len(empty_search_response.json()), 0) tool_response = self._post("tools", data=search_payload) self.dataset_populator.wait_for_tool_run(history_id, run_response=tool_response) self._search(search_payload, expected_search_count=1) return tool_response def _search_payload(self, history_id, tool_id, inputs, state='ok'): search_payload = dict(tool_id=tool_id, inputs=inputs, history_id=history_id, state=state) return search_payload def _search(self, payload, expected_search_count=1): # in case job and history aren't updated at exactly the same # time give time to wait for _ in range(5): search_count = self._search_count(payload) if search_count == expected_search_count: break time.sleep(1) assert search_count == expected_search_count, "expected to find %d jobs, got %d jobs" % ( expected_search_count, search_count) return search_count def _search_count(self, search_payload): search_response = self._post("jobs/search", data=search_payload) self._assert_status_code_is(search_response, 200) search_json = search_response.json() return len(search_json) def __uploads_with_state(self, *states): jobs_response = self._get("jobs", data=dict(state=states)) self._assert_status_code_is(jobs_response, 200) jobs = jobs_response.json() assert not [j for j in jobs if not j['state'] in states] return [j for j in jobs if j['tool_id'] == 'upload1'] def __history_with_new_dataset(self, history_id): dataset_id = self.dataset_populator.new_dataset(history_id)["id"] return dataset_id def __history_with_ok_dataset(self, history_id): dataset_id = self.dataset_populator.new_dataset(history_id, wait=True)["id"] return dataset_id def __history_with_ok_collection(self, collection_type='list', history_id=None): if not history_id: history_id = self.dataset_populator.new_history() if collection_type == 'list': fetch_response = self.dataset_collection_populator.create_list_in_history( history_id, direct_upload=True).json() elif collection_type == 'pair': fetch_response = self.dataset_collection_populator.create_pair_in_history( history_id, direct_upload=True).json() elif collection_type == 'list:pair': fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history( history_id).json() self.dataset_collection_populator.wait_for_fetched_collection( fetch_response) return fetch_response["outputs"][0]['id'] def __jobs_index(self, **kwds): jobs_response = self._get("jobs", **kwds) self._assert_status_code_is(jobs_response, 200) jobs = jobs_response.json() assert isinstance(jobs, list) return jobs
def setUp(self): super(ExtendedMetadataMappingIntegrationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor)
class HistoriesApiTestCase(ApiTestCase): def setUp(self): super(HistoriesApiTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor) def test_create_history(self): # Create a history. create_response = self._create_history("TestHistory1") created_id = create_response["id"] # Make sure new history appears in index of user's histories. index_response = self._get("histories").json() indexed_history = [h for h in index_response if h["id"] == created_id][0] self.assertEqual(indexed_history["name"], "TestHistory1") def test_show_history(self): history_id = self._create_history("TestHistoryForShow")["id"] show_response = self._show(history_id) self._assert_has_key(show_response, 'id', 'name', 'annotation', 'size', 'contents_url', 'state', 'state_details', 'state_ids') state_details = show_response["state_details"] state_ids = show_response["state_ids"] states = [ 'discarded', 'empty', 'error', 'failed_metadata', 'new', 'ok', 'paused', 'queued', 'running', 'setting_metadata', 'upload' ] assert isinstance(state_details, dict) assert isinstance(state_ids, dict) self._assert_has_keys(state_details, *states) self._assert_has_keys(state_ids, *states) def test_show_most_recently_used(self): history_id = self._create_history("TestHistoryRecent")["id"] show_response = self._get("histories/most_recently_used").json() assert show_response["id"] == history_id def test_index_order(self): slightly_older_history_id = self._create_history( "TestHistorySlightlyOlder")["id"] newer_history_id = self._create_history("TestHistoryNewer")["id"] index_response = self._get("histories").json() assert index_response[0]["id"] == newer_history_id assert index_response[1]["id"] == slightly_older_history_id def test_delete(self): # Setup a history and ensure it is in the index history_id = self._create_history("TestHistoryForDelete")["id"] index_response = self._get("histories").json() assert index_response[0]["id"] == history_id show_response = self._show(history_id) assert not show_response["deleted"] # Delete the history self._delete("histories/%s" % history_id) # Check can view it - but it is deleted show_response = self._show(history_id) assert show_response["deleted"] # Verify it is dropped from history index index_response = self._get("histories").json() assert len( index_response) == 0 or index_response[0]["id"] != history_id # Add deleted filter to index to view it index_response = self._get("histories", {"deleted": "true"}).json() assert index_response[0]["id"] == history_id def test_purge(self): history_id = self._create_history("TestHistoryForPurge")["id"] data = {'purge': True} self._delete("histories/%s" % history_id, data=data) show_response = self._show(history_id) assert show_response["deleted"] assert show_response["purged"] def test_undelete(self): history_id = self._create_history( "TestHistoryForDeleteAndUndelete")["id"] self._delete("histories/%s" % history_id) self._post("histories/deleted/%s/undelete" % history_id) show_response = self._show(history_id) assert not show_response["deleted"] def test_update(self): history_id = self._create_history("TestHistoryForUpdating")["id"] self._update(history_id, {"name": "New Name"}) show_response = self._show(history_id) assert show_response["name"] == "New Name" unicode_name = u'桜ゲノム' self._update(history_id, {"name": unicode_name}) show_response = self._show(history_id) assert show_response["name"] == unicode_name, show_response quoted_name = "'MooCow'" self._update(history_id, {"name": quoted_name}) show_response = self._show(history_id) assert show_response["name"] == quoted_name self._update(history_id, {"deleted": True}) show_response = self._show(history_id) assert show_response["deleted"], show_response self._update(history_id, {"deleted": False}) show_response = self._show(history_id) assert not show_response["deleted"] self._update(history_id, {"published": True}) show_response = self._show(history_id) assert show_response["published"] self._update(history_id, {"genome_build": "hg18"}) show_response = self._show(history_id) assert show_response["genome_build"] == "hg18" self._update(history_id, {"annotation": "The annotation is cool"}) show_response = self._show(history_id) assert show_response["annotation"] == "The annotation is cool" self._update(history_id, {"annotation": unicode_name}) show_response = self._show(history_id) assert show_response["annotation"] == unicode_name, show_response self._update(history_id, {"annotation": quoted_name}) show_response = self._show(history_id) assert show_response["annotation"] == quoted_name def test_update_invalid_attribute(self): history_id = self._create_history( "TestHistoryForInvalidUpdating")["id"] put_response = self._update(history_id, {"invalidkey": "moo"}) assert "invalidkey" not in put_response.json() def test_update_invalid_types(self): history_id = self._create_history( "TestHistoryForUpdatingInvalidTypes")["id"] for str_key in ["name", "annotation"]: assert self._update(history_id, { str_key: False }).status_code == 400 for bool_key in ['deleted', 'importable', 'published']: assert self._update(history_id, { bool_key: "a string" }).status_code == 400 assert self._update(history_id, { "tags": "a simple string" }).status_code == 400 assert self._update(history_id, {"tags": [True]}).status_code == 400 def test_invalid_keys(self): invalid_history_id = "1234123412341234" assert self._get("histories/%s" % invalid_history_id).status_code == 400 assert self._update(invalid_history_id, { "name": "new name" }).status_code == 400 assert self._delete("histories/%s" % invalid_history_id).status_code == 400 assert self._post("histories/deleted/%s/undelete" % invalid_history_id).status_code == 400 def test_create_anonymous_fails(self): post_data = dict(name="CannotCreate") # Using lower-level _api_url will cause key to not be injected. histories_url = self._api_url("histories") create_response = post(url=histories_url, data=post_data) self._assert_status_code_is(create_response, 403) def test_import_export(self): history_name = "for_export_default" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_populator.new_dataset(history_id, content="1 2 3") deleted_hda = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True) self.dataset_populator.delete_dataset(history_id, deleted_hda["id"]) deleted_details = self.dataset_populator.get_history_dataset_details( history_id, id=deleted_hda["id"]) assert deleted_details["deleted"] imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2) def upload_job_check(job): assert job["tool_id"] == "upload1" def check_discarded(hda): assert hda["deleted"] assert hda["state"] == "discarded", hda assert hda["purged"] is True self._check_imported_dataset(history_id=imported_history_id, hid=1, job_checker=upload_job_check) self._check_imported_dataset(history_id=imported_history_id, hid=2, has_job=False, hda_checker=check_discarded, job_checker=upload_job_check) imported_content = self.dataset_populator.get_history_dataset_content( history_id=imported_history_id, hid=1, ) assert imported_content == "1 2 3\n" def test_import_1901_histories(self): f = open( self.test_data_resolver.get_filename( "exports/1901_two_datasets.tgz"), 'rb') import_data = dict(archive_source='', archive_file=f) self._import_history_and_wait(import_data, "API Test History", wait_on_history_length=2) def test_import_export_include_deleted(self): history_name = "for_export_include_deleted" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_populator.new_dataset(history_id, content="1 2 3") deleted_hda = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True) self.dataset_populator.delete_dataset(history_id, deleted_hda["id"]) imported_history_id = self._reimport_history( history_id, history_name, wait_on_history_length=2, export_kwds={"include_deleted": "True"}) self._assert_history_length(imported_history_id, 2) def upload_job_check(job): assert job["tool_id"] == "upload1" def check_deleted_not_purged(hda): assert hda["state"] == "ok", hda assert hda["deleted"] is True, hda assert hda["purged"] is False, hda self._check_imported_dataset(history_id=imported_history_id, hid=1, job_checker=upload_job_check) self._check_imported_dataset(history_id=imported_history_id, hid=2, hda_checker=check_deleted_not_purged, job_checker=upload_job_check) imported_content = self.dataset_populator.get_history_dataset_content( history_id=imported_history_id, hid=1, ) assert imported_content == "1 2 3\n" @skip_without_tool("job_properties") def test_import_export_failed_job(self): history_name = "for_export_include_failed_job" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_populator.run_tool('job_properties', inputs={'failbool': True}, history_id=history_id, assert_ok=False) self.dataset_populator.wait_for_history(history_id, assert_ok=False) imported_history_id = self._reimport_history( history_id, history_name, assert_ok=False, wait_on_history_length=4, export_kwds={"include_deleted": "True"}) self._assert_history_length(imported_history_id, 4) def check_failed(hda_or_job): print(hda_or_job) assert hda_or_job["state"] == "error", hda_or_job self.dataset_populator._summarize_history(imported_history_id) self._check_imported_dataset(history_id=imported_history_id, hid=1, assert_ok=False, hda_checker=check_failed, job_checker=check_failed) def test_import_metadata_regeneration(self): history_name = "for_import_metadata_regeneration" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_populator.new_dataset( history_id, content=open(self.test_data_resolver.get_filename("1.bam"), 'rb'), file_type='bam', wait=True) imported_history_id = self._reimport_history(history_id, history_name) self._assert_history_length(imported_history_id, 1) self._check_imported_dataset(history_id=imported_history_id, hid=1) import_bam_metadata = self.dataset_populator.get_history_dataset_details( history_id=imported_history_id, hid=1, ) # The cleanup() method of the __IMPORT_HISTORY__ job (which is executed # after the job has entered its final state): # - creates a new dataset with 'ok' state and adds it to the history # - starts a __SET_METADATA__ job to regenerate the dataset metadata, if # needed # We need to wait a bit for the creation of the __SET_METADATA__ job. time.sleep(1) self.dataset_populator.wait_for_history_jobs(imported_history_id, assert_ok=True) bai_metadata = import_bam_metadata["meta_files"][0] assert bai_metadata["file_type"] == "bam_index" api_url = bai_metadata["download_url"].split("api/", 1)[1] bai_response = self._get(api_url) self._assert_status_code_is(bai_response, 200) assert len(bai_response.content) > 4 def test_import_export_collection(self): history_name = "for_export_with_collections" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_collection_populator.create_list_in_history( history_id, contents=["Hello", "World"], direct_upload=True) imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3) self._assert_history_length(imported_history_id, 3) def check_elements(elements): assert len(elements) == 2 element0 = elements[0]["object"] element1 = elements[1]["object"] for element in [element0, element1]: assert not element["visible"] assert not element["deleted"] assert element["state"] == "ok" assert element0["hid"] == 2 assert element1["hid"] == 3 self._check_imported_collection(imported_history_id, hid=1, collection_type="list", elements_checker=check_elements) def test_import_export_nested_collection(self): history_name = "for_export_with_nested_collections" history_id = self.dataset_populator.new_history(name=history_name) self.dataset_collection_populator.create_list_of_pairs_in_history( history_id) imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3) self._assert_history_length(imported_history_id, 3) def check_elements(elements): assert len(elements) == 1 element0 = elements[0]["object"] self._assert_has_keys(element0, "elements", "collection_type") child_elements = element0["elements"] assert len(child_elements) == 2 assert element0["collection_type"] == "paired" self._check_imported_collection(imported_history_id, hid=1, collection_type="list:paired", elements_checker=check_elements) def _reimport_history(self, history_id, history_name, wait_on_history_length=None, assert_ok=True, export_kwds={}): # Ensure the history is ready to go... self.dataset_populator.wait_for_history(history_id, assert_ok=assert_ok) return self.dataset_populator.reimport_history( history_id, history_name, wait_on_history_length=wait_on_history_length, export_kwds=export_kwds, url=self.url, api_key=self.galaxy_interactor.api_key) def _import_history_and_wait(self, import_data, history_name, wait_on_history_length=None): imported_history_id = self.dataset_populator.import_history_and_wait_for_name( import_data, history_name) if wait_on_history_length: self.dataset_populator.wait_on_history_length( imported_history_id, wait_on_history_length) return imported_history_id def _assert_history_length(self, history_id, n): contents_response = self._get("histories/%s/contents" % history_id) self._assert_status_code_is(contents_response, 200) contents = contents_response.json() assert len(contents) == n, contents def _check_imported_dataset(self, history_id, hid, assert_ok=True, has_job=True, hda_checker=None, job_checker=None): imported_dataset_metadata = self.dataset_populator.get_history_dataset_details( history_id=history_id, hid=hid, assert_ok=assert_ok, ) assert imported_dataset_metadata["history_content_type"] == "dataset" assert imported_dataset_metadata["history_id"] == history_id if hda_checker is not None: hda_checker(imported_dataset_metadata) assert "creating_job" in imported_dataset_metadata job_id = imported_dataset_metadata["creating_job"] if has_job: assert job_id job_details = self.dataset_populator.get_job_details(job_id, full=True) assert job_details.status_code == 200, job_details.content job = job_details.json() assert 'history_id' in job, job assert job['history_id'] == history_id, job if job_checker is not None: job_checker(job) def _check_imported_collection(self, history_id, hid, collection_type=None, elements_checker=None): imported_collection_metadata = self.dataset_populator.get_history_collection_details( history_id=history_id, hid=hid, ) assert imported_collection_metadata[ "history_content_type"] == "dataset_collection" assert imported_collection_metadata["history_id"] == history_id assert "collection_type" in imported_collection_metadata assert "elements" in imported_collection_metadata if collection_type is not None: assert imported_collection_metadata[ "collection_type"] == collection_type, imported_collection_metadata if elements_checker is not None: elements_checker(imported_collection_metadata["elements"]) def test_create_tag(self): post_data = dict(name="TestHistoryForTag") history_id = self._post("histories", data=post_data).json()["id"] tag_data = dict(value="awesometagvalue") tag_url = "histories/%s/tags/awesometagname" % history_id tag_create_response = self._post(tag_url, data=tag_data) self._assert_status_code_is(tag_create_response, 200) def _show(self, history_id): return self._get("histories/%s" % history_id).json() def _update(self, history_id, data): update_url = self._api_url("histories/%s" % history_id, use_key=True) put_response = put(update_url, json=data) return put_response def _create_history(self, name): post_data = dict(name=name) create_response = self._post("histories", data=post_data).json() self._assert_has_keys(create_response, "name", "id") self.assertEqual(create_response["name"], name) return create_response