def test_export_copied_collection(): app, sa_session, h = _setup_history_for_export("Collection History with copied collection") d1, d2 = _create_datasets(sa_session, h, 2) c1 = model.DatasetCollection(collection_type="paired") hc1 = model.HistoryDatasetCollectionAssociation(history=h, hid=3, collection=c1, name="HistoryCollectionTest1") h.hid_counter = 4 dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) sa_session.add_all((dce1, dce2, d1, d2, hc1)) sa_session.flush() hc2 = hc1.copy(element_destination=h) h.add_pending_items() assert h.hid_counter == 7 sa_session.add(hc2) sa_session.flush() assert hc2.copied_from_history_dataset_collection_association == hc1 imported_history = _import_export(app, h) assert imported_history.hid_counter == 7 assert len(imported_history.dataset_collections) == 2 assert len(imported_history.datasets) == 4 _assert_distinct_hids(imported_history) imported_by_hid = _hid_dict(imported_history) assert imported_by_hid[4].copied_from_history_dataset_association == imported_by_hid[1] assert imported_by_hid[5].copied_from_history_dataset_association == imported_by_hid[2] assert imported_by_hid[6].copied_from_history_dataset_collection_association == imported_by_hid[3]
def __assert_output_format_is(expected, output, input_extensions=[], param_context=[], add_collection=False): inputs = {} last_ext = "data" i = 1 for name, ext in input_extensions: hda = model.HistoryDatasetAssociation(extension=ext) hda.metadata.random_field = str(i) # Populate a random metadata field for testing inputs[name] = hda last_ext = ext i += 1 input_collections = {} if add_collection: hda_forward = model.HistoryDatasetAssociation(extension="txt") hda_reverse = model.HistoryDatasetAssociation(extension="txt") c1 = model.DatasetCollection(collection_type="pair") hc1 = model.HistoryDatasetCollectionAssociation(collection=c1, name="HistoryCollectionTest1") dce1 = model.DatasetCollectionElement(collection=c1, element=hda_forward, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=hda_reverse, element_identifier="reverse", element_index=1) c1.elements = [dce1, dce2] input_collections["hdcai"] = [(hc1, False)] actual_format = determine_output_format(output, param_context, inputs, input_collections, last_ext) assert actual_format == expected, "Actual format %s, does not match expected %s" % (actual_format, expected)
def test_export_collection_hids(): app, sa_session, h = _setup_history_for_export("Collection History with dataset from this history") d1, d2 = _create_datasets(sa_session, h, 2) c1 = model.DatasetCollection(collection_type="paired") hc1 = model.HistoryDatasetCollectionAssociation(history=h, hid=3, collection=c1, name="HistoryCollectionTest1") h.hid_counter = 4 dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) sa_session.add(dce1) sa_session.add(dce2) sa_session.add(d1) sa_session.add(d2) sa_session.add(hc1) sa_session.flush() imported_history = _import_export(app, h) assert imported_history.hid_counter == 4, imported_history.hid_counter assert len(imported_history.dataset_collections) == 1 assert len(imported_history.datasets) == 2 for hdca in imported_history.dataset_collections: assert hdca.hid == 3, hdca.hid for hda in imported_history.datasets: assert hda.hid in [1, 2], hda.hid _assert_distinct_hids(imported_history)
def test_export_collection_with_copied_datasets_and_overlapping_hids(): app, sa_session, h = _setup_history_for_export("Collection History with dataset from other history") dataset_history = model.History(name="Dataset History", user=h.user) d1, d2 = _create_datasets(sa_session, dataset_history, 2) sa_session.add(d1) sa_session.add(d2) sa_session.add(dataset_history) sa_session.flush() app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True) app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True) d1_copy = d1.copy() d2_copy = d2.copy() d1_copy.history = h d2_copy.history = h c1 = model.DatasetCollection(collection_type="paired") hc1 = model.HistoryDatasetCollectionAssociation(history=h, hid=3, collection=c1, name="HistoryCollectionTest1") h.hid_counter = 5 dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) sa_session.add(dce1) sa_session.add(dce2) sa_session.add(d1_copy) sa_session.add(d2_copy) sa_session.add(hc1) sa_session.flush() _import_export(app, h)
def test_collections_in_histories(self): model = self.model u = model.User(email="*****@*****.**", password="******") h1 = model.History(name="History 1", user=u) d1 = model.HistoryDatasetAssociation(extension="txt", history=h1, create_dataset=True, sa_session=model.session) d2 = model.HistoryDatasetAssociation(extension="txt", history=h1, create_dataset=True, sa_session=model.session) c1 = model.DatasetCollection(collection_type="pair") hc1 = model.HistoryDatasetCollectionAssociation( history=h1, collection=c1, name="HistoryCollectionTest1") dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="left") dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="right") self.persist(u, h1, d1, d2, c1, hc1, dce1, dce2) loaded_dataset_collection = self.query( model.HistoryDatasetCollectionAssociation).filter( model.HistoryDatasetCollectionAssociation.name == "HistoryCollectionTest1").first().collection self.assertEqual(len(loaded_dataset_collection.elements), 2) assert loaded_dataset_collection.collection_type == "pair" assert loaded_dataset_collection["left"] == dce1 assert loaded_dataset_collection["right"] == dce2
def _create_output_dataset_collection(self, **kwd): output_dataset_collection = model.HistoryDatasetCollectionAssociation( **kwd) self.history.add_dataset_collection(output_dataset_collection) assert output_dataset_collection.collection self.app.model.session.add(output_dataset_collection) self.app.model.session.flush() return output_dataset_collection
def test_annotations(self): model = self.model u = model.User(email="*****@*****.**", password="******") self.persist(u) def persist_and_check_annotation(annotation_class, **kwds): annotated_association = annotation_class() annotated_association.annotation = "Test Annotation" annotated_association.user = u for key, value in kwds.items(): setattr(annotated_association, key, value) self.persist(annotated_association) self.expunge() stored_annotation = self.query(annotation_class).all()[0] assert stored_annotation.annotation == "Test Annotation" assert stored_annotation.user.email == "*****@*****.**" sw = model.StoredWorkflow() sw.user = u self.persist(sw) persist_and_check_annotation(model.StoredWorkflowAnnotationAssociation, stored_workflow=sw) workflow = model.Workflow() workflow.stored_workflow = sw self.persist(workflow) ws = model.WorkflowStep() ws.workflow = workflow self.persist(ws) persist_and_check_annotation(model.WorkflowStepAnnotationAssociation, workflow_step=ws) h = model.History(name="History for Annotation", user=u) self.persist(h) persist_and_check_annotation(model.HistoryAnnotationAssociation, history=h) d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) self.persist(d1) persist_and_check_annotation(model.HistoryDatasetAssociationAnnotationAssociation, hda=d1) page = model.Page() page.user = u self.persist(page) persist_and_check_annotation(model.PageAnnotationAssociation, page=page) visualization = model.Visualization() visualization.user = u self.persist(visualization) persist_and_check_annotation(model.VisualizationAnnotationAssociation, visualization=visualization) dataset_collection = model.DatasetCollection(collection_type="paired") history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) self.persist(history_dataset_collection) persist_and_check_annotation(model.HistoryDatasetCollectionAssociationAnnotationAssociation, history_dataset_collection=history_dataset_collection) library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) self.persist(library_dataset_collection) persist_and_check_annotation(model.LibraryDatasetCollectionAnnotationAssociation, library_dataset_collection=library_dataset_collection)
def test_export_copied_objects_copied_outside_history(): app, sa_session, h = _setup_history_for_export("Collection History with copied objects") d1, d2 = _create_datasets(sa_session, h, 2) c1 = model.DatasetCollection(collection_type="paired") hc1 = model.HistoryDatasetCollectionAssociation(history=h, hid=3, collection=c1, name="HistoryCollectionTest1") h.hid_counter = 4 dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) sa_session.add_all((dce1, dce2, d1, d2, hc1)) sa_session.flush() hc2 = hc1.copy(element_destination=h) h.add_dataset_collection(hc2) sa_session.add(hc2) other_h = model.History(name=h.name + "-other", user=h.user) sa_session.add(other_h) hc3 = hc2.copy(element_destination=other_h) other_h.add_dataset_collection(hc3) sa_session.add(hc3) sa_session.flush() hc4 = hc3.copy(element_destination=h) h.add_dataset_collection(hc4) sa_session.add(hc4) sa_session.flush() assert h.hid_counter == 10 original_by_hid = _hid_dict(h) assert original_by_hid[7].copied_from_history_dataset_association != original_by_hid[4] assert original_by_hid[8].copied_from_history_dataset_association != original_by_hid[5] assert original_by_hid[9].copied_from_history_dataset_collection_association != original_by_hid[6] imported_history = _import_export(app, h) assert imported_history.hid_counter == 10 assert len(imported_history.dataset_collections) == 3 assert len(imported_history.datasets) == 6 _assert_distinct_hids(imported_history) imported_by_hid = _hid_dict(imported_history) assert imported_by_hid[4].copied_from_history_dataset_association == imported_by_hid[1] assert imported_by_hid[5].copied_from_history_dataset_association == imported_by_hid[2] assert imported_by_hid[6].copied_from_history_dataset_collection_association == imported_by_hid[3] assert imported_by_hid[7].copied_from_history_dataset_association == imported_by_hid[4] assert imported_by_hid[8].copied_from_history_dataset_association == imported_by_hid[5] assert imported_by_hid[9].copied_from_history_dataset_collection_association == imported_by_hid[6]
def test_history_collection_copy(list_size=NUM_DATASETS): with _setup_mapping_and_user() as (test_config, object_store, model, old_history): for i in range(NUM_COLLECTIONS): hdas = [] for i in range(list_size * 2): hda_path = test_config.write("moo", "test_metadata_original_%d" % i) hda = _create_hda(model, object_store, old_history, hda_path, visible=False, include_metadata_file=False) hdas.append(hda) list_elements = [] list_collection = model.DatasetCollection(collection_type="list:paired") for j in range(list_size): paired_collection = model.DatasetCollection(collection_type="paired") forward_dce = model.DatasetCollectionElement(collection=paired_collection, element=hdas[j * 2]) reverse_dce = model.DatasetCollectionElement(collection=paired_collection, element=hdas[j * 2 + 1]) paired_collection.elements = [forward_dce, reverse_dce] paired_collection_element = model.DatasetCollectionElement(collection=list_collection, element=paired_collection) list_elements.append(paired_collection_element) model.context.add_all([forward_dce, reverse_dce, paired_collection_element]) list_collection.elements = list_elements history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=list_collection) history_dataset_collection.user = old_history.user model.context.add(history_dataset_collection) model.context.flush() old_history.add_dataset_collection(history_dataset_collection) history_dataset_collection.add_item_annotation(model.context, old_history.user, history_dataset_collection, "annotation #%d" % history_dataset_collection.hid) model.context.flush() annotation_str = history_dataset_collection.get_item_annotation_str(model.context, old_history.user, history_dataset_collection) # Saving magic SA invocations for detecting full flushes that may harm performance. # from sqlalchemy import event # @event.listens_for(model.context, "before_flush") # def track_instances_before_flush(session, context, instances): # if not instances: # print("FULL FLUSH...") # else: # print("Flushing just %s" % instances) history_copy_timer = ExecutionTimer() new_history = old_history.copy(target_user=old_history.user) print("history copied %s" % history_copy_timer) for i, hda in enumerate(new_history.active_datasets): assert hda.get_size() == 3 annotation_str = hda.get_item_annotation_str(model.context, old_history.user, hda) assert annotation_str == "annotation #%d" % hda.hid, annotation_str assert len(new_history.active_dataset_collections) == NUM_COLLECTIONS for hdca in new_history.active_dataset_collections: annotation_str = hdca.get_item_annotation_str(model.context, old_history.user, hdca) assert annotation_str == "annotation #%d" % hdca.hid, annotation_str
def _create_instance_for_collection(self, trans, parent, name, dataset_collection, implicit_output_name=None, implicit_inputs=None, tags=None, set_hid=True, flush=True): if isinstance(parent, model.History): dataset_collection_instance = model.HistoryDatasetCollectionAssociation( collection=dataset_collection, name=name, ) if implicit_inputs: for input_name, input_collection in implicit_inputs: dataset_collection_instance.add_implicit_input_collection( input_name, input_collection) if implicit_output_name: dataset_collection_instance.implicit_output_name = implicit_output_name log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements))) if set_hid: parent.add_dataset_collection(dataset_collection_instance) elif isinstance(parent, model.LibraryFolder): dataset_collection_instance = model.LibraryDatasetCollectionAssociation( collection=dataset_collection, folder=parent, name=name, ) else: message = "Internal logic error - create called with unknown parent type %s" % type( parent) log.exception(message) raise MessageException(message) # Tags may be coming in as a dictionary of tag model objects if copying them from other # existing Galaxy objects or as a list of strings if the tags are coming from user supplied # values. if isinstance(tags, list): assert implicit_inputs is None, implicit_inputs tags = self.tag_handler.add_tags_from_list( trans.user, dataset_collection_instance, tags) else: tags = self._append_tags(dataset_collection_instance, implicit_inputs, tags) return self.__persist(dataset_collection_instance, flush=flush)
def test_dataset_dbkeys_and_extensions_summary(self): model = self.model u = model.User(email="*****@*****.**", password="******") h1 = model.History(name="History 1", user=u) d1 = model.HistoryDatasetAssociation(extension="bam", dbkey="hg19", history=h1, create_dataset=True, sa_session=model.session) d2 = model.HistoryDatasetAssociation(extension="txt", dbkey="hg19", history=h1, create_dataset=True, sa_session=model.session) c1 = model.DatasetCollection(collection_type='paired') dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) hdca = model.HistoryDatasetCollectionAssociation(collection=c1, history=h1) model.session.add_all([d1, d2, c1, dce1, dce2, hdca]) model.session.flush() assert hdca.dataset_dbkeys_and_extensions_summary[0] == {"hg19"} assert hdca.dataset_dbkeys_and_extensions_summary[1] == {"bam", "txt"}
def test_ratings(self): model = self.model u = model.User(email="*****@*****.**", password="******") self.persist(u) def persist_and_check_rating(rating_class, **kwds): rating_association = rating_class() rating_association.rating = 5 rating_association.user = u for key, value in kwds.items(): setattr(rating_association, key, value) self.persist(rating_association) self.expunge() stored_annotation = self.query(rating_class).all()[0] assert stored_annotation.rating == 5 assert stored_annotation.user.email == "*****@*****.**" sw = model.StoredWorkflow() sw.user = u self.persist(sw) persist_and_check_rating(model.StoredWorkflowRatingAssociation, stored_workflow=sw) h = model.History(name="History for Rating", user=u) self.persist(h) persist_and_check_rating(model.HistoryRatingAssociation, history=h) d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) self.persist(d1) persist_and_check_rating(model.HistoryDatasetAssociationRatingAssociation, hda=d1) page = model.Page() page.user = u self.persist(page) persist_and_check_rating(model.PageRatingAssociation, page=page) visualization = model.Visualization() visualization.user = u self.persist(visualization) persist_and_check_rating(model.VisualizationRatingAssociation, visualization=visualization) dataset_collection = model.DatasetCollection(collection_type="paired") history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) self.persist(history_dataset_collection) persist_and_check_rating(model.HistoryDatasetCollectionRatingAssociation, history_dataset_collection=history_dataset_collection) library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) self.persist(library_dataset_collection) persist_and_check_rating(model.LibraryDatasetCollectionRatingAssociation, library_dataset_collection=library_dataset_collection)
def test_ratings(self): model = self.model user_email = "*****@*****.**" u = model.User(email=user_email, password="******") self.persist(u) def persist_and_check_rating(rating_class, item): rating = 5 rating_association = rating_class(u, item, rating) self.persist(rating_association) self.expunge() stored_rating = self.query(rating_class).all()[0] assert stored_rating.rating == rating assert stored_rating.user.email == user_email sw = model.StoredWorkflow() sw.user = u self.persist(sw) persist_and_check_rating(model.StoredWorkflowRatingAssociation, sw) h = model.History(name="History for Rating", user=u) self.persist(h) persist_and_check_rating(model.HistoryRatingAssociation, h) d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) self.persist(d1) persist_and_check_rating(model.HistoryDatasetAssociationRatingAssociation, d1) page = model.Page() page.user = u self.persist(page) persist_and_check_rating(model.PageRatingAssociation, page) visualization = model.Visualization() visualization.user = u self.persist(visualization) persist_and_check_rating(model.VisualizationRatingAssociation, visualization) dataset_collection = model.DatasetCollection(collection_type="paired") history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) self.persist(history_dataset_collection) persist_and_check_rating(model.HistoryDatasetCollectionRatingAssociation, history_dataset_collection) library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) self.persist(library_dataset_collection) persist_and_check_rating(model.LibraryDatasetCollectionRatingAssociation, library_dataset_collection)
def test_history_collection_paired(self): hdca = model.HistoryDatasetCollectionAssociation() hdca.name = "cool name" hdca.collection = self._new_pair_collection() hdca.id = 1 self.trans.app.dataset_collections_service.get_dataset_collection_instance.return_value = hdca example = """# Example ```galaxy history_dataset_collection_display(history_dataset_collection_id=1) ``` """ result = self._to_basic(example) assert "**Dataset Collection:** cool name\n" in result assert "**Element:** forward" in result, result assert "**Element Contents:**\n" in result assert "\n Forward dataset.\n" in result assert "**Element:** reverse" in result, result assert "\n Reverse dataset.\n" in result
def test_export_dataset_collection_paired(self): hdca = model.HistoryDatasetCollectionAssociation() hdca.name = "cool name" hdca.collection = self._new_pair_collection() hdca.id = 1 hdca.history_id = 1 hdca.collection_id = hdca.collection.id self.trans.app.dataset_collections_service.get_dataset_collection_instance.return_value = hdca example = """# Example ```galaxy history_dataset_collection_display(history_dataset_collection_id=1) ``` """ # Patch out url_for since we haven't initialized an app. from galaxy.managers.hdcas import HDCASerializer with mock.patch.object(HDCASerializer, 'url_for', return_value="http://google.com"): export, extra_data = self._ready_export(example) assert "history_dataset_collections" in extra_data assert len(extra_data.get("history_dataset_collections")) == 1
def test_tags(self): model = self.model my_tag = model.Tag(name="Test Tag") u = model.User(email="*****@*****.**", password="******") self.persist(my_tag, u) def tag_and_test(taggable_object, tag_association_class, backref_name): assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 0 tag_association = tag_association_class() tag_association.tag = my_tag taggable_object.tags = [tag_association] self.persist(tag_association, taggable_object) assert len(getattr(self.query(model.Tag).filter(model.Tag.name == "Test Tag").all()[0], backref_name)) == 1 sw = model.StoredWorkflow() sw.user = u tag_and_test(sw, model.StoredWorkflowTagAssociation, "tagged_workflows") h = model.History(name="History for Tagging", user=u) tag_and_test(h, model.HistoryTagAssociation, "tagged_histories") d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) tag_and_test(d1, model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations") page = model.Page() page.user = u tag_and_test(page, model.PageTagAssociation, "tagged_pages") visualization = model.Visualization() visualization.user = u tag_and_test(visualization, model.VisualizationTagAssociation, "tagged_visualizations") dataset_collection = model.DatasetCollection(collection_type="paired") history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) tag_and_test(history_dataset_collection, model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) tag_and_test(library_dataset_collection, model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections")
def test_export_collection_history(): app, sa_session, h = _setup_history_for_export("Collection History") d1, d2, d3, d4 = _create_datasets(sa_session, h, 4) c1 = model.DatasetCollection(collection_type="paired") hc1 = model.HistoryDatasetCollectionAssociation( history=h, hid=1, collection=c1, name="HistoryCollectionTest1") dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) c2 = model.DatasetCollection(collection_type="list:paired") hc2 = model.HistoryDatasetCollectionAssociation( history=h, hid=2, collection=c2, name="HistoryCollectionTest2") cleaf = model.DatasetCollection(collection_type="paired") dce2leaf1 = model.DatasetCollectionElement(collection=cleaf, element=d3, element_identifier="forward", element_index=0) dce2leaf2 = model.DatasetCollectionElement(collection=cleaf, element=d4, element_identifier="reverse", element_index=1) dce21 = model.DatasetCollectionElement(collection=c2, element=cleaf, element_identifier="listel", element_index=0) j = model.Job() j.user = h.user j.tool_id = "cat1" j.add_input_dataset_collection("input1_collect", hc1) j.add_output_dataset_collection("output_collect", hc2) sa_session.add(dce1) sa_session.add(dce2) sa_session.add(dce21) sa_session.add(dce2leaf1) sa_session.add(dce2leaf2) sa_session.add(hc1) sa_session.add(hc2) sa_session.add(j) sa_session.flush() imported_history = _import_export(app, h) datasets = imported_history.datasets assert len(datasets) == 4 dataset_collections = list( imported_history.contents_iter(types=["dataset_collection"])) assert len(dataset_collections) == 2 imported_hdca1 = dataset_collections[0] imported_hdca2 = dataset_collections[1] imported_collection_2 = imported_hdca2.collection assert imported_hdca1.collection.collection_type == "paired" assert imported_collection_2.collection_type == "list:paired" assert len(imported_collection_2.elements) == 1 imported_top_level_element = imported_collection_2.elements[0] assert imported_top_level_element.element_identifier == "listel", imported_top_level_element.element_identifier assert imported_top_level_element.element_index == 0, imported_top_level_element.element_index imported_nested_collection = imported_top_level_element.child_collection assert len(imported_nested_collection.elements) == 2 assert imported_nested_collection.collection_type == "paired", imported_nested_collection.collection_type assert len(imported_history.jobs) == 1 imported_job = imported_history.jobs[0] assert imported_job assert len(imported_job.input_dataset_collections) == 1, len( imported_job.input_dataset_collections) assert len(imported_job.output_dataset_collection_instances) == 1 assert imported_job.id != j.id
def test_import_export_edit_collection(): """Test modifying existing collections with imports.""" app = _mock_app() sa_session = app.model.context u = model.User(email="*****@*****.**", password="******") h = model.History(name="Test History", user=u) c1 = model.DatasetCollection(collection_type="list", populated=False) hc1 = model.HistoryDatasetCollectionAssociation( history=h, hid=1, collection=c1, name="HistoryCollectionTest1") sa_session.add(hc1) sa_session.add(h) sa_session.flush() import_history = model.History(name="Test History for Import", user=u) sa_session.add(import_history) temp_directory = mkdtemp() with store.DirectoryModelExportStore(temp_directory, app=app, for_edit=True) as export_store: export_store.add_dataset_collection(hc1) # Fabric editing metadata for collection... collections_metadata_path = os.path.join(temp_directory, store.ATTRS_FILENAME_COLLECTIONS) datasets_metadata_path = os.path.join(temp_directory, store.ATTRS_FILENAME_DATASETS) with open(collections_metadata_path, "r") as f: hdcas_metadata = json.load(f) assert len(hdcas_metadata) == 1 hdca_metadata = hdcas_metadata[0] assert hdca_metadata assert "id" in hdca_metadata assert "collection" in hdca_metadata collection_metadata = hdca_metadata["collection"] assert "populated_state" in collection_metadata assert collection_metadata[ "populated_state"] == model.DatasetCollection.populated_states.NEW collection_metadata[ "populated_state"] = model.DatasetCollection.populated_states.OK d1 = model.HistoryDatasetAssociation(extension="txt", create_dataset=True, flush=False) d1.hid = 1 d2 = model.HistoryDatasetAssociation(extension="txt", create_dataset=True, flush=False) d2.hid = 2 serialization_options = model.SerializationOptions(for_edit=True) dataset_list = [ d1.serialize(app.security, serialization_options), d2.serialize(app.security, serialization_options) ] dc = model.DatasetCollection( id=collection_metadata["id"], collection_type="list", element_count=2, ) dc.populated_state = model.DatasetCollection.populated_states.OK dce1 = model.DatasetCollectionElement( element=d1, element_index=0, element_identifier="first", ) dce2 = model.DatasetCollectionElement( element=d2, element_index=1, element_identifier="second", ) dc.elements = [dce1, dce2] with open(datasets_metadata_path, "w") as datasets_f: json.dump(dataset_list, datasets_f) hdca_metadata["collection"] = dc.serialize(app.security, serialization_options) with open(collections_metadata_path, "w") as collections_f: json.dump(hdcas_metadata, collections_f) _perform_import_from_directory(temp_directory, app, u, import_history, store.ImportOptions(allow_edit=True)) sa_session.refresh(c1) assert c1.populated_state == model.DatasetCollection.populated_states.OK, c1.populated_state assert len(c1.elements) == 2
def test_export_collection_with_mapping_history(): app, sa_session, h = _setup_history_for_export( "Collection Mapping History") d1, d2, d3, d4 = _create_datasets(sa_session, h, 4) c1 = model.DatasetCollection(collection_type="list") hc1 = model.HistoryDatasetCollectionAssociation( history=h, hid=1, collection=c1, name="HistoryCollectionTest1") dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="el1", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="el2", element_index=1) c2 = model.DatasetCollection(collection_type="list") hc2 = model.HistoryDatasetCollectionAssociation( history=h, hid=2, collection=c2, name="HistoryCollectionTest2") dce3 = model.DatasetCollectionElement(collection=c2, element=d3, element_identifier="el1", element_index=0) dce4 = model.DatasetCollectionElement(collection=c2, element=d4, element_identifier="el2", element_index=1) hc2.add_implicit_input_collection("input1", hc1) j1 = model.Job() j1.user = h.user j1.tool_id = "cat1" j1.add_input_dataset("input1", d1) j1.add_output_dataset("out_file1", d3) j2 = model.Job() j2.user = h.user j2.tool_id = "cat1" j2.add_input_dataset("input1", d2) j2.add_output_dataset("out_file1", d4) sa_session.add(dce1) sa_session.add(dce2) sa_session.add(dce3) sa_session.add(dce4) sa_session.add(hc1) sa_session.add(hc2) sa_session.add(j1) sa_session.add(j2) sa_session.flush() implicit_collection_jobs = model.ImplicitCollectionJobs() j1.add_output_dataset_collection("out_file1", hc2) # really? ija1 = model.ImplicitCollectionJobsJobAssociation() ija1.order_index = 0 ija1.implicit_collection_jobs = implicit_collection_jobs ija1.job = j1 j2.add_output_dataset_collection("out_file1", hc2) # really? ija2 = model.ImplicitCollectionJobsJobAssociation() ija2.order_index = 1 ija2.implicit_collection_jobs = implicit_collection_jobs ija2.job = j2 sa_session.add(implicit_collection_jobs) sa_session.add(ija1) sa_session.add(ija2) sa_session.flush() imported_history = _import_export(app, h) assert len(imported_history.jobs) == 2 imported_job0 = imported_history.jobs[0] imported_icj = imported_job0.implicit_collection_jobs_association.implicit_collection_jobs assert imported_icj assert len(imported_icj.jobs) == 2, len(imported_icj.jobs)