def run(self): print("Starting MultiDatasetBenchmarkTool...", flush=True) PathBuilder.build(self.result_path) specs = self._split_specs_file() self._extract_reports() instruction_states = {} for index, specs_name in enumerate(specs.keys()): print( f"Running nested cross-validation on dataset {specs_name} ({index+1}/{len(list(specs.keys()))})..", flush=True) app = ImmuneMLApp(specification_path=specs[specs_name], result_path=f"{self.result_path}/{specs_name}/") instruction_states[specs_name] = app.run()[0] print( f"Finished nested cross-validation on dataset {specs_name} ({index+1}/{len(list(specs.keys()))})..", flush=True) print( "Running reports on the results of nested cross-validation on all datasets...", flush=True) report_results = self._run_reports(instruction_states) print("Finished reports, now generating HTML output...", flush=True) MultiDatasetBenchmarkHTMLBuilder.build( report_results, self.result_path, { specs_name: f"{self.result_path}/{specs_name}/" for specs_name in specs.keys() }) print("MultiDatasetBenchmarkTool finished.", flush=True)
def run(self, result_path: str): result_path = self.build_path(result_path) specs_file = self.create_specfication(result_path) app = ImmuneMLApp(specs_file, result_path + "quickstart/") app.run()
def _run(self): PathBuilder.build(self.result_path) self.update_specs() app = ImmuneMLApp(self.yaml_path, self.result_path) output_file_path = app.run() return output_file_path
def _run(self): yaml_path = main(self.args) PathBuilder.build(self.result_path) app = ImmuneMLApp(yaml_path, self.result_path) output_file_path = app.run() return output_file_path
def run_example(self, specs: dict, path: str): PathBuilder.build(path) specs_filename = f"{path}specs.yaml" with open(specs_filename, "w") as file: yaml.dump(specs, file) app = ImmuneMLApp(specs_filename, path + "result/") app.run() shutil.rmtree(path)
def test_dataset_generation(self): path = PathBuilder.build(EnvironmentSettings.tmp_test_path + "cv_split_variant/") repertoire_specs = self.build_specs(path) specs_filename = f"{path}specs.yaml" with open(specs_filename, "w") as file: yaml.dump(repertoire_specs, file) app = ImmuneMLApp(specs_filename, path + "result/") app.run() shutil.rmtree(path)
def test_subsampling(self): import faulthandler faulthandler.enable() path = PathBuilder.build(EnvironmentSettings.tmp_test_path + "subsampling_workflow/") repertoire_specs = self.build_specs(path) specs_filename = f"{path}specs.yaml" with open(specs_filename, "w") as file: yaml.dump(repertoire_specs, file) app = ImmuneMLApp(specs_filename, path + "result/") app.run() shutil.rmtree(path)
def test(self): path = PathBuilder.build(EnvironmentSettings.tmp_test_path + "integration_dataset_gen_html/") dataset_path = f"{path}initial_dataset/" specs = { "definitions": { "datasets": { "d1": { "format": "RandomRepertoireDataset", "params": { "repertoire_count": 10, "sequence_count_probabilities": { 10: 1 }, "sequence_length_probabilities": { 12: 1 }, "labels": {}, "result_path": dataset_path } } } }, "instructions": { "instr1": { "type": "DatasetExport", "export_formats": ["Pickle", "AIRR"], "datasets": ["d1"] } }, "output": { "format": "HTML" } } specs_path = f"{path}specs.yaml" with open(specs_path, "w") as file: yaml.dump(specs, file) app = ImmuneMLApp(specs_path, path + "result/") app.run() shutil.rmtree(path)
def _run(self): PathBuilder.build(self.result_path) self._prepare_specs() app = ImmuneMLApp(self.yaml_path, self.result_path) app.run() model_locations = list( glob(self.result_path + f"/{self.instruction_name}/optimal_*/zip/*.zip")) model_export_path = PathBuilder.build(self.result_path + 'exported_models/') for model_location in model_locations: shutil.copyfile( model_location, model_export_path + os.path.basename(model_location)) logging.info( f"{GalaxyTrainMLModel.__name__}: immuneML has finished and the trained models were exported." )
def test_simulation(self): path = EnvironmentSettings.tmp_test_path + "integration_simulation/" self.prepare_dataset(path) specs_path = self.prepare_specs(path) PathBuilder.build(path + "result/") app = ImmuneMLApp(specification_path=specs_path, result_path=path + "result/") app.run() self.assertTrue(os.path.isfile(path + "result/inst1/metadata.csv")) metadata_df = pd.read_csv(path + "result/inst1/metadata.csv", comment=Constants.COMMENT_SIGN) self.assertTrue("signal_signal1" in metadata_df.columns) self.assertEqual(17, sum(metadata_df["signal_signal1"])) self.assertTrue(os.path.isfile(path + "result/index.html")) self.assertTrue( os.path.isfile( path + "result/inst1/exported_dataset/pickle/d1.iml_dataset")) shutil.rmtree(path)
def test_ml(self): path = PathBuilder.build(EnvironmentSettings.tmp_test_path + "integration_ml/") specs_path = self.prepare_specs(path) PathBuilder.build(path + "result_export/") app = ImmuneMLApp(specification_path=specs_path, result_path=path + "result_export/") states = app.run() self.assertTrue(os.path.isfile(path + "result_export/index.html")) specs_path = self.prepare_import_specs(path) app = ImmuneMLApp(specs_path, path + 'result_import/') result_path = app.run() self.assertTrue(os.path.isfile(path + "result_import/index.html")) shutil.rmtree(path)
def test_encoding(self): path = EnvironmentSettings.tmp_test_path + "integration_test_emerson_encoding/" PathBuilder.build(path) ref_path = path + "reference.csv" pd.DataFrame({ "sequence_aas": ["GGG", "III", "TTT", "EFEF"], "v_alleles": ["TRBV6-1*01", "TRBV6-1*01", "TRBV6-1*01", "TRBV6-1*01"], 'j_alleles': ["TRBJ2-7", "TRBJ2-7", "TRBJ2-7", "TRBJ2-7"] }).to_csv(ref_path, index=False) repertoires, metadata = RepertoireBuilder.build( [["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF", "III", "LLL", "MMM"], ["CCC", "FFF", "MMM"], ["AAA", "CCC", "EEE", "FFF", "LLL", "MMM"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF", "III", "LLL", "MMM"], ["CCC", "FFF", "MMM"], ["AAA", "CCC", "EEE", "FFF", "LLL", "MMM"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF", "III", "LLL", "MMM"], ["CCC", "FFF", "MMM"], ["AAA", "CCC", "EEE", "FFF", "LLL", "MMM"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF", "III", "LLL", "MMM"], ["CCC", "FFF", "MMM"], ["AAA", "CCC", "EEE", "FFF", "LLL", "MMM"]], labels={ "l1": [ True, True, False, False, True, True, False, False, True, True, False, False, True, True, False, False ] }, path=path) dataset = RepertoireDataset(repertoires=repertoires, metadata_file=metadata, params={"l1": [True, False]}) PickleExporter.export(dataset, path) specs = { "definitions": { "datasets": { "d1": { "format": "Pickle", "params": { "path": path + f"{dataset.name}.iml_dataset", } } }, "encodings": { "e1": { "SequenceAbundance": { 'comparison_attributes': ["sequence_aas", "v_alleles", "j_alleles"] } } }, "ml_methods": { "knn": { "KNN": { "n_neighbors": 1 }, } }, "reports": { "r1": { "ReferenceSequenceOverlap": { "reference_path": ref_path, 'comparison_attributes': ["sequence_aas", "v_alleles", "j_alleles"] } } } }, "instructions": { "inst1": { "type": "TrainMLModel", "settings": [{ "encoding": "e1", "ml_method": "knn" }], "assessment": { "split_strategy": "random", "split_count": 1, "training_percentage": 0.7, "reports": {} }, "selection": { "split_strategy": "random", "split_count": 1, "training_percentage": 0.7, }, "labels": [{ "l1": { "positive_class": True } }], "dataset": "d1", "strategy": "GridSearch", "metrics": ["accuracy"], "number_of_processes": 2, "reports": ["r1"], "optimization_metric": "balanced_accuracy", "refit_optimal_model": True, "store_encoded_data": False } } } specs_file = path + "specs.yaml" with open(specs_file, "w") as file: yaml.dump(specs, file) app = ImmuneMLApp(specs_file, path + "result/") app.run() shutil.rmtree(path)
def test(self): path = PathBuilder.build(EnvironmentSettings.tmp_test_path + "integration_receptor_cnn_workflow/") specs = { "definitions": { "datasets": { "d1": { "format": "RandomReceptorDataset", "params": { "result_path": path + "generated_dataset/", "receptor_count": 500, "chain_1_length_probabilities": { 5: 1. }, "chain_2_length_probabilities": { 6: 1. }, "labels": { "cmv_epitope": { True: 0.5, False: 0.5 } } } } }, "encodings": { "enc1": { "OneHot": { "use_positional_info": True } } }, "ml_methods": { "cnn": { "ReceptorCNN": { "iteration_count": 1000, "evaluate_at": 10, "batch_size": 100, "number_of_threads": 4 } } } }, "instructions": { "instr1": { "type": "TrainMLModel", "settings": [{ "encoding": "enc1", "ml_method": "cnn" }], "assessment": { "split_strategy": "random", "split_count": 1, "training_percentage": 0.7, }, "selection": { "split_strategy": "random", "split_count": 1, "training_percentage": 1, }, "labels": ["cmv_epitope"], "dataset": "d1", "strategy": "GridSearch", "metrics": ["accuracy"], "number_of_processes": 4, "reports": None, "optimization_metric": "balanced_accuracy", "refit_optimal_model": False, "store_encoded_data": False } } } with open(path + "specs.yaml", "w") as file: yaml.dump(specs, file) app = ImmuneMLApp(path + "specs.yaml", path + 'result/') app.run() shutil.rmtree(path)
def run_tool(yaml_path, result_path): PathBuilder.build(result_path) app = ImmuneMLApp(yaml_path, result_path) app.run()
def test_simulation_receptors(self): path = PathBuilder.build(EnvironmentSettings.tmp_test_path + "integration_simulation_receptor/") specs = { "definitions": { "datasets": { "d1": { "format": "RandomReceptorDataset", "params": { "receptor_count": 100, "chain_1_length_probabilities": { 10: 1 }, "chain_2_length_probabilities": { 10: 1 }, "result_path": path + "dataset/", "labels": {} } }, }, "motifs": { "motif1": { "seed_chain1": "CC/C", "name_chain1": "ALPHA", "name_chain2": "BETA", "seed_chain2": "F/FF", "instantiation": { "GappedKmer": { "max_gap": 1, "alphabet_weights": None, "position_weights": None }, } }, "motif2": { "seed_chain1": "CCC", "name_chain1": "ALPHA", "name_chain2": "BETA", "seed_chain2": "FFF", "instantiation": "GappedKmer" } }, "signals": { "signal1": { "motifs": ["motif1", "motif2"], "implanting": "Receptor", "sequence_position_weights": None } }, "simulations": { "sim1": { "var1": { "signals": ["signal1"], "dataset_implanting_rate": 0.5 }, "var2": { "signals": ["signal1"], "dataset_implanting_rate": 0.5, "is_noise": True } } } }, "instructions": { "inst1": { "type": "Simulation", "dataset": "d1", "simulation": "sim1", "export_formats": ["Pickle"] } }, "output": { "format": "HTML" } } with open(path + "specs.yaml", "w") as file: yaml.dump(specs, file) app = ImmuneMLApp(path + "specs.yaml", path + "result/") app.run() self.assertTrue(os.path.isfile(path + "result/index.html")) self.assertTrue( os.path.isfile( path + "result/inst1/exported_dataset/pickle/d1.iml_dataset")) dataset = PickleImport.import_dataset( { "path": path + "result/inst1/exported_dataset/pickle/d1.iml_dataset" }, "d1") self.assertEqual(100, dataset.get_example_count()) self.assertEqual( 100, len([ receptor for receptor in dataset.get_data() if "signal_signal1" in receptor.metadata ])) shutil.rmtree(path)
def test_generate(self): path = EnvironmentSettings.tmp_test_path + "disease_assoc_seq_cv/" PathBuilder.build(path) repertoires, metadata = RepertoireBuilder.build( [["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF"], ["GGG", "III", "LLL", "MMM"], ["DDD", "EEE", "FFF"]], labels={ "l1": [ True, False, True, False, True, False, True, False, True, False, True, False, True, False ] }, path=path) dataset = RepertoireDataset(repertoires=repertoires, metadata_file=metadata, params={"l1": [True, False]}) PickleExporter.export(dataset, path) specs = { "definitions": { "datasets": { "d1": { "format": "Pickle", "params": { "path": path + f"{dataset.name}.iml_dataset", } } }, "encodings": { "e1": { "SequenceAbundance": { 'p_value_threshold': 0.5 } } }, "ml_methods": { "knn": { "KNN": { "n_neighbors": 1 }, } }, "reports": { "r1": { "DiseaseAssociatedSequenceCVOverlap": { "compare_in_selection": True, "compare_in_assessment": True } } } }, "instructions": { "inst1": { "type": "TrainMLModel", "settings": [{ "encoding": "e1", "ml_method": "knn" }], "assessment": { "split_strategy": "random", "split_count": 1, "training_percentage": 0.5, "reports": {} }, "selection": { "split_strategy": "random", "split_count": 1, "training_percentage": 0.5, }, "labels": [{ "l1": { "positive_class": True } }], "dataset": "d1", "strategy": "GridSearch", "metrics": ["accuracy"], "number_of_processes": 2, "reports": ["r1"], "optimization_metric": "balanced_accuracy", "refit_optimal_model": True, "store_encoded_data": False } } } specs_file = path + "specs.yaml" with open(specs_file, "w") as file: yaml.dump(specs, file) app = ImmuneMLApp(specs_file, path + "result/") state = app.run()[0] self.assertEqual(1, len(state.report_results)) self.assertTrue(len(state.report_results[0].output_figures) > 0) self.assertTrue(len(state.report_results[0].output_tables) > 0) for fig in state.report_results[0].output_figures: self.assertTrue(os.path.isfile(fig.path)) for table in state.report_results[0].output_tables: self.assertTrue(os.path.isfile(table.path)) shutil.rmtree(path)
def _run(self): PathBuilder.build(self.result_path) self.update_specs() state = ImmuneMLApp(self.yaml_path, self.result_path).run()[0] shutil.copytree(list(list(state.paths.values())[0].values())[0], self.result_path + "result/") print("Exported dataset.")