def test_parse_ml_methods(self):
params = {
"LR1": {
"LogisticRegression": {
"max_iter": 1000,
"penalty": "l1",
}
},
"LR2": "LogisticRegression",
"SVM1": {
"SVM": {
"max_iter": [1000, 2000],
"penalty": ["l1", "l2"]
},
"model_selection_cv": True,
"model_selection_n_folds": 5
},
"SVM2": {
"SVM": {},
"model_selection_cv": False,
"model_selection_n_folds": -1
}
}
symbol_table = SymbolTable()
symbol_table, desc = MLParser.parse(params, symbol_table)
self.assertTrue(symbol_table.get("SVM1")._parameter_grid is not None and len(symbol_table.get("SVM1")._parameter_grid["max_iter"]) == 2)
self.assertTrue(symbol_table.get("LR1")._parameters is not None and symbol_table.get("LR1")._parameters["penalty"] == "l1")
self.assertTrue(isinstance(symbol_table.get("LR2"), LogisticRegression))
self.assertTrue("SVM" in desc["SVM1"].keys())
def test_parse_reports(self):
reports = {"r1": {"SequenceLengthDistribution": {}}}
symbol_table = SymbolTable()
symbol_table, specs = ReportParser.parse_reports(reports, symbol_table)
self.assertTrue(symbol_table.contains("r1"))
self.assertTrue(
isinstance(symbol_table.get("r1"), SequenceLengthDistribution))
def test_parse(self):
workflow_specs = {
"seq1": [{
"filter_chain_B": {
"ChainRepertoireFilter": {
"keep_chain": "A"
}
}
}],
"seq2": [{
"filter_chain_A": {
"ChainRepertoireFilter": {
"keep_chain": "B"
}
}
}]
}
symbol_table = SymbolTable()
table, specs = PreprocessingParser.parse(workflow_specs, symbol_table)
self.assertTrue(table.contains("seq1"))
self.assertTrue(table.contains("seq2"))
self.assertTrue(
isinstance(table.get("seq1"), list)
and len(table.get("seq1")) == 1)
self.assertEqual(list(workflow_specs.keys()), list(specs.keys()))
def test_parse_receptor_dataset(self):
file_content = """complex.id Gene CDR3 V J Species MHC A MHC B MHC class Epitope Epitope gene Epitope species Reference Method Meta CDR3fix Score
3050 TRB CASSPPRVYSNGAGLAGVGWRNEQFF TRBV5-4*01 TRBJ2-1*01 HomoSapiens HLA-A*11:01 B2M MHCI AVFDRKSDAK EBNA4 EBV https://www.10xgenomics.com/resources/application-notes/a-new-way-of-exploring-immunity-linking-highly-multiplexed-antigen-recognition-to-immune-repertoire-and-phenotype/# {"frequency": "1/11684", "identification": "dextramer-sort", "sequencing": "rna-seq", "singlecell": "yes", "verification": ""} {"cell.subset": "", "clone.id": "", "donor.MHC": "", "donor.MHC.method": "", "epitope.id": "", "replica.id": "", "samples.found": 1, "structure.id": "", "studies.found": 1, "study.id": "", "subject.cohort": "", "subject.id": "1", "tissue": ""} {"cdr3": "CASSPPRVYSNGAGLAGVGWRNEQFF", "cdr3_old": "CASSPPRVYSNGAGLAGVGWRNEQFF", "fixNeeded": false, "good": true, "jCanonical": true, "jFixType": "NoFixNeeded", "jId": "TRBJ2-1*01", "jStart": 21, "vCanonical": true, "vEnd": 4, "vFixType": "NoFixNeeded", "vId": "TRBV5-4*01"} 0
15760 TRB CASSWTWDAATLWGQGALGGANVLTF TRBV5-5*01 TRBJ2-6*01 HomoSapiens HLA-A*03:01 B2M MHCI KLGGALQAK IE1 CMV https://www.10xgenomics.com/resources/application-notes/a-new-way-of-exploring-immunity-linking-highly-multiplexed-antigen-recognition-to-immune-repertoire-and-phenotype/# {"frequency": "1/25584", "identification": "dextramer-sort", "sequencing": "rna-seq", "singlecell": "yes", "verification": ""} {"cell.subset": "", "clone.id": "", "donor.MHC": "", "donor.MHC.method": "", "epitope.id": "", "replica.id": "", "samples.found": 1, "structure.id": "", "studies.found": 1, "study.id": "", "subject.cohort": "", "subject.id": "3", "tissue": ""} {"cdr3": "CASSWTWDAATLWGQGALGGANVLTF", "cdr3_old": "CASSWTWDAATLWGQGALGGANVLTF", "fixNeeded": false, "good": true, "jCanonical": true, "jFixType": "NoFixNeeded", "jId": "TRBJ2-6*01", "jStart": 19, "vCanonical": true, "vEnd": 4, "vFixType": "NoFixNeeded", "vId": "TRBV5-5*01"} 0
3050 TRA CAAIYESRGSTLGRLYF TRAV13-1*01 TRAJ18*01 HomoSapiens HLA-A*11:01 B2M MHCI AVFDRKSDAK EBNA4 EBV https://www.10xgenomics.com/resources/application-notes/a-new-way-of-exploring-immunity-linking-highly-multiplexed-antigen-recognition-to-immune-repertoire-and-phenotype/# {"frequency": "1/11684", "identification": "dextramer-sort", "sequencing": "rna-seq", "singlecell": "yes", "verification": ""} {"cell.subset": "", "clone.id": "", "donor.MHC": "", "donor.MHC.method": "", "epitope.id": "", "replica.id": "", "samples.found": 1, "structure.id": "", "studies.found": 1, "study.id": "", "subject.cohort": "", "subject.id": "1", "tissue": ""} {"cdr3": "CAAIYESRGSTLGRLYF", "cdr3_old": "CAAIYESRGSTLGRLYF", "fixNeeded": false, "good": true, "jCanonical": true, "jFixType": "NoFixNeeded", "jId": "TRAJ18*01", "jStart": 7, "oldVEnd": -1, "oldVFixType": "FailedBadSegment", "oldVId": null, "vCanonical": true, "vEnd": 3, "vFixType": "ChangeSegment", "vId": "TRAV13-1*01"} 0
15760 TRA CALRLNNQGGKLIF TRAV9-2*01 TRAJ23*01 HomoSapiens HLA-A*03:01 B2M MHCI KLGGALQAK IE1 CMV https://www.10xgenomics.com/resources/application-notes/a-new-way-of-exploring-immunity-linking-highly-multiplexed-antigen-recognition-to-immune-repertoire-and-phenotype/# {"frequency": "1/25584", "identification": "dextramer-sort", "sequencing": "rna-seq", "singlecell": "yes", "verification": ""} {"cell.subset": "", "clone.id": "", "donor.MHC": "", "donor.MHC.method": "", "epitope.id": "", "replica.id": "", "samples.found": 1, "structure.id": "", "studies.found": 1, "study.id": "", "subject.cohort": "", "subject.id": "3", "tissue": ""} {"cdr3": "CALRLNNQGGKLIF", "cdr3_old": "CALRLNNQGGKLIF", "fixNeeded": false, "good": true, "jCanonical": true, "jFixType": "NoFixNeeded", "jId": "TRAJ23*01", "jStart": 6, "vCanonical": true, "vEnd": 3, "vFixType": "NoFixNeeded", "vId": "TRAV9-2*01"} 0
"""
path = EnvironmentSettings.root_path / "test/tmp/dslimportparservdj/"
data_path = EnvironmentSettings.root_path / "test/tmp/dslimportparservdj/receptor_data/"
PathBuilder.build(data_path)
with open(data_path / "receptors.tsv", "w") as file:
file.writelines(file_content)
st, desc = ImportParser.parse(
{
"datasets": {
"d1": {
"format": "VDJdb",
"params": {
"is_repertoire": False,
"paired": True,
"receptor_chains": "TRA_TRB",
"path": data_path
}
}
}
}, SymbolTable(), path)
dataset = st.get("d1")
self.assertTrue(isinstance(dataset, ReceptorDataset))
self.assertEqual(2, dataset.get_example_count())
shutil.rmtree(path)
def test_parse(self):
specs = {
"type": "DatasetExport",
"export_formats": ["Pickle", "AIRR"],
"datasets": ["d1"]
}
symbol_table = SymbolTable()
symbol_table.add("d1", SymbolType.DATASET, RepertoireDataset())
instruction = DatasetExportParser().parse("instr1", specs,
symbol_table)
self.assertTrue(isinstance(instruction, DatasetExportInstruction))
self.assertEqual(2, len(instruction.exporters))
self.assertEqual(1, len(instruction.datasets))
def test_parse(self):
path = PathBuilder.build(
f'{EnvironmentSettings.tmp_test_path}subsampling_parser/')
dataset = RandomDatasetGenerator.generate_receptor_dataset(
30, {3: 1}, {2: 1}, {}, path)
symbol_table = SymbolTable()
symbol_table.add("d1", SymbolType.DATASET, dataset)
SubsamplingParser().parse(
'inst1', {
'dataset': 'd1',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 20],
'dataset_export_formats': ['Pickle']
}, symbol_table)
with self.assertRaises(AssertionError):
SubsamplingParser().parse(
'inst1', {
'dataset': 'd1',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 50],
'dataset_export_formats': ['Pickle']
}, symbol_table)
with self.assertRaises(AssertionError):
SubsamplingParser().parse(
'inst1', {
'dataset': 'd2',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 20],
'dataset_export_formats': ['Pickle']
}, symbol_table)
with self.assertRaises(AssertionError):
SubsamplingParser().parse(
'inst1', {
'dataset': 'd2',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 20],
'dataset_export_formats': ['Random']
}, symbol_table)
shutil.rmtree(path)
def parse(workflow_specification: dict, file_path, result_path):
symbol_table = SymbolTable()
def_parser_output, specs_defs = DefinitionParser.parse(
workflow_specification, symbol_table, result_path)
symbol_table, specs_instructions = InstructionParser.parse(
def_parser_output, result_path)
app_output = OutputParser.parse(workflow_specification, symbol_table)
path = ImmuneMLParser._output_specs(file_path=file_path,
result_path=result_path,
definitions=specs_defs,
instructions=specs_instructions,
output=app_output)
return symbol_table, path
def _extract_reports(self):
with self.specification_path.open("r") as file:
workflow_specification = yaml.safe_load(file)
report_keys = list(workflow_specification['instructions'].values()
)[0]['benchmark_reports']
ParameterValidator.assert_all_in_valid_list(
report_keys,
list(workflow_specification['definitions']['reports'].keys()),
MultiDatasetBenchmarkTool.__name__, "benchmark_reports")
reports = {
key: value
for key, value in workflow_specification['definitions']
['reports'].items() if key in report_keys
}
symbol_table, _ = ReportParser.parse_reports(reports, SymbolTable())
self.reports = [
entry.item for entry in symbol_table.get_by_type(SymbolType.REPORT)
]
def test_parse_simulation(self):
simulation = {
"sim1": {
"var1": {
"signals": ["signal1"],
"dataset_implanting_rate": 0.5,
"repertoire_implanting_rate": 0.1
}
}
}
symbol_table = SymbolTable()
symbol_table.add("motif1", SymbolType.MOTIF, Motif("motif1", GappedKmerInstantiation(position_weights={0: 1}), seed="CAS"))
symbol_table.add("signal1", SymbolType.SIGNAL, Signal("signal1", [symbol_table.get("motif1")],
HealthySequenceImplanting(GappedMotifImplanting(), implanting_computation=ImplantingComputation.ROUND)))
symbol_table, specs = SimulationParser.parse_simulations(simulation, symbol_table)
self.assertTrue(symbol_table.contains("sim1"))
sim1 = symbol_table.get("sim1")
self.assertEqual(1, len(sim1.implantings))
def test_add(self):
symbol_table = SymbolTable()
symbol_table.add("svm1", SymbolType.ML_METHOD, {})
with self.assertWarns(Warning):
symbol_table.add("svm1", SymbolType.ML_METHOD, {})
def test_parse(self):
path = EnvironmentSettings.root_path / "test/tmp/parser/"
PathBuilder.build(path / "tmp_input/")
with open(path / "tmp_input/CD1_clones_TRA.csv", "w") as file:
writer = csv.DictWriter(file,
delimiter="\t",
fieldnames=[
"patient", "dilution", "cloneCount",
"allVHitsWithScore",
"allJHitsWithScore", "nSeqCDR1",
"nSeqCDR2", "nSeqCDR3", "minQualCDR3",
"aaSeqCDR1", "aaSeqCDR2", "aaSeqCDR3",
"sampleID"
])
dicts = [{
"patient": "CD12",
"dilution": "108'",
"cloneCount": 3,
"allVHitsWithScore": "TRAV13-1*00(735)",
"allJHitsWithScore": "TRAJ15*00(243)",
"nSeqCDR1": "TGTGCAGCAA",
"nSeqCDR2": "TGTGCAGCAA",
"nSeqCDR3": "TGTGCAGCAA",
"minQualCDR3": 10,
"aaSeqCDR1": "VFAVFA",
"aaSeqCDR2": "VFAVFA",
"aaSeqCDR3": "VFAVFA",
"sampleID": "2"
}, {
"patient": "CD12",
"dilution": "108'",
"cloneCount": 5,
"allVHitsWithScore": "TRAV14-1*00(735)",
"allJHitsWithScore": "TRAJ12*00(243)",
"nSeqCDR1": "CAATGTGA",
"nSeqCDR2": "CAATGTGA",
"nSeqCDR3": "CAATGTGA",
"minQualCDR3": 10,
"aaSeqCDR1": "CASCAS",
"aaSeqCDR2": "CASCAS",
"aaSeqCDR3": "CASCAS",
"sampleID": "3"
}]
writer.writeheader()
writer.writerows(dicts)
with open(path / "tmp_input/HC2_clones_TRB.csv", "w") as file:
writer = csv.DictWriter(file,
delimiter="\t",
fieldnames=[
"patient", "dilution", "cloneCount",
"allVHitsWithScore",
"allJHitsWithScore", "nSeqCDR1",
"nSeqCDR2", "nSeqCDR3", "minQualCDR3",
"aaSeqCDR1", "aaSeqCDR2", "aaSeqCDR3",
"sampleID"
])
dicts = [{
"patient": "CD12",
"dilution": "108'",
"cloneCount": 3,
"allVHitsWithScore": "TRAV13-1*00(735)",
"allJHitsWithScore": "TRAJ15*00(243)",
"nSeqCDR1": "TGTGCAGCAA",
"nSeqCDR2": "TGTGCAGCAA",
"nSeqCDR3": "TGTGCAGCAA",
"minQualCDR3": 10,
"aaSeqCDR1": "CAASNQA",
"aaSeqCDR2": "CAASNQA",
"aaSeqCDR3": "CAASNQA",
"sampleID": "1"
}, {
"patient": "CD12",
"dilution": "108'",
"cloneCount": 6,
"allVHitsWithScore": "TRAV19-1*00(735)",
"allJHitsWithScore": "TRAJ12*00(243)",
"nSeqCDR1": "CAATGTGA",
"nSeqCDR2": "CAATGTGA",
"nSeqCDR3": "CAATGTGA",
"minQualCDR3": 10,
"aaSeqCDR1": "CAASNTTA",
"aaSeqCDR2": "CAASNTTA",
"aaSeqCDR3": "CAASNTTA",
"sampleID": 1
}, {
"patient": "CD12",
"dilution": "108'",
"cloneCount": 6,
"allVHitsWithScore": "TRAV19-1*00(735)",
"allJHitsWithScore": "TRAJ12*00(243)",
"nSeqCDR1": "CAATGTGA",
"nSeqCDR2": "CAATGTGA",
"nSeqCDR3": "CAATGTGA",
"minQualCDR3": 10,
"aaSeqCDR1": "CAASNTTA",
"aaSeqCDR2": "CAASNTTA",
"aaSeqCDR3": "CAASNTTA",
"sampleID": 1
}]
writer.writeheader()
writer.writerows(dicts)
metadata = pd.DataFrame({
"filename": ["HC2_clones_TRB.csv", "CD1_clones_TRA.csv"],
"subject_id": ["HC2", "CD1"],
"CD": [False, True]
})
metadata.to_csv(path / "metadata.csv")
specs = {
"datasets": {
"d1": {
"format": "MiXCR",
"params": {
"is_repertoire": True,
"path": path / "tmp_input/",
"metadata_file": path / "metadata.csv",
"number_of_processes": 2,
}
}
}
}
st, desc = ImportParser.parse(specs, SymbolTable(),
path / "tmp_output/")
self.assertTrue(isinstance(st.get("d1"), RepertoireDataset))
self.assertEqual(2, len(st.get("d1").get_data()))
shutil.rmtree(path)
def test_parse(self):
path = EnvironmentSettings.tmp_test_path / "explanalysisparser/"
PathBuilder.build(path)
dataset = self.prepare_dataset(path)
report1 = SequenceLengthDistribution()
file_content = """complex.id Gene CDR3 V J Species MHC A MHC B MHC class Epitope Epitope gene Epitope species Reference Method Meta CDR3fix Score
100a TRA AAAC TRAV12 TRAJ1 HomoSapiens HLA-A*11:01 B2M MHCI AVFDRKSDAK EBNA4 EBV
"""
with open(path / "refs.tsv", "w") as file:
file.writelines(file_content)
refs = {
"params": {
"path": path / "refs.tsv",
"region_type": "FULL_SEQUENCE"
},
"format": "VDJdb"
}
report2 = Matches.build_object()
encoding = MatchedSequencesEncoder
p1 = [SubjectRepertoireCollector()]
instruction = {
"type": "ExploratoryAnalysis",
"number_of_processes": 32,
"analyses": {
"1": {
"dataset": "d1",
"report": "r1",
"preprocessing_sequence": "p1"
},
"2": {
"dataset": "d1",
"report": "r2",
"encoding": "e1",
},
"3": {
"dataset": "d1",
"report": "r2",
"encoding": "e1",
"labels": ["l1"]
}
}
}
symbol_table = SymbolTable()
symbol_table.add("d1", SymbolType.DATASET, dataset)
symbol_table.add("r1", SymbolType.REPORT, report1)
symbol_table.add("r2", SymbolType.REPORT, report2)
symbol_table.add(
"e1", SymbolType.ENCODING, encoding,
{"encoder_params": {
"max_edit_distance": 1,
"reference": refs
}})
symbol_table.add("p1", SymbolType.PREPROCESSING, p1)
process = ExploratoryAnalysisParser().parse("a", instruction,
symbol_table)
self.assertEqual(
3, len(list(process.state.exploratory_analysis_units.values())))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[0].report, SequenceLengthDistribution))
# testing matches with and without labels
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[1].report, Matches))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[1].encoder, MatchedSequencesEncoder))
self.assertEqual(
1,
len(
list(process.state.exploratory_analysis_units.values())
[1].encoder.reference_sequences))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[2].report, Matches))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[2].encoder, MatchedSequencesEncoder))
self.assertEqual(
1,
len(
list(process.state.exploratory_analysis_units.values())
[2].encoder.reference_sequences))
self.assertEqual(
"l1",
list(process.state.exploratory_analysis_units.values())
[2].label_config.get_labels_by_name()[0])
self.assertEqual(
32,
process.state.exploratory_analysis_units["2"].number_of_processes)
shutil.rmtree(path)