def _parse_dataset(key: str, dataset_specs: dict,
symbol_table: SymbolTable,
result_path: Path) -> SymbolTable:
location = "ImportParser"
ParameterValidator.assert_keys(list(dataset_specs.keys()),
ImportParser.valid_keys, location,
f"datasets:{key}", False)
valid_formats = ReflectionHandler.all_nonabstract_subclass_basic_names(
DataImport, "Import", "IO/dataset_import/")
ParameterValidator.assert_in_valid_list(dataset_specs["format"],
valid_formats, location,
"format")
import_cls = ReflectionHandler.get_class_by_name("{}Import".format(
dataset_specs["format"]))
params = ImportParser._prepare_params(dataset_specs, result_path, key)
if "is_repertoire" in params:
ParameterValidator.assert_type_and_value(params["is_repertoire"],
bool, location,
"is_repertoire")
if params["is_repertoire"] == True:
if import_cls != IReceptorImport:
assert "metadata_file" in params, f"{location}: Missing parameter: metadata_file under {key}/params/"
ParameterValidator.assert_type_and_value(
params["metadata_file"], Path, location,
"metadata_file")
if params["is_repertoire"] == False:
assert "paired" in params, f"{location}: Missing parameter: paired under {key}/params/"
ParameterValidator.assert_type_and_value(
params["paired"], bool, location, "paired")
if params["paired"] == True:
assert "receptor_chains" in params, f"{location}: Missing parameter: receptor_chains under {key}/params/"
ParameterValidator.assert_in_valid_list(
params["receptor_chains"],
["_".join(cp.value) for cp in ChainPair], location,
"receptor_chains")
try:
dataset = import_cls.import_dataset(params, key)
dataset.name = key
symbol_table.add(key, SymbolType.DATASET, dataset)
except KeyError as key_error:
raise KeyError(
f"{key_error}\n\nAn error occurred during parsing of dataset {key}. "
f"The keyword {key_error.args[0]} was missing. This either means this argument was "
f"not defined under definitions/datasets/{key}/params, or this column was missing from "
f"an input data file. ")
except Exception as ex:
raise Exception(
f"{ex}\n\nAn error occurred while parsing the dataset {key}. See the log above for more details."
)
return symbol_table
def parse(encodings: dict, symbol_table: SymbolTable):
for key in encodings.keys():
encoder, params = EncodingParser.parse_encoder(key, encodings[key])
symbol_table.add(key, SymbolType.ENCODING, encoder,
{"encoder_params": params})
return symbol_table, encodings
def _parse_report(key: str, params: dict, symbol_table: SymbolTable):
valid_values = ReflectionHandler.all_nonabstract_subclass_basic_names(Report, "", "reports/")
report_object, params = ObjectParser.parse_object(params, valid_values, "", "reports/", "ReportParser", key, builder=True,
return_params_dict=True)
symbol_table.add(key, SymbolType.REPORT, report_object)
return symbol_table, params
def parse(specs: dict, symbol_table: SymbolTable) -> dict:
if "output" in specs:
ParameterValidator.assert_keys(specs["output"], ["format"], "OutputParser", "output")
ParameterValidator.assert_in_valid_list(specs["output"]["format"], ["HTML"], "OutputParser", "format")
else:
specs["output"] = {"format": "HTML"}
symbol_table.add("output", SymbolType.OUTPUT, specs["output"])
return specs["output"]
def parse(specification: dict, symbol_table: SymbolTable):
for ml_method_id in specification.keys():
ml_method, config = MLParser._parse_ml_method(
ml_method_id, specification[ml_method_id])
specification[ml_method_id] = config
symbol_table.add(ml_method_id, SymbolType.ML_METHOD, ml_method,
config)
return symbol_table, specification
def parse_motifs(motifs: dict, symbol_table: SymbolTable):
valid_motif_keys = ["seed", "instantiation", "seed_chain1", "seed_chain2", "name_chain1", "name_chain2"]
for key in motifs.keys():
ParameterValidator.assert_keys(motifs[key].keys(), valid_motif_keys, "MotifParser", key, exclusive=False)
motif = MotifParser._parse_motif(key, motifs[key])
symbol_table.add(key, SymbolType.MOTIF, motif)
return symbol_table, motifs
def parse_instruction(key: str, instruction: dict, symbol_table: SymbolTable, path) -> tuple:
ParameterValidator.assert_keys_present(list(instruction.keys()), ["type"], InstructionParser.__name__, key)
valid_instructions = [cls[:-6] for cls in ReflectionHandler.discover_classes_by_partial_name("Parser", "dsl/instruction_parsers/")]
ParameterValidator.assert_in_valid_list(instruction["type"], valid_instructions, "InstructionParser", "type")
default_params = DefaultParamsLoader.load("instructions/", instruction["type"])
instruction = {**default_params, **instruction}
parser = ReflectionHandler.get_class_by_name("{}Parser".format(instruction["type"]), "instruction_parsers/")()
instruction_object = parser.parse(key, instruction, symbol_table, path)
symbol_table.add(key, SymbolType.INSTRUCTION, instruction_object)
return instruction, symbol_table
def test_parse(self):
specs = {
"type": "DatasetExport",
"export_formats": ["Pickle", "AIRR"],
"datasets": ["d1"]
}
symbol_table = SymbolTable()
symbol_table.add("d1", SymbolType.DATASET, RepertoireDataset())
instruction = DatasetExportParser().parse("instr1", specs,
symbol_table)
self.assertTrue(isinstance(instruction, DatasetExportInstruction))
self.assertEqual(2, len(instruction.exporters))
self.assertEqual(1, len(instruction.datasets))
def test_parse(self):
path = PathBuilder.build(
f'{EnvironmentSettings.tmp_test_path}subsampling_parser/')
dataset = RandomDatasetGenerator.generate_receptor_dataset(
30, {3: 1}, {2: 1}, {}, path)
symbol_table = SymbolTable()
symbol_table.add("d1", SymbolType.DATASET, dataset)
SubsamplingParser().parse(
'inst1', {
'dataset': 'd1',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 20],
'dataset_export_formats': ['Pickle']
}, symbol_table)
with self.assertRaises(AssertionError):
SubsamplingParser().parse(
'inst1', {
'dataset': 'd1',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 50],
'dataset_export_formats': ['Pickle']
}, symbol_table)
with self.assertRaises(AssertionError):
SubsamplingParser().parse(
'inst1', {
'dataset': 'd2',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 20],
'dataset_export_formats': ['Pickle']
}, symbol_table)
with self.assertRaises(AssertionError):
SubsamplingParser().parse(
'inst1', {
'dataset': 'd2',
'type': 'Subsampling',
'subsampled_dataset_sizes': [10, 20],
'dataset_export_formats': ['Random']
}, symbol_table)
shutil.rmtree(path)
def _parse_sequence(key: str, preproc_sequence: list,
symbol_table: SymbolTable) -> SymbolTable:
sequence = []
valid_preprocessing_classes = ReflectionHandler.all_nonabstract_subclass_basic_names(
Preprocessor, "", "preprocessing/")
for item in preproc_sequence:
for step_key, step in item.items():
obj, params = ObjectParser.parse_object(
step, valid_preprocessing_classes, "", "preprocessing/",
"PreprocessingParser", step_key, True, True)
step = params
sequence.append(obj)
symbol_table.add(key, SymbolType.PREPROCESSING, sequence)
return symbol_table
def test_parse_simulation(self):
simulation = {
"sim1": {
"var1": {
"signals": ["signal1"],
"dataset_implanting_rate": 0.5,
"repertoire_implanting_rate": 0.1
}
}
}
symbol_table = SymbolTable()
symbol_table.add("motif1", SymbolType.MOTIF, Motif("motif1", GappedKmerInstantiation(position_weights={0: 1}), seed="CAS"))
symbol_table.add("signal1", SymbolType.SIGNAL, Signal("signal1", [symbol_table.get("motif1")],
HealthySequenceImplanting(GappedMotifImplanting(), implanting_computation=ImplantingComputation.ROUND)))
symbol_table, specs = SimulationParser.parse_simulations(simulation, symbol_table)
self.assertTrue(symbol_table.contains("sim1"))
sim1 = symbol_table.get("sim1")
self.assertEqual(1, len(sim1.implantings))
def _parse_simulation(key: str, simulation: dict, symbol_table: SymbolTable) -> SymbolTable:
location = "SimulationParser"
valid_implanting_keys = ["dataset_implanting_rate", "repertoire_implanting_rate", "signals", "is_noise"]
implantings = []
for impl_key, implanting in simulation.items():
ParameterValidator.assert_keys(implanting.keys(), valid_implanting_keys, location, impl_key, exclusive=False)
ParameterValidator.assert_keys(implanting["signals"], symbol_table.get_keys_by_type(SymbolType.SIGNAL), location, impl_key, False)
implanting_params = copy.deepcopy(implanting)
implanting_params["signals"] = [symbol_table.get(signal) for signal in implanting["signals"]]
implanting_params["name"] = impl_key
implantings.append(Implanting(**implanting_params))
assert sum([settings["dataset_implanting_rate"] for settings in simulation.values()]) <= 1, \
"The total dataset implanting rate can not exceed 1."
symbol_table.add(key, SymbolType.SIMULATION, Simulation(implantings))
return symbol_table
def parse_signals(signals: dict, symbol_table: SymbolTable):
for key, signal_spec in signals.items():
ParameterValidator.assert_keys_present(signal_spec.keys(),
SignalParser.VALID_KEYS,
"SignalParser", key)
implanting_strategy = SignalParser._get_implanting_strategy(
key, signal_spec)
ParameterValidator.assert_keys(
signal_spec["motifs"],
symbol_table.get_keys_by_type(SymbolType.MOTIF),
"SignalParser", f"motifs in signal {key}", False)
signal_motifs = [
symbol_table.get(motif_id)
for motif_id in signal_spec["motifs"]
]
signal = Signal(key, signal_motifs, implanting_strategy)
symbol_table.add(key, SymbolType.SIGNAL, signal)
return symbol_table, signals
def test_add(self):
symbol_table = SymbolTable()
symbol_table.add("svm1", SymbolType.ML_METHOD, {})
with self.assertWarns(Warning):
symbol_table.add("svm1", SymbolType.ML_METHOD, {})
def test_parse(self):
path = EnvironmentSettings.tmp_test_path / "explanalysisparser/"
PathBuilder.build(path)
dataset = self.prepare_dataset(path)
report1 = SequenceLengthDistribution()
file_content = """complex.id Gene CDR3 V J Species MHC A MHC B MHC class Epitope Epitope gene Epitope species Reference Method Meta CDR3fix Score
100a TRA AAAC TRAV12 TRAJ1 HomoSapiens HLA-A*11:01 B2M MHCI AVFDRKSDAK EBNA4 EBV
"""
with open(path / "refs.tsv", "w") as file:
file.writelines(file_content)
refs = {
"params": {
"path": path / "refs.tsv",
"region_type": "FULL_SEQUENCE"
},
"format": "VDJdb"
}
report2 = Matches.build_object()
encoding = MatchedSequencesEncoder
p1 = [SubjectRepertoireCollector()]
instruction = {
"type": "ExploratoryAnalysis",
"number_of_processes": 32,
"analyses": {
"1": {
"dataset": "d1",
"report": "r1",
"preprocessing_sequence": "p1"
},
"2": {
"dataset": "d1",
"report": "r2",
"encoding": "e1",
},
"3": {
"dataset": "d1",
"report": "r2",
"encoding": "e1",
"labels": ["l1"]
}
}
}
symbol_table = SymbolTable()
symbol_table.add("d1", SymbolType.DATASET, dataset)
symbol_table.add("r1", SymbolType.REPORT, report1)
symbol_table.add("r2", SymbolType.REPORT, report2)
symbol_table.add(
"e1", SymbolType.ENCODING, encoding,
{"encoder_params": {
"max_edit_distance": 1,
"reference": refs
}})
symbol_table.add("p1", SymbolType.PREPROCESSING, p1)
process = ExploratoryAnalysisParser().parse("a", instruction,
symbol_table)
self.assertEqual(
3, len(list(process.state.exploratory_analysis_units.values())))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[0].report, SequenceLengthDistribution))
# testing matches with and without labels
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[1].report, Matches))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[1].encoder, MatchedSequencesEncoder))
self.assertEqual(
1,
len(
list(process.state.exploratory_analysis_units.values())
[1].encoder.reference_sequences))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[2].report, Matches))
self.assertTrue(
isinstance(
list(process.state.exploratory_analysis_units.values())
[2].encoder, MatchedSequencesEncoder))
self.assertEqual(
1,
len(
list(process.state.exploratory_analysis_units.values())
[2].encoder.reference_sequences))
self.assertEqual(
"l1",
list(process.state.exploratory_analysis_units.values())
[2].label_config.get_labels_by_name()[0])
self.assertEqual(
32,
process.state.exploratory_analysis_units["2"].number_of_processes)
shutil.rmtree(path)