def test(self):
path = EnvironmentSettings.tmp_test_path / "integration_sequence_classification/"
dataset = self.create_dataset(path)
os.environ["cache_type"] = "test"
encoder_params = {
"normalization_type": NormalizationType.RELATIVE_FREQUENCY.name,
"reads": ReadsType.UNIQUE.name,
"sequence_encoding": SequenceEncodingType.CONTINUOUS_KMER.name,
"k": 3
}
hp_setting = HPSetting(encoder=KmerFrequencyEncoder.build_object(
dataset, **encoder_params),
encoder_params=encoder_params,
ml_method=LogisticRegression(),
ml_params={
"model_selection_cv": False,
"model_selection_n_folds": -1
},
preproc_sequence=[])
lc = LabelConfiguration()
lc.add_label("l1", [1, 2])
instruction = TrainMLModelInstruction(
dataset, GridSearch([hp_setting]), [hp_setting],
SplitConfig(SplitType.RANDOM, 1, 0.5, reports=ReportConfig()),
SplitConfig(SplitType.RANDOM, 1, 0.5, reports=ReportConfig()),
{Metric.BALANCED_ACCURACY}, Metric.BALANCED_ACCURACY, lc, path)
result = instruction.run(result_path=path)
shutil.rmtree(path)
def test_run(self):
path = EnvironmentSettings.root_path / "test/tmp/smmodel/"
PathBuilder.build(path)
repertoires, metadata = RepertoireBuilder.build(
[["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"],
["TTTT"], ["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"], ["TTTT"],
["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"],
["TTTT"], ["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"], ["TTTT"],
["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"],
["TTTT"], ["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"], ["TTTT"],
["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"], ["TTTT"],
["AAA", "CCC"], ["TTTT"], ["AAA", "CCC"], ["TTTT"]], path, {
"default": [
1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1,
2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2
]
})
dataset = RepertoireDataset(repertoires=repertoires,
labels={"default": [1, 2]},
metadata_file=metadata)
label_config = LabelConfiguration()
label_config.add_label("default", [1, 2])
hp_settings = [
HPSetting(
Word2VecEncoder.build_object(
dataset, **{
"vector_size": 8,
"model_type": ModelType.SEQUENCE.name,
"k": 3
}), {
"vector_size": 8,
"model_type": ModelType.SEQUENCE.name,
"k": 3
}, LogisticRegression(), {
"model_selection_cv": False,
"model_selection_n_folds": -1
}, [])
]
split_config_assessment = SplitConfig(SplitType.RANDOM, 1, 0.5,
ReportConfig())
split_config_selection = SplitConfig(SplitType.RANDOM, 1, 0.5,
ReportConfig())
instruction = TrainMLModelInstruction(
dataset, GridSearch(hp_settings), hp_settings,
split_config_assessment, split_config_selection,
{Metric.BALANCED_ACCURACY}, Metric.BALANCED_ACCURACY, label_config,
path)
semantic_model = SemanticModel([instruction], path)
semantic_model.run()
shutil.rmtree(path)
def _parse_split_config(self, instruction_key, instruction: dict, split_key: str, symbol_table: SymbolTable, settings_count: int) -> SplitConfig:
try:
default_params = DefaultParamsLoader.load("instructions/", SplitConfig.__name__)
report_config_input = self._prepare_report_config(instruction_key, instruction, split_key, symbol_table)
instruction[split_key] = {**default_params, **instruction[split_key]}
split_strategy = SplitType[instruction[split_key]["split_strategy"].upper()]
training_percentage = float(instruction[split_key]["training_percentage"]) if split_strategy == SplitType.RANDOM else -1
if split_strategy == SplitType.RANDOM and training_percentage == 1 and settings_count > 1:
raise ValueError(f"{TrainMLModelParser.__name__}: all data under {instruction_key}/{split_key} was specified to be used for "
f"training, but {settings_count} settings were specified for evaluation. Please define a test/validation set by "
f"reducing the training percentage (e.g., to 0.7) or use only one hyperparameter setting to run the analysis.")
return SplitConfig(split_strategy=split_strategy,
split_count=int(instruction[split_key]["split_count"]),
training_percentage=training_percentage,
reports=ReportConfig(**report_config_input),
manual_config=ManualSplitConfig(**instruction[split_key]["manual_config"]) if "manual_config" in instruction[split_key] else None,
leave_one_out_config=LeaveOneOutConfig(**instruction[split_key]["leave_one_out_config"])
if "leave_one_out_config" in instruction[split_key] else None)
except KeyError as key_error:
raise KeyError(f"{TrainMLModelParser.__name__}: parameter {key_error.args[0]} was not defined under {split_key}.")
def __init__(self, split_strategy: SplitType, split_count: int, training_percentage: float = None, reports: ReportConfig = None,
manual_config: ManualSplitConfig = None, leave_one_out_config: LeaveOneOutConfig = None):
self.split_strategy = split_strategy
self.split_count = split_count
self.training_percentage = training_percentage
self.reports = reports if reports is not None else ReportConfig()
self.manual_config = manual_config
self.leave_one_out_config = leave_one_out_config
def test_run(self):
path = EnvironmentSettings.tmp_test_path / "hpoptimproc/"
PathBuilder.build(path)
repertoires, metadata = RepertoireBuilder.build(
sequences=[["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"],
["AAA", "CCC", "DDD"], ["AAA", "CCC", "DDD"]],
path=path,
labels={
"l1": [
1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2,
1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2
],
"l2": [
0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
]
})
dataset = RepertoireDataset(repertoires=repertoires,
metadata_file=metadata,
labels={
"l1": [1, 2],
"l2": [0, 1]
})
enc1 = {
"k": 3,
"model_type": ModelType.SEQUENCE.name,
"vector_size": 4
}
enc2 = {
"k": 3,
"model_type": ModelType.SEQUENCE.name,
"vector_size": 6
}
hp_settings = [
HPSetting(Word2VecEncoder.build_object(dataset, **enc1), enc1,
LogisticRegression(), {
"model_selection_cv": False,
"model_selection_n_folds": -1
}, []),
HPSetting(
Word2VecEncoder.build_object(dataset, **enc2), enc2, SVM(), {
"model_selection_cv": False,
"model_selection_n_folds": -1
},
[ClonesPerRepertoireFilter(lower_limit=-1, upper_limit=1000)])
]
report = SequenceLengthDistribution()
label_config = LabelConfiguration(
[Label("l1", [1, 2]), Label("l2", [0, 1])])
process = TrainMLModelInstruction(
dataset, GridSearch(hp_settings), hp_settings,
SplitConfig(SplitType.RANDOM,
1,
0.5,
reports=ReportConfig(data_splits={"seqlen": report})),
SplitConfig(SplitType.RANDOM,
1,
0.5,
reports=ReportConfig(data_splits={"seqlen": report})),
{Metric.BALANCED_ACCURACY}, Metric.BALANCED_ACCURACY, label_config,
path)
state = process.run(result_path=path)
self.assertTrue(isinstance(state, TrainMLModelState))
self.assertEqual(1, len(state.assessment_states))
self.assertTrue("l1" in state.assessment_states[0].label_states)
self.assertTrue("l2" in state.assessment_states[0].label_states)
shutil.rmtree(path)