def get_nih(db): nih_df = pd.read_csv(from_root("temp\\test_data\\nih_cleaned.csv")) db.insert(nih_df, "tmp_nih", "dbo") nih_metamap_df = annotate(nih_df) db.insert(nih_metamap_df, "tmp_nih_metamap", "dbo") write_df(from_root("temp\\test_data\\nih_metamap.csv"), nih_metamap_df)
def __init__(self, cfg_filepath: str): set_deterministic_seed() with open(from_root(cfg_filepath), "r") as file: self.cfg = DictObject(json.load(file)) for sub_dirname in ("logs", "checkpoints", "debug"): os.makedirs(self.from_out(sub_dirname), exist_ok=True) self.model = create_resnet(self.cfg) self.model = self.model.to(self.cfg.model.device) self.train_loader, self.infer_train_loader, self.infer_val_loader, \ train_mean, train_std = create_dataset(self.cfg) self.model.normalize.set_parameters(train_mean, train_std) self.logger = SummaryWriter(log_dir=self.from_out("logs")) self.optimizer = create_optimizer(self.model, self.logger, self.cfg) self.scheduler = optim.lr_scheduler.MultiStepLR( self.optimizer, self.cfg.scheduler.milestones, gamma=self.cfg.scheduler.gamma) if isinstance(self.model, ResNet_Softmax): self.criterion = lambda logits, targets: F.nll_loss( logits, targets, reduction="mean") else: assert isinstance(self.model, ResNet_Gaussian) self.criterion = lambda sqr_distances, targets: mmc_loss( sqr_distances, targets, reduction="mean") Tracker.reset(self.cfg.optimizer.n_epochs) if self.cfg.load_checkpoint is not None: self.load_checkpoint(self.cfg.load_checkpoint)
def test_log_domain(self) -> None: images = [ Image.open(from_root("test_data/log_bilinear_4x/1.png")), Image.open(from_root("test_data/log_bilinear_4x/2.png")) ] transform = transforms.ToTensor() x = torch.stack([transform(image) for image in images], dim=0) expected = F.interpolate(x, scale_factor=4, mode="bilinear") expected = torch.log(expected) actual = torch.log(x) actual = log_bilinear_4x(actual) self.assert_no_nan(actual) self.assert_tensors_almost_equal(expected, actual, 1e-6)
def classify_culture(db, to_module, l1ml_module, l1s_module, l2_module): culture_df = db.extract(from_root("temp\\sql\\test_culture.sql")) keys = ["test_key", "result_key"] culture_to_results = to_module.classify(culture_df) culture_l1ml_results = l1ml_module.classify(culture_df) culture_l1s_results = l1s_module.classify(culture_df) culture_l2_results = l2_module.classify(culture_df) culture_results = culture_to_results\ .merge(culture_l1ml_results, how="inner", on=keys)\ .merge(culture_l1s_results, how="inner", on=keys)\ .merge(culture_l2_results, how="inner", on=keys) db.insert(culture_results, "tmp_culture_predictions", "dbo") write_df(from_root("temp\\predictions\\culture.csv"), culture_results)
def classify_random(db, to_module, l1ml_module, l1s_module, l2_module): random_df = db.extract(from_root("temp\\sql\\test_random.sql")) keys = ["test_key", "result_key"] random_to_results = to_module.classify(random_df) random_l1ml_results = l1ml_module.classify(random_df) random_l1s_results = l1s_module.classify(random_df) random_l2_results = l2_module.classify(random_df) random_results = random_to_results\ .merge(random_l1ml_results, how="inner", on=keys)\ .merge(random_l1s_results, how="inner", on=keys)\ .merge(random_l2_results, how="inner", on=keys) db.insert(random_results, "tmp_random_predictions", "dbo") write_df(from_root("temp\\predictions\\random.csv"), random_results)
def classify_nih(db, tp_module, to_module, l1ml_module, l1s_module, l2_module): nih_df = db.extract(from_root("temp\\sql\\test_nih.sql")) keys = ["test_key", "result_key"] nih_tp_results = tp_module.classify(nih_df) nih_to_results = to_module.classify(nih_df) nih_l1ml_results = l1ml_module.classify(nih_df) nih_l1s_results = l1s_module.classify(nih_df) nih_l2_results = l2_module.classify(nih_df) nih_results = nih_tp_results\ .merge(nih_to_results, how="inner", on=keys)\ .merge(nih_l1ml_results, how="inner", on=keys)\ .merge(nih_l1s_results, how="inner", on=keys)\ .merge(nih_l2_results, how="inner", on=keys) db.insert(nih_results, "tmp_nih_predictions", "dbo") write_df(from_root("temp\\predictions\\nih.csv"), nih_results)
def test_linear_domain(self) -> None: images = [ Image.open(from_root("test_data/log_bilinear_4x/1.png")), Image.open(from_root("test_data/log_bilinear_4x/2.png")) ] transform = transforms.ToTensor() x = torch.stack([transform(image) for image in images], dim=0) os.makedirs(from_root("test_results/log_bilinear_4x"), exist_ok=True) expected = F.interpolate(x, scale_factor=4, mode="bilinear") torchvision.utils.save_image( expected, from_root("test_results/log_bilinear_4x/expected.png")) actual = torch.log(x) actual = log_bilinear_4x(actual) actual = torch.exp(actual) torchvision.utils.save_image( actual, from_root("test_results/log_bilinear_4x/actual.png")) self.assert_no_nan(actual) self.assert_tensors_almost_equal(expected, actual, 1e-6)
def init_surrogate_model(self, surrogate_cfg_filepath: str) -> None: with open(from_root(surrogate_cfg_filepath), "r") as file: cfg = DictObject(json.load(file)) self.surrogate_model = create_resnet(cfg) self.surrogate_model = self.surrogate_model.to(cfg.model.device) best_epoch_filepath = os.path.join(from_root(cfg.out_dirpath), "checkpoints/best_epoch.txt") with open(best_epoch_filepath, "r") as file: epoch = int(file.read()) checkpoint_filepath = os.path.join( from_root(cfg.out_dirpath), f"checkpoints/checkpoint_{epoch}.pth") checkpoint = torch.load(checkpoint_filepath, map_location=model_device( self.surrogate_model)) self.surrogate_model.load_state_dict(checkpoint["model_state_dict"]) self.surrogate_model.eval() for param in self.surrogate_model.parameters(): param.requires_grad = False
def load_modules(): # Test performed tp_module = TestPerformedModule.load_from_file( from_root("temp\\pkl\\test_performed_module.pkl")) # Test outcome to_module = TestOutcomeModule.load_from_file( from_root("temp\\pkl\\test_outcome_module.pkl")) # Level 1 (machine learning) l1ml_module = Level1MLModule.load_from_file( from_root("temp\\pkl\\level_1_ml_module.pkl")) # Level 1 (symbolic) l1s_module = Level1SymbolicModule(to_module).load_from_file( from_root("temp\\pkl\\level_1_symbolic_module.pkl")) # Level 2 l2_module = Level2Module(l1ml_module).load_from_file( from_root("temp\\pkl\\level_2_module.pkl")) print("Finished loading modules.") return tp_module, to_module, l1ml_module, l1s_module, l2_module
def __init__(self, cfg_filepath: str): set_deterministic_seed() with open(from_root(cfg_filepath), "r") as file: self.cfg = DictObject(json.load(file)) for sub_dirname in ("logs", "checkpoints", "debug"): os.makedirs(self.from_out(sub_dirname), exist_ok=True) self.model = RefineNet_4Cascaded() self.model = self.model.cuda() state_dict = torch.load("TODO", map_location=model_device(self.model)) self.model.backbone.load_state_dict(state_dict) self.train_loader = load_pascal_voc_train(16) self.infer_train_loader = load_pascal_voc_infer("train", 16) self.infer_val_loader = load_pascal_voc_infer("val", 16)
def __init__(self, cfg_filepath: str, data_loader: DataLoader, adversary: Optional[Adversary], out_name: str, visualize_adversary: int): set_deterministic_seed() with open(from_root(cfg_filepath), "r") as file: self.cfg = DictObject(json.load(file)) os.makedirs(self.from_out("inference"), exist_ok=True) os.makedirs(self.from_out(f"inference_debug/{out_name}"), exist_ok=True) self.model = create_resnet(self.cfg) self.model = self.model.to(self.cfg.model.device) self.load_best_epoch() self.loader = data_loader self.adversary = adversary self.out_name = out_name self.visualize_adversary = visualize_adversary Tracker.reset()
def train_test_performed(db): tp_df = db.extract(from_root("temp\\sql\\train_test_performed.sql")) tp_module = TestPerformedModule(organisms=True) tp_module.retrain(tp_df) tp_module.save_to_file(from_root("temp\\pkl\\test_performed_module.pkl"))
import multiprocessing import os from typing import Union import torchvision from torch.utils.data import DataLoader, ConcatDataset, Subset from torchvision import transforms from root import from_root from src.misc.utils import read_lines DATA_DIRPATH = from_root("data/svhn") SPLIT_DIRPATH = from_root("splits/svhn") SVHN_TRAIN_MEAN = [0.4310, 0.4303, 0.4464] SVHN_TRAIN_STD = [0.1965, 0.1983, 0.1994] def load_svhn_infer(split: str, batch_size: int, n_workers: Union[str, int]) -> DataLoader: if split not in {"train", "val", "test"}: raise ValueError("Split must be 'train', 'val', or 'test'!") if batch_size <= 0: raise ValueError("Batch_size must be positive!") if type(n_workers) == str and n_workers != "n_cores": raise ValueError("If n_workers is a string, it must be 'n_cores'!") if type(n_workers) == int and n_workers < 0: raise ValueError("If n_workers is an int, it must be non-negative!") transform = transforms.ToTensor() if split == "train":
def get_culture(db): culture_df = db.extract(from_root("temp\\sql\\get_test_culture.sql")) culture_df = culture_df.sample(n=100) db.insert(culture_df, "tmp_culture", "dbo") write_df(from_root("temp\\test_data\\culture.csv"), culture_df)
def get_random(db): random_df = db.extract(from_root("temp\\sql\\get_test_random.sql")) random_df = random_df.sample(n=100) db.insert(random_df, "tmp_random", "dbo") write_df(from_root("temp\\test_data\\random.csv"), random_df)
import os import ssl import torch import torchvision from root import from_root from src.datasets.load_imagenet import IMAGENET_TRAIN_MEAN, IMAGENET_TRAIN_STD OUT_FILEPATH = from_root("weights/softmax_resnet50_imagenet/pretrained.pth") def main(): ssl._create_default_https_context = ssl._create_unverified_context torch_model = torchvision.models.resnet50(pretrained=True) new_state_dict = { "normalize.mean": torch.tensor(IMAGENET_TRAIN_MEAN).reshape(1, -1, 1, 1), "normalize.std": torch.tensor(IMAGENET_TRAIN_STD).reshape(1, -1, 1, 1) } for key, value in torch_model.state_dict().items(): breadcrumbs = key.split(".") # change "layer1" to "stage2" if breadcrumbs[0].startswith("layer"): index = int(breadcrumbs[0][5:]) breadcrumbs[0] = "stage" + str(index + 1) # change "downsample" to "projection" if len(breadcrumbs) >= 3 and breadcrumbs[2] == "downsample":
def annotate(df, observations=False, options=""): """ Returns MetaMap annotations for the given DataFrame. :param df: the DataFrame containing the result_full_descriptions to annotate - required columns: {"test_key", "result_key", "obs_seq_nbr" (if observations is True), "result_full_description"} :param observations: True if the data is given at the observation level, False if the data is given at the test level :param options: MetaMap options; input string of form "-y -D" or "-yD" :return: a DataFrame containing the MetaMap annotations - columns: {"test_key", "result_key", "obs_seq_nbr" (if observations is True), "tags", "candidates"} """ proc = subprocess.Popen( ["java", "-jar", from_root("libs\\MetaMapBuild.jar")], shell=False) # initialize variables to suppress PyCharm warnings gateway = None metamap = None api = None try: connected = False for i in range(5): try: gateway = JavaGateway() metamap = gateway.entry_point api = metamap.getApi() connected = True print(f"Connected to Java server on attempt {i + 1}") break except: time.sleep(1) if not connected: raise Exception("Error connecting to Java server") tags = [] candidates = [] if options.strip(): # Process a string with additional options string api.setOptions(options + " -c") else: # Process a string without additional options string api.setOptions("-c") for description in df["result_full_description"]: result = api.processCitationsFromString(description).get(0) # parse result from MetaMap to tags and candidates tags.append(metamap.formatOneResultToString(result, "tags")) candidates.append( metamap.formatOneResultToString(result, "candidates")) keys = get_keys(observations) return_value = df.loc[:, keys] return_value["tags"] = tags return_value["candidates"] = candidates return return_value finally: if gateway is not None: gateway.shutdown() proc.terminate()
def annotate(df, observations=False, options=""): proc = subprocess.Popen( ["java", "-jar", from_root("libs\\MetaMapBuild.jar")], shell=False) try: attempts = 0 successful = False while attempts < 5: try: gateway = JavaGateway() metamap = gateway.entry_point api = metamap.getApi() print("Connected to Java server on attempt " + str(attempts + 1)) successful = True break except: time.sleep(1) attempts += 1 if not successful: raise Exception("Error connecting to Java server") tags = [] candidates = [] if options.strip(): # Process a string with additional options string api.setOptions(options + " -c") else: # Process a string without additional options string api.setOptions("-c") f = open(from_root("pre-tagging_log.txt"), "w") connected = True for index, row in df.iterrows(): description = row["result_full_description"] test_key = row["test_key"] result_key = row["result_key"] if description is "": tags.append("{}") candidates.append("{}") else: if connected is False: f.write("test_key: " + str(test_key) + " ") f.write("result_key: " + str(result_key) + " ") f.write("MetaMap Connection Error \n") tags.append("MetaMap Connection Errors") candidates.append("MetaMap Connection Errors") else: try: result = api.processCitationsFromString( description).get(0) except Py4JJavaError as e: error_msg = e.java_exception.getMessage() f.write("test_key: " + str(test_key) + " ") f.write("result_key: " + str(result_key) + " ") if "Index 0 out-of-bounds for length 0" in error_msg: f.write("Memory Error \n") tags.append("Memory Error") candidates.append("Memory Error") continue if "Connection refused" in error_msg: f.write("MetaMap Connection Error \n") tags.append("MetaMap Connection Error") candidates.append("MetaMap Connection Error") print( "Quit tagging latter rows due to MetaMap Server connection error." ) connected = False continue except Exception: tags.append("Other Errors") candidates.append("Other Errors") f.write("test_key: " + str(test_key) + " ") f.write("result_key: " + str(result_key) + " ") f.write("Other Error \n") continue # parse result from MetaMap to tags and candidates tags.append(metamap.formatOneResultToString( result, "tags")) candidates.append( metamap.formatOneResultToString(result, "candidates")) print("MetaMap server connection is: ", metamap.isConnectAPI()) if metamap.isConnectAPI() == "true": api.disconnect() if metamap.isConnectAPI() == "false": print("disconnect API from MetaMap server") return_value = df.loc[:, ["test_key", "result_key"]] return_value["tags"] = tags return_value["candidates"] = candidates finally: gateway.shutdown() proc.terminate() return return_value
org_false_results = tp_org_false_results\ .merge(to_org_false_results, how="outer", on=["test_key", "result_key"]) retall_results = l1s_retall_results\ .merge(l2_retall_results, how="outer", on=["test_key", "result_key"]) write_df(from_root("results\\predictions.csv"), results) write_df(from_root("results\\predictions_org_false.csv"), org_false_results) write_df(from_root("results\\predictions_retall.csv"), retall_results) db.insert(results, "predictions", "dbo") print("Finished writing results to CSV and database.") if __name__ == "__main__": print("Started executing script.\n") start_time = datetime.now() logger = logging.getLogger(__name__) set_params(logger, from_root("log\\test.log")) try: main() except Exception as e: logger.exception("test.py: Fatal error") sys.exit(1) print(f"\nExecution time: {datetime.now() - start_time}") print("Finished executing script.")
# Symbolic l1s_module = Level1SymbolicModule(to_module) l1s_module.retrain(l1_df) l1s_module.save_to_file(from_root("pkl\\level_1_symbolic_module.pkl")) # ========================================================================== # Level 2 l2_df = db.extract(from_root("sql\\train\\level_2.sql")) l2_module = Level2Module(l1s_module) l2_module.retrain(l2_df) l2_module.save_to_file(from_root("pkl\\level_2_module.pkl")) if __name__ == "__main__": print("Started executing script.\n") start_time = datetime.now() logger = logging.getLogger(__name__) set_params(logger, from_root("log\\train.log")) try: main() except Exception as e: logger.exception("train.py: Fatal error") sys.exit(1) print(f"\nExecution time: {datetime.now() - start_time}") print("Finished executing script.")
import matplotlib.pyplot as plt from datetime import datetime from io_.db import Database from io_.fs import write_plot from modules.level_1_ml_module import Level1MLModule from modules.level_1_symbolic_module import Level1SymbolicModule from modules.level_2_module import Level2Module from modules.test_outcome_module import TestOutcomeModule from modules.test_performed_module import TestPerformedModule from root import from_root from util.logger import set_params from util.timer import timer TP_SQL = from_root("sql\\train\\test_performed.sql") TO_SQL = from_root("sql\\train\\test_outcome.sql") L1_SQL = from_root("sql\\train\\level_1.sql") L2_SQL = from_root("sql\\train\\level_2.sql") SIZES = [i for i in range(2000, 100000 + 1, 2000)] ORGANISMS = True SAVE_TO = from_root("results\\complexity.png") def main(): db = Database.get_instance() # ========================================================================== # Test performed
import os import random import torchvision from root import from_root from src.misc.utils import set_deterministic_seed, write_lines DATA_DIRPATH = from_root("data/mnist") SPLIT_DIRPATH = from_root("splits/mnist") TRAIN_SIZE = 50000 def main() -> None: set_deterministic_seed() dataset = torchvision.datasets.MNIST(DATA_DIRPATH, train=True, download=True) size = len(dataset) indices = list(range(size)) random.shuffle(indices) train_indices = indices[:TRAIN_SIZE] write_lines(os.path.join(SPLIT_DIRPATH, "train.txt"), train_indices) val_indices = indices[TRAIN_SIZE:] write_lines(os.path.join(SPLIT_DIRPATH, "val.txt"), val_indices) if __name__ == "__main__": main()
import multiprocessing import os from typing import Union import torchvision from torch.utils.data import DataLoader, Subset from torchvision import transforms from root import from_root from src.misc.utils import read_lines DATA_DIRPATH = from_root("data/cifar10") SPLIT_DIRPATH = from_root("splits/cifar10") CIFAR10_TRAIN_MEAN = [0.4913, 0.4820, 0.4464] CIFAR10_TRAIN_STD = [0.2470, 0.2434, 0.2616] def load_cifar10_train(batch_size: int, n_workers: Union[str, int]) -> DataLoader: if batch_size <= 0: raise ValueError("Batch_size must be positive!") if type(n_workers) == str and n_workers != "n_cores": raise ValueError("If n_workers is a string, it must be 'n_cores'!") if type(n_workers) == int and n_workers < 0: raise ValueError("If n_workers is an int, it must be non-negative!") transform = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, 4), transforms.ToTensor() ])
def train_test_outcome(db): to_df = db.extract(from_root("temp\\sql\\train_test_outcome.sql")) to_module = TestOutcomeModule(organisms=True) to_module.retrain(to_df) to_module.save_to_file(from_root("temp\\pkl\\test_outcome_module.pkl")) return to_module
def main(): db = Database.get_instance() # ========================================================================== # Test performed tp_df = db.extract(from_root("sql\\train\\test_performed.sql")) tp_module = TestPerformedModule() tp_module.retrain(tp_df) tp_module.save_to_file(from_root("pkl\\test_performed_module.pkl")) tp_module_org_false = TestPerformedModule(organisms=False) tp_module_org_false.retrain(tp_df) tp_module_org_false.save_to_file( from_root("pkl\\test_performed_organisms_false_module.pkl")) # ========================================================================== # Test outcome to_df = db.extract(from_root("sql\\train\\test_outcome.sql")) to_module = TestOutcomeModule() to_module.retrain(to_df) to_module.save_to_file(from_root("pkl\\test_outcome_module.pkl")) to_module_org_false = TestOutcomeModule(organisms=False) to_module_org_false.retrain(to_df) to_module_org_false.save_to_file( from_root("pkl\\test_outcome_organisms_false_module.pkl")) # ========================================================================== # Level 1 l1_df = db.extract(from_root("sql\\train\\level_1.sql")) # Machine learning l1ml_module = Level1MLModule() l1ml_module.retrain(l1_df) l1ml_module.save_to_file(from_root("pkl\\level_1_ml_module.pkl")) # Symbolic l1s_module = Level1SymbolicModule(to_module) l1s_module.retrain(l1_df) l1s_module.save_to_file(from_root("pkl\\level_1_symbolic_module.pkl")) # ========================================================================== # Level 2 l2_df = db.extract(from_root("sql\\train\\level_2.sql")) l2_module = Level2Module(l1s_module) l2_module.retrain(l2_df) l2_module.save_to_file(from_root("pkl\\level_2_module.pkl"))
def train_level_1_machine_learning(db): l1ml_df = db.extract(from_root("temp\\sql\\train_level_1_ml.sql")) l1ml_module = Level1MLModule() l1ml_module.retrain(l1ml_df) l1ml_module.save_to_file(from_root("temp\\pkl\\level_1_ml_module.pkl")) return l1ml_module
def main(): # ========================================================================== # Load the DataFrames to classify db = Database.get_instance() tp_df = db.extract(from_root("sql\\test\\test_performed.sql")) to_df = db.extract(from_root("sql\\test\\test_outcome.sql")) l1_df = db.extract(from_root("sql\\test\\level_1.sql")) l2_df = db.extract(from_root("sql\\test\\level_2.sql")) print("Finished loading the DataFrames.") # ========================================================================== # Load modules tp_module = TestPerformedModule.load_from_file( from_root("pkl\\test_performed_module.pkl")) to_module = TestOutcomeModule.load_from_file( from_root("pkl\\test_outcome_module.pkl")) l1ml_module = Level1MLModule.load_from_file( from_root("pkl\\level_1_ml_module.pkl")) l1s_module = Level1SymbolicModule(to_module).load_from_file( from_root("pkl\\level_1_symbolic_module.pkl")) l2_module = Level2Module(l1ml_module).load_from_file( from_root("pkl\\level_2_module.pkl")) tp_module_org_false = TestPerformedModule.load_from_file( from_root("pkl\\test_performed_organisms_false_module.pkl")) to_module_org_false = TestOutcomeModule.load_from_file( from_root("pkl\\test_outcome_organisms_false_module.pkl")) print("Finished loading modules.") # ========================================================================== # Classify the DataFrames tp_results = tp_module.classify(tp_df) to_results = to_module.classify(to_df) l1ml_results = l1ml_module.classify(l1_df) l1s_results = l1s_module.classify(l1_df) l2_results = l2_module.classify(l2_df) tp_org_false_results = tp_module_org_false.classify(tp_df) to_org_false_results = to_module_org_false.classify(to_df) l1s_retall_results = l1s_module.classify(l1_df, return_all=True) l2_retall_results = l2_module.classify(l2_df, return_all=True) print("Finished classifying the DataFrames.") # ========================================================================== # Write final prediction results to CSV and database results = tp_results\ .merge(to_results, how="outer", on=["test_key", "result_key"])\ .merge(l1ml_results, how="outer", on=["test_key", "result_key"])\ .merge(l1s_results, how="outer", on=["test_key", "result_key"])\ .merge(l2_results, how="outer", on=["test_key", "result_key"]) org_false_results = tp_org_false_results\ .merge(to_org_false_results, how="outer", on=["test_key", "result_key"]) retall_results = l1s_retall_results\ .merge(l2_retall_results, how="outer", on=["test_key", "result_key"]) write_df(from_root("results\\predictions.csv"), results) write_df(from_root("results\\predictions_org_false.csv"), org_false_results) write_df(from_root("results\\predictions_retall.csv"), retall_results) db.insert(results, "predictions", "dbo") print("Finished writing results to CSV and database.")
def train_level_1_symbolic(db, to_module): l1s_df = db.extract(from_root("sql\\train\\level_1.sql")) l1s_module = Level1SymbolicModule(to_module) l1s_module.retrain(l1s_df) l1s_module.save_to_file( from_root("temp\\pkl\\level_1_symbolic_module.pkl"))
def from_out(self, relative_path: str) -> str: return os.path.join(from_root(self.cfg.out_dirpath), relative_path)
def train_level_2(db, l1ml_module): l2_df = db.extract(from_root("sql\\train\\level_2.sql")) l2_module = Level2Module(l1ml_module) l2_module.retrain(l2_df) l2_module.save_to_file(from_root("temp\\pkl\\level_2_module.pkl"))