def my_app(cfg: Config) -> None: print(OmegaConf.to_yaml(cfg)) working_dir = get_original_cwd() print(f"Orig working directory : {working_dir}") print(f"Current working directory : {os.getcwd()}") if EProgramMode.PrepareData in cfg.mode: logger.info(f"STAGE: PrepareData") prepare_data( cfg.prepare_data, to_absolute_path(cfg.train_dataset), to_absolute_path(cfg.test_dataset), to_absolute_path(cfg.vocab_path), ) if EProgramMode.Train in cfg.mode: logger.info(f"STAGE: Train") main_train_model(cfg) if EProgramMode.Predict in cfg.mode: logger.info(f"STAGE: Train") predict_model(cfg) if EProgramMode.Visualize in cfg.mode: run_visualization( to_absolute_path(cfg.prepare_data.train_file), cfg.train.report_path, cfg.train.dump_model, )
def prepare_data(args: PrepareDataArgs, train_output, test_output, vocab_output): """Runs data processing scripts to turn raw data from (../raw) into cleaned data ready to be analyzed (saved in ../processed). """ logger.info("Processing raw data to final data set") train_df = load_raw_csv(to_absolute_path(args.train_file)) test_df = load_raw_csv(to_absolute_path(args.test_file)) label_encoder = LabelEncoder() label_encoder.fit(train_df.label.values) tokenizer = get_tokenizer(args.tokenizer_name) vocab = build_vocab( train_df.text.values, tokenizer, args.pretrained_vectors, to_absolute_path(args.vectors_cache_directory), ) logger.info(f"Save vocab into {vocab_output}") torch.save(vocab, vocab_output) train_ds = make_dataset( train_df, label_encoder, transforms=None, tokenizer=tokenizer, vocab=vocab ) save_dataset(train_ds, train_output) test_ds = make_dataset( test_df, label_encoder, transforms=None, tokenizer=tokenizer, vocab=vocab ) save_dataset(test_ds, test_output)
def setup(self, stage=None): super().setup(stage=stage) cfg = self.cfg if HydraConfig.initialized(): # Current directory changed! # Update paths with original_cwd csv_path = to_absolute_path(cfg.data.csv_path) root_dir = to_absolute_path(cfg.data.root_dir) else: # Keep relative paths csv_path = cfg.data.csv_path root_dir = cfg.data.root_dir df = pd.read_csv(csv_path) df[["x", "y", "x1", "y1"]] = pd.DataFrame( np.stack(df["box"].apply(ast.literal_eval)).astype(np.float32) ) train_df = df.loc[df["fold"] != cfg.data.valid_fold].copy() valid_df = df.loc[df["fold"] == cfg.data.valid_fold].copy() self.train_dataset = PennFudanDataset( train_df, root_dir=root_dir, transforms=self.train_transforms, mode="train", ) self.val_dataset = PennFudanDataset( valid_df, root_dir=root_dir, transforms=self.val_transforms, mode="val" )
def main(cfg: DictConfig) -> None: # set up mlflow experiment id mlflow.set_tracking_uri(f"file://{to_absolute_path(cfg.path_to_mlflow)}") experiment = mlflow.get_experiment_by_name(cfg.experiment_name) if experiment is not None: # fetch existing experiment id run_kwargs = {'experiment_id': experiment.experiment_id} else: # create new experiment experiment_id = mlflow.create_experiment(cfg.experiment_name) run_kwargs = {'experiment_id': experiment_id} # run the training with mlflow tracking with mlflow.start_run(**run_kwargs) as active_run: setup_gpu(cfg.gpu_cfg) training_cfg = OmegaConf.to_object(cfg.training_cfg) # convert to python dictionary scaling_cfg = to_absolute_path(cfg.scaling_cfg) dataloader = DataLoaderReco.DataLoader(training_cfg, scaling_cfg) dl_config = dataloader.config model = model = MyGNN(dl_config) input_shape, _ = dataloader.get_shape() # print(input_shape[0]) # compile_build = tf.ones(input_shape[0], dtype=tf.float32, name=None) model.build(list(input_shape[0])) compile_model(model, dl_config["SetupNN"]["mode"], dl_config["SetupNN"]["learning_rate"]) fit_hist = run_training(model, dataloader, False, cfg.log_suffix) mlflow.log_dict(training_cfg, 'input_cfg/training_cfg.yaml') mlflow.log_artifact(scaling_cfg, 'input_cfg') mlflow.log_artifact(to_absolute_path("Training_SNNv0.py"), 'input_cfg') mlflow.log_artifact(to_absolute_path("../commonReco.py"), 'input_cfg') mlflow.log_artifacts('.hydra', 'input_cfg/hydra') mlflow.log_artifact('Training_SNNv0.log', 'input_cfg/hydra') mlflow.log_param('run_id', active_run.info.run_id) print(f'\nTraining has finished! Corresponding MLflow experiment name (ID): {cfg.experiment_name}({run_kwargs["experiment_id"]}), and run ID: {active_run.info.run_id}\n')
def get_data_loaders(config): data_loaders = {} for phase in ["train_no_dev", "dev"]: in_dir = to_absolute_path(config.data[phase].in_dir) out_dir = to_absolute_path(config.data[phase].out_dir) train = phase.startswith("train") in_feats = FileSourceDataset(NpyFileSource(in_dir)) out_feats = FileSourceDataset(NpyFileSource(out_dir)) in_feats = MemoryCacheDataset(in_feats, cache_size=10000) out_feats = MemoryCacheDataset(out_feats, cache_size=10000) dataset = Dataset(in_feats, out_feats) data_loaders[phase] = data_utils.DataLoader( dataset, batch_size=config.data.batch_size, collate_fn=collate_fn, pin_memory=config.data.pin_memory, num_workers=config.data.num_workers, shuffle=train) for x, y, l in data_loaders[phase]: logger.info(f"{x.shape}, {y.shape}, {l.shape}") return data_loaders
def preprocess_dataset(cfg): in_dir = Path(utils.to_absolute_path(cfg.in_dir)) out_dir = Path(utils.to_absolute_path(cfg.out_dir)) out_dir.mkdir(parents=True, exist_ok=True) executor = ProcessPoolExecutor(max_workers=cpu_count()) print("Extracting features for train set") futures = [] split_path = out_dir / "train" with open(split_path.with_suffix(".json")) as file: metadata = json.load(file) for in_path, out_path in metadata: wav_path = in_dir / in_path out_path = out_dir / out_path out_path.parent.mkdir(parents=True, exist_ok=True) futures.append( executor.submit(process_wav, wav_path, out_path, cfg.preprocess)) results = [future.result() for future in tqdm(futures)] lengths = {result[0].stem: result[1] for result in results} with open(out_dir / "lengths.json", "w") as file: json.dump(lengths, file, indent=4) frames = sum(lengths.values()) frame_shift_ms = cfg.preprocess.hop_length / cfg.preprocess.sr hours = frames * frame_shift_ms / 3600 print( f"Wrote {len(lengths)} utterances, {frames} frames ({hours:.2f} hours)" )
def my_app(config: DictConfig) -> None: logger = getLogger(config.verbose) logger.info(OmegaConf.to_yaml(config)) utt_list = to_absolute_path(config.utt_list) in_dir = to_absolute_path(config.in_dir) out_dir = to_absolute_path(config.out_dir) utt_ids = load_utt_list(utt_list) stream_sizes, has_dynamic_features = get_world_stream_info( config.acoustic.sample_rate, config.acoustic.mgc_order, config.acoustic.num_windows, config.acoustic.vibrato_mode, ) os.makedirs(out_dir, exist_ok=True) with ProcessPoolExecutor(max_workers=config.max_workers) as executor: futures = [ executor.submit( _extract_static_features, in_dir, out_dir, utt_id, config.acoustic.num_windows, stream_sizes, has_dynamic_features, ) for utt_id in utt_ids ] for future in tqdm(futures): future.result()
def from_hydra(cfg): """Factory method. Instantiates :class:`TetrisScheduling` from a hydra configuration object. Args: cfg: a hydra configuration object """ # Set the platform platform = hydra.utils.instantiate(cfg["platform"]) # Read applications and mappings base_apps_dir = to_absolute_path(cfg["tetris_apps_dir"]) apps = read_applications(base_apps_dir, platform) # Read jobs file reqs = read_requests(to_absolute_path(cfg["input_jobs"]), apps) # Initialize tetris scheduler scheduler = hydra.utils.instantiate(cfg["resource_manager"], platform) manager = ResourceManager(platform, scheduler) tracer = TracePlayer(manager, reqs) management = TetrisManagement(manager, tracer, reqs) return management
def my_app(_cfg): print("Current working directory : {}".format(os.getcwd())) print("Original working directory : {}".format(utils.get_original_cwd())) print("to_absolute_path('foo') : {}".format( utils.to_absolute_path("foo"))) print("to_absolute_path('/foo') : {}".format( utils.to_absolute_path("/foo")))
def __init__( self, train_path: str, train_batch_size: int, train_num_workers: int, val_path: str, val_batch_size: int, val_num_workers: int, test_path: str, test_batch_size: int, test_num_workers: int, word_vocab_file: str, ) -> None: super().__init__() self.train_path = to_absolute_path(train_path) self.train_batch_size = train_batch_size self.train_num_workers = train_num_workers self.val_path = to_absolute_path(val_path) self.val_batch_size = val_batch_size self.val_num_workers = val_num_workers self.test_path = to_absolute_path(test_path) self.test_batch_size = test_batch_size self.test_num_workers = test_num_workers with open(to_absolute_path(word_vocab_file), "r") as f: word_vocab = [word.strip() for word in f.readlines()] self.preprocessor = SpacyPreprocessor(word_vocab)
def my_app(cfg: DictConfig) -> None: batch_size = 400 train_data = get_mp4_data(to_absolute_path(cfg.dataset.train_mp4)) train_labels = get_txt_data(to_absolute_path(cfg.dataset.train_txt)) train_data = train_data[:batch_size] train_labels = train_labels[:batch_size] train(train_data, train_labels, batch_size)
def preprocess_dataset(cfg): in_dir = Path(utils.to_absolute_path(cfg.in_dir)) out_dir = Path(utils.to_absolute_path("datasets")) / str( cfg.dataset.dataset) out_dir.mkdir(parents=True, exist_ok=True) executor = ProcessPoolExecutor(max_workers=cpu_count()) for split in ["train", "test"]: print("Extracting features for {} set".format(split)) futures = [] split_path = out_dir / cfg.dataset.language / split with open(split_path.with_suffix(".json")) as file: metadata = json.load(file) for in_path, start, duration, out_path in metadata: wav_path = in_dir / in_path out_path = out_dir / out_path out_path.parent.mkdir(parents=True, exist_ok=True) futures.append( executor.submit( partial(process_wav, wav_path, out_path, **cfg.preprocessing, offset=start, duration=duration))) results = [future.result() for future in tqdm(futures)] lengths = [x[-1] for x in results] frames = sum(lengths) frame_shift_ms = cfg.preprocessing.hop_length / cfg.preprocessing.sr hours = frames * frame_shift_ms / 3600 print("Wrote {} utterances, {} frames ({:.2f} hours)".format( len(lengths), frames, hours))
def generate_report(config: ReportConfig): out_path = to_absolute_path(config.output_path) in_path = to_absolute_path(config.input_path) data = pd.read_csv(in_path) profile = ProfileReport(data, title="Profiling Report", explorative=True) logger.info("Save report to %s", out_path) check_dir(os.path.split(out_path)[0]) profile.to_file(str(out_path))
def main(cfg): if cfg.wandb.project: import wandb from wandb.keras import WandbCallback wandb.init(project=cfg.wandb.project) callbacks = [WandbCallback()] else: callbacks = [] csv_path = Path(to_absolute_path(__file__)).parent.joinpath( "meta", f"{cfg.data.db}.csv") df = pd.read_csv(str(csv_path)) train, val = train_test_split(df, random_state=42, test_size=0.1) train_gen = ImageSequence(cfg, train, "train") val_gen = ImageSequence(cfg, val, "val") strategy = tf.distribute.MirroredStrategy() with strategy.scope(): model = get_model(cfg) opt = get_optimizer(cfg) scheduler = get_scheduler(cfg) model.compile(optimizer=opt, loss=[ "sparse_categorical_crossentropy", "sparse_categorical_crossentropy" ], metrics=['accuracy']) dir_parent_checkpoint = "/content/drive/My Drive/deep_learning/age-gender-estimation" if os.path.exists(dir_parent_checkpoint): checkpoint_dir = Path(dir_parent_checkpoint).joinpath("checkpoint") else: checkpoint_dir = Path( to_absolute_path(__file__)).parent.joinpath("checkpoint") checkpoint_dir.mkdir(exist_ok=True) print(f"checkpoint_dir: {checkpoint_dir}") filename = "_".join([ cfg.model.model_name, str(cfg.model.img_size), "weights.{epoch:02d}-{val_loss:.2f}.hdf5" ]) callbacks.extend([ LearningRateScheduler(schedule=scheduler), ModelCheckpoint(str(checkpoint_dir) + "/" + filename, monitor="val_loss", verbose=1, save_best_only=True, mode="auto") ]) model.fit(train_gen, epochs=cfg.train.epochs, callbacks=callbacks, validation_data=val_gen, workers=multiprocessing.cpu_count())
def my_app(config: Configuration) -> None: # create the model model = nn.Sequential( K.contrib.VisionTransformer(image_size=32, patch_size=16, embed_dim=128, num_heads=3), K.contrib.ClassificationHead(embed_size=128, num_classes=10), ) # create the dataset train_dataset = torchvision.datasets.CIFAR10( root=to_absolute_path(config.data_path), train=True, download=True, transform=T.ToTensor()) valid_dataset = torchvision.datasets.CIFAR10( root=to_absolute_path(config.data_path), train=False, download=True, transform=T.ToTensor()) # create the dataloaders train_dataloader = torch.utils.data.DataLoader( train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=8, pin_memory=True) valid_daloader = torch.utils.data.DataLoader( valid_dataset, batch_size=config.batch_size, shuffle=True, num_workers=8, pin_memory=True) # create the loss function criterion = nn.CrossEntropyLoss() # instantiate the optimizer and scheduler optimizer = torch.optim.AdamW(model.parameters(), lr=config.lr) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, config.num_epochs * len(train_dataloader)) # define some augmentations _augmentations = nn.Sequential( K.augmentation.RandomHorizontalFlip(p=0.75), K.augmentation.RandomVerticalFlip(p=0.75), K.augmentation.RandomAffine(degrees=10.), K.augmentation.PatchSequential( K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=0.8), grid_size=(2, 2), # cifar-10 is 32x32 and vit is patch 16 patchwise_apply=False, ), ) def augmentations(self, sample: dict) -> dict: out = _augmentations(sample["input"]) return {"input": out, "target": sample["target"]} model_checkpoint = ModelCheckpoint( filepath="./outputs", monitor="top5", ) trainer = ImageClassifierTrainer( model, train_dataloader, valid_daloader, criterion, optimizer, scheduler, config, callbacks={ "augmentations": augmentations, "on_checkpoint": model_checkpoint, } ) trainer.fit()
def segment(cfg): # Algorithm segment_func = getattr(wordseg_algorithms, cfg.wordseg_algorithm.function) # Directories phoneseg_dir = ( Path(utils.to_absolute_path("exp"))/cfg.model/cfg.dataset/cfg.split/ cfg.phoneseg_tag/"intervals" ) # Read phone intervals phoneseg_interval_dict = {} print("Reading: {}".format(phoneseg_dir)) phoneseg_interval_dict = get_intervals_from_dir(phoneseg_dir) utterances = phoneseg_interval_dict.keys() # Segmentation print("Segmenting:") prepared_text = [] for utt_key in utterances: prepared_text.append( " ".join([i[2] + "_" for i in phoneseg_interval_dict[utt_key]]) ) kwargs = dict(cfg.wordseg_algorithm) kwargs.pop("function") word_segmentation = segment_func(prepared_text, **kwargs) # print(prepared_text[:10]) wordseg_interval_dict = {} for i_utt, utt_key in tqdm(enumerate(utterances)): words_segmented = word_segmentation[i_utt].split(" ") word_start = 0 word_label = "" i_word = 0 wordseg_interval_dict[utt_key] = [] for (phone_start, phone_end, phone_label) in phoneseg_interval_dict[utt_key]: word_label += phone_label + "_" if words_segmented[i_word] == word_label: wordseg_interval_dict[utt_key].append(( word_start, phone_end, word_label )) word_label = "" word_start = phone_end i_word += 1 # Write intervals output_dir = ( Path(utils.to_absolute_path("exp"))/cfg.model/cfg.dataset/cfg.split/ cfg.output_tag/"intervals" ) output_dir.mkdir(exist_ok=True, parents=True) print("Writing to: {}".format(output_dir)) for utt_key in tqdm(wordseg_interval_dict): with open((output_dir/utt_key).with_suffix(".txt"), "w") as f: for start, end, label in wordseg_interval_dict[utt_key]: f.write("{:d} {:d} {}\n".format(start, end, label))
def get_training_dataset(cfg: DictConfig = None) -> Dict[str, Dataset]: """ Get training and validation datasets. Parameters ---------- cfg : DictConfig, optional Project configuration, by default None Returns ------- Dict[str, Dataset] {"train": train_dataset, "valid": valid_dataset} """ images_dir = to_absolute_path(cfg.data.images_folder_path) data = pd.read_csv(to_absolute_path(cfg.data.dataset_path)) data["x1"] = data["x"] + data["w"] data["y1"] = data["y"] + data["h"] data["area"] = data["w"] * data["h"] train_ids, valid_ids = train_test_split( data["image_id"].unique(), test_size=cfg.data.validation_split, random_state=cfg.training.seed, ) # for fast training if cfg.training.debug: train_ids = train_ids[:10] valid_ids = valid_ids[:10] train_df = data.loc[data["image_id"].isin(train_ids)] valid_df = data.loc[data["image_id"].isin(valid_ids)] train_augs_list = [ load_obj(i["class_name"])(**i["params"]) for i in cfg["augmentation"]["train"]["augs"] ] train_bbox_params = OmegaConf.to_container( (cfg["augmentation"]["train"]["bbox_params"]) ) train_augs = Compose(train_augs_list, bbox_params=train_bbox_params) valid_augs_list = [ load_obj(i["class_name"])(**i["params"]) for i in cfg["augmentation"]["valid"]["augs"] ] valid_bbox_params = OmegaConf.to_container( (cfg["augmentation"]["valid"]["bbox_params"]) ) valid_augs = Compose(valid_augs_list, bbox_params=valid_bbox_params) train_dataset = XrayDataset(train_df, "train", images_dir, cfg, train_augs) valid_dataset = XrayDataset(valid_df, "valid", images_dir, cfg, valid_augs) return {"train": train_dataset, "valid": valid_dataset}
def convert(cfg): dataset_path = Path(utils.to_absolute_path("datasets")) / cfg.dataset.path with open(dataset_path / "speakers.json") as file: speakers = sorted(json.load(file)) synthesis_list_path = Path(utils.to_absolute_path(cfg.synthesis_list)) with open(synthesis_list_path) as file: synthesis_list = json.load(file) in_dir = Path(utils.to_absolute_path(cfg.in_dir)) out_dir = Path(utils.to_absolute_path(cfg.out_dir)) out_dir.mkdir(exist_ok=True, parents=True) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") encoder = Encoder(**cfg.model.encoder) decoder = Decoder(**cfg.model.decoder) encoder.to(device) decoder.to(device) print("Load checkpoint from: {}:".format(cfg.checkpoint)) checkpoint_path = utils.to_absolute_path(cfg.checkpoint) checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage) encoder.load_state_dict(checkpoint["encoder"]) decoder.load_state_dict(checkpoint["decoder"]) encoder.eval() decoder.eval() for wav_path, speaker_id, out_filename in tqdm(synthesis_list): wav_path = in_dir / wav_path wav, _ = librosa.load( wav_path.with_suffix(".wav"), sr=cfg.preprocessing.sr) wav = wav / np.abs(wav).max() * 0.999 mel = librosa.feature.melspectrogram( preemphasis(wav, cfg.preprocessing.preemph), sr=cfg.preprocessing.sr, n_fft=cfg.preprocessing.n_fft, n_mels=cfg.preprocessing.n_mels, hop_length=cfg.preprocessing.hop_length, win_length=cfg.preprocessing.win_length, fmin=cfg.preprocessing.fmin, power=1) logmel = librosa.amplitude_to_db(mel, top_db=cfg.preprocessing.top_db) logmel = logmel / cfg.preprocessing.top_db + 1 mel = torch.FloatTensor(logmel).unsqueeze(0).to(device) speaker = torch.LongTensor([speakers.index(speaker_id)]).to(device) with torch.no_grad(): z, _ = encoder.encode(mel) output = decoder.generate(z, speaker) path = out_dir / out_filename librosa.output.write_wav(path.with_suffix(".wav"), output.astype(np.float32), sr=cfg.preprocessing.sr)
def run(cfg: DictConfig) -> None: """ Run model inference on the entire test dataset. Parameters ---------- cfg : DictConfig Project configuration object """ device = torch.device("cuda") model = torch.load( to_absolute_path(cfg.logging.best_model_path), map_location=device ) model.eval() detection_threshold = 0.5 results = [] test_dataset = get_test_dataset(cfg) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=cfg.data.batch_size, num_workers=cfg.data.num_workers, shuffle=False, collate_fn=collate_fn, ) for images, _, image_ids in tqdm(test_loader): images = list(image.to(device) for image in images) outputs = model(images) for i, image in enumerate(images): boxes = outputs[i]["boxes"].data.cpu().numpy() scores = outputs[i]["scores"].data.cpu().numpy() labels = outputs[i]["labels"].data.cpu().numpy() boxes = boxes[scores >= detection_threshold].astype(np.int32) labels = labels[scores >= detection_threshold] scores = scores[scores >= detection_threshold] image_id = image_ids[i] result = { "image_id": image_id, "predictions": format_prediction_string(boxes, scores), "labels": ",".join([str(x) for x in labels]), } results.append(result) test_df = pd.DataFrame( results, columns=["image_id", "predictions", "labels"] ) test_df.to_csv(to_absolute_path("predictions.csv"), index=False)
def encode_dataset(cfg): out_dir = Path(utils.to_absolute_path(cfg.out_dir)) out_dir.mkdir(exist_ok=True, parents=True) root_path = Path(utils.to_absolute_path("datasets")) / cfg.dataset.path with open(root_path / "test.json") as file: metadata = json.load(file) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") encoder = Encoder(**cfg.model.encoder) encoder.to(device) print("Load checkpoint from: {}:".format(cfg.checkpoint)) checkpoint_path = utils.to_absolute_path(cfg.checkpoint) checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage) encoder.load_state_dict(checkpoint["encoder"]) encoder.eval() if cfg.save_auxiliary: auxiliary = [] def hook(module, input, output): auxiliary.append(output.clone()) encoder.encoder[-1].register_forward_hook(hook) for _, _, _, path in tqdm(metadata): path = root_path.parent / path mel = torch.from_numpy(np.load( path.with_suffix(".mel.npy"))).unsqueeze(0).to(device) with torch.no_grad(): z, c, indices = encoder.encode(mel) z = z.squeeze().cpu().numpy() out_path = out_dir / path.stem with open(out_path.with_suffix(".txt"), "w") as file: np.savetxt(file, z, fmt="%.16f") if cfg.save_auxiliary: aux_path = out_dir.parent / "auxiliary_embedding1" aux_path.mkdir(exist_ok=True, parents=True) out_path = aux_path / path.stem c = c.squeeze().cpu().numpy() with open(out_path.with_suffix(".txt"), "w") as file: np.savetxt(file, c, fmt="%.16f") aux_path = out_dir.parent / "auxiliary_embedding2" aux_path.mkdir(exist_ok=True, parents=True) out_path = aux_path / path.stem aux = auxiliary.pop().squeeze().cpu().numpy() with open(out_path.with_suffix(".txt"), "w") as file: np.savetxt(file, aux, fmt="%.16f")
def main_train_model(cfg: Config): ds = load_processed_dataset(to_absolute_path(cfg.train_dataset)) vocab = torch.load(to_absolute_path(cfg.vocab_path)) args = cfg.train train_dataset, val_dataset = ds.train_test_split(args.test_size) train_dataloader = make_text_dataloader(train_dataset, args.batch_size, vocab[PAD]) val_dataloader = make_text_dataloader(train_dataset, args.batch_size, vocab[PAD]) model = make_model(args.model, vocab) loss_fn = make_loss_fn(args.loss_fn) optimizer = make_optimizer(args.optimizer, model, args.learning_rate) device = get_device(args.gpu) score_functions = dict( accuracy=accuracy_from_predictions, recall=recall_from_predictions, precision=precision_from_predictions, ) train_one_epoch = make_one_epoch_runner( args.model, model, loss_fn, train_dataloader, optimizer, device, is_train=True, score_functions=score_functions, ) val_one_epoch = make_one_epoch_runner( args.model, model, loss_fn, val_dataloader, None, device, is_train=False, score_functions=score_functions, ) lr_scheduler = make_lr_scheduler(optimizer, args.lr_scheduler) plot_fn = plot_train_val_loss if args.interactive else None results = train_cycle( model, train_one_epoch, val_one_epoch, args.epochs, ensure_path(args.dump_model), lr_scheduler, plot_fn, ) save_train_report(results, ensure_path(args.report_path))
def __init__(self, labels: list, data_cfg: DataConfig, normalize: bool, is_distributed: bool): super().__init__() self.train_path = to_absolute_path(data_cfg.train_path) self.val_path = to_absolute_path(data_cfg.val_path) self.labels = labels self.data_cfg = data_cfg self.spect_cfg = data_cfg.spect self.aug_cfg = data_cfg.augmentation self.normalize = normalize self.is_distributed = is_distributed
def compare_input(cfg, print_grid=False): assert (cfg.compare_input) path_to_artifacts = to_absolute_path( f'{cfg.path_to_mlflow}/{cfg.experiment_id}/{cfg.run_id}/artifacts/') file_cmssw_path = to_absolute_path( f'{cfg.path_to_input_dir}/{cfg.compare_input.input_cmssw}') files_cmssw_python = f'{path_to_artifacts}/predictions/{cfg.sample_alias}/{cfg.input_filename}_pred_input' with open(file_cmssw_path) as file: json_input = json.load(file) data_cmssw = {} for tensor_name in json_input.keys(): data_cmssw[tensor_name] = np.array(json_input[tensor_name]) event = cfg.compare_input.input_python.event index = cfg.compare_input.input_python.tau_index data_python = np.load(f'{files_cmssw_python}/tensor_{event}_{index}.npy', allow_pickle=True)[()] for key in data_cmssw.keys(): print("Shape consistency check:", data_cmssw[key].shape, data_python[key].shape) assert (data_cmssw[key].shape == data_python[key].shape) map_f = FeatureDecoder(cfg) for key in list(json_input.keys()): print("\n--------->", key, "<---------") delta = np.abs(data_cmssw[key] - data_python[key]) if key == list(json_input.keys())[0]: print("cmssw tau:", data_cmssw[key]) print("python tau:", data_python[key]) f_idx = np.where(delta > epsilon) print(f"Inconsistent features:\n") for f in np.unique(f_idx): print(map_f.get(key, f)) else: row_idx, col_idx, f_idx = np.where(delta > epsilon) if row_idx.size != 0: print("cmssw grid:", data_cmssw[key][row_idx[0]][col_idx[0]][f_idx]) print("python grid:", data_python[key][row_idx[0]][col_idx[0]][f_idx]) print(f"Inconsistent features:\n") for f in np.unique(f_idx): print(map_f.get(key, f)) if print_grid and key != list(json_input.keys( ))[0]: # if print & not first tensor (plane features) grid_idx = np.stack([row_idx, col_idx], axis=1).tolist() print("\nInconsistent cells:") print( grid(data_cmssw[key].shape[0], data_cmssw[key].shape[1], grid_idx))
def predict_model(cfg: Config): test_dataset = load_processed_dataset(to_absolute_path(cfg.test_dataset)) vocab = torch.load(to_absolute_path(cfg.vocab_path)) args = cfg.train device = get_device(args.gpu) model = load_model(args.model, vocab, args.dump_model) model.eval() model.to(device) predictions = make_predictions(model, test_dataset, device) save_predictions(predictions, cfg.predict.result_file)
def my_app(config : DictConfig) -> None: global logger logger = getLogger(config.verbose) logger.info(config.pretty()) if use_cuda: from torch.backends import cudnn cudnn.benchmark = config.train.cudnn.benchmark cudnn.deterministic = config.train.cudnn.deterministic logger.info(f"cudnn.deterministic: {cudnn.deterministic}") logger.info(f"cudnn.benchmark: {cudnn.benchmark}") logger.info(f"Random seed: {config.seed}") init_seed(config.seed) device = torch.device("cuda" if use_cuda else "cpu") if config.train.use_detect_anomaly: torch.autograd.set_detect_anomaly(True) logger.info("Set to use torch.autograd.detect_anomaly") # Model model = hydra.utils.instantiate(config.model.netG).to(device) # Optimizer optimizer_class = getattr(optim, config.train.optim.optimizer.name) optimizer = optimizer_class(model.parameters(), **config.train.optim.optimizer.params) # Scheduler lr_scheduler_class = getattr(optim.lr_scheduler, config.train.optim.lr_scheduler.name) lr_scheduler = lr_scheduler_class(optimizer, **config.train.optim.lr_scheduler.params) data_loaders = get_data_loaders(config) # Resume if config.train.resume.checkpoint is not None and len(config.train.resume.checkpoint) > 0: logger.info("Load weights from {}".format(config.train.resume.checkpoint)) checkpoint = torch.load(to_absolute_path(config.train.resume.checkpoint)) model.load_state_dict(checkpoint["state_dict"]) if config.train.resume.load_optimizer: logger.info("Load optimizer state") optimizer.load_state_dict(checkpoint["optimizer_state"]) lr_scheduler.load_state_dict(checkpoint["lr_scheduler_state"]) # Save model definition out_dir = to_absolute_path(config.train.out_dir) os.makedirs(out_dir, exist_ok=True) with open(join(out_dir, "model.yaml"), "w") as f: OmegaConf.save(config.model, f) # Run training loop train_loop(config, device, model, optimizer, lr_scheduler, data_loaders)
def main(cfg): if cfg.wandb.project: import wandb from wandb.keras import WandbCallback wandb.init(project=cfg.wandb.project) callbacks = [WandbCallback()] else: callbacks = [] weight_file = cfg.train.weight_file csv_path = Path(to_absolute_path(__file__)).parent.joinpath("meta", f"{cfg.data.db}.csv") df = pd.read_csv(str(csv_path)) train, val = train_test_split(df, random_state=42, test_size=0.2) train_gen = ImageSequence(cfg, train, "train") val_gen = ImageSequence(cfg, val, "val") strategy = tf.distribute.MirroredStrategy() initial_epoch = 0 if weight_file: _, file_meta, *_ = weight_file.split('.') prev_epoch, new_epoch, _ = file_meta.split('-') initial_epoch = int(prev_epoch) + int(new_epoch) with strategy.scope(): model = get_model(cfg) opt = get_optimizer(cfg) scheduler = get_scheduler(cfg, initial_epoch) model.compile(optimizer=opt, loss=["sparse_categorical_crossentropy", "sparse_categorical_crossentropy"], metrics=['accuracy']) if cfg.train.is_collab: checkpoint_dir = Path(to_absolute_path(__file__)).parent.parent.joinpath('drive', 'MyDrive', 'AgeGenderCheckpoint') else: checkpoint_dir = Path(to_absolute_path(__file__)).parent.joinpath('checkpoints') checkpoint_dir.mkdir(exist_ok=True) filename = "_".join([cfg.model.model_name, str(cfg.model.img_size), f"weights.{initial_epoch:02d}-" + "{epoch:02d}-{val_loss:.2f}.hdf5"]) callbacks.extend([ LearningRateScheduler(schedule=scheduler), get_logger(checkpoint_dir, initial_epoch, cfg.train.lr), ModelCheckpoint(str(checkpoint_dir) + "/" + filename, monitor="val_loss", verbose=1, save_best_only=True, mode="auto") ]) if weight_file: model.load_weights(str(checkpoint_dir) + "/" + weight_file) model.fit(train_gen, epochs=cfg.train.epochs, callbacks=callbacks, validation_data=val_gen, workers=multiprocessing.cpu_count())
def fit_scalenet(cfg): device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') writer = get_tensorboard_writer(cfg) # train and validation data train_data, eval_data = get_data_generators(cfg) # model model, optimizer = get_model(cfg) if cfg.load_from_checkpoint: load_dict = model.load(to_absolute_path(cfg.checkpoint_path)) # optimizer.load_state_dict(load_dict['optimizer']) print( f'Loaded checkpoint from {to_absolute_path(cfg.checkpoint_path)}') model.to(device) model.summary() # fit train_loss = 0. train_accuracy = 0. for step in range(cfg.training_steps): x_train, y_train = get_data(train_data, device) pred = model(x_train) loss = binary_cross_entropy(pred, y_train) loss.backward() optimizer.step() optimizer.zero_grad() train_loss += loss.item() train_accuracy += accuracy(pred, y_train).item() if step % cfg.log_every_steps == 0: log(writer=writer, step=step, loss=train_loss, accuracy=train_accuracy, averaging=cfg.log_every_steps if step != 0 else 1, mode='train') train_loss = 0. train_accuracy = 0. if step % cfg.eval_every_steps == 0: eval_scalenet(cfg=cfg, step=step, model=model, eval_data=eval_data, device=device, writer=writer) if step % cfg.save_every_steps == 0: model.save(to_absolute_path(cfg.checkpoint_path), optimizer)
def predict(predict_config): test_df = read_data(to_absolute_path(predict_config.test_data_path)) test_df = test_df.drop(predict_config.feature_params.target_col, axis=1) model_path = to_absolute_path(predict_config.output_model_path) model = load_model(model_path) transformer = load_transformer( to_absolute_path(predict_config.feature_transformer_path)) test_features = make_features(transformer, test_df) y_pred = pd.DataFrame(model.predict_proba(test_features)[:, 1], columns=["target"]) y_pred.to_csv(to_absolute_path(predict_config.predict_path), index=False)
def CBIR_test(cfg): database = DataBase(cfg) # database.load_svs(path=to_absolute_path('/home/artem/data/PATH-DT-MSU-WSI/WSS1/04.svs'), dataset_name='PATH-DT', # scales=[5, 10, 15, 20, 25, 30, 35, 40]) # database.extract_features('PATH-DT') # database.serialize(to_absolute_path('CBIR_serialized')) database.deserialize(to_absolute_path(cfg.features_serialization.path)) query = np.array(Image.open(to_absolute_path(cfg.query))) search_result = database.search(query, top_n=10, detect_scale=True, log=True) print(search_result)
def my_app(config: DictConfig) -> None: global logger logger = getLogger(config.verbose) logger.info(OmegaConf.to_yaml(config)) in_dir = to_absolute_path(config.in_dir) out_dir = to_absolute_path(config.out_dir) scaler_path = to_absolute_path(config.scaler_path) scaler = joblib.load(scaler_path) inverse = config.inverse num_workers = config.num_workers os.makedirs(out_dir, exist_ok=True) apply_normalization_dir2dir(in_dir, out_dir, scaler, inverse, num_workers)