def main(config, base_model, fine_tuned_model_dir, target_class, seed): config['metrics'] = ["pred_acc"] model, data_loader, loss_fn, metrics = load_model(config, base_model, target_class, seed) if len(target_class) == 10: cp.print_progress("< Base Model >") util.print_setting(data_loader, None, model, loss_fn, metrics, None, None) base_model_evaluation = evaluate(model, data_loader, loss_fn, metrics) model = combine_model(model, fine_tuned_model_dir, target_class) cp.print_progress("< Combined Model >") util.print_setting(data_loader, None, model, loss_fn, metrics, None, None) combined_model_evaluation = evaluate(model, data_loader, loss_fn, metrics)
def evaluate(model, data_loader, loss_fn, metrics): # prepare model for testing device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = model.to(device) model.eval() total_loss = 0.0 total_metrics = torch.zeros(len(metrics)) with torch.no_grad(): for i, (data, target) in enumerate(tqdm(data_loader)): one_hot_target = torch.eye(model.output_size)[target] data, target, one_hot_target = data.to(device), target.to( device), one_hot_target.to(device) output = model(data) # computing loss, metrics on test set loss = loss_fn(output, one_hot_target) # loss = loss_fn(output, target) batch_size = data.shape[0] total_loss += loss.item() * batch_size for i, metric in enumerate(metrics): total_metrics[i] += metric(output, target) * batch_size n_samples = len(data_loader.sampler) log = {'loss': total_loss / n_samples} log.update({ met.__name__: total_metrics[i].item() / n_samples for i, met in enumerate(metrics) }) test_result_str = 'TEST RESULTS\n' for key, val in log.items(): test_result_str += ('\t' + str(key) + ' : ' + str(val) + '\n') cp.print_progress(test_result_str) return log
def fine_tune_model(config, base_model): target_class = config['target_class'] cp.print_progress('Fine tune model with', target_class) config['data_loader']['args']['target_class'] = target_class train_logger = Logger() # setup data_loader instances data_loader = util.get_instance(data_loaders, 'data_loader', config) valid_data_loader = data_loader.split_validation() # build model architecture model = util.get_instance(models, 'model', config) # get function handles of loss and metrics loss_fn = getattr(loss_functions, config['loss']) metrics = [getattr(metric_functions, met) for met in config['metrics']] util.print_setting(data_loader, valid_data_loader, model, loss_fn, metrics, None, None) # build base model trainer = FineTuner(model, loss_fn, metrics, base_model=base_model, config=config, data_loader=data_loader, valid_data_loader=valid_data_loader, train_logger=train_logger, target_class=target_class) trainer.train() cp.print_progress('Fine tuning is completed for ', target_class) return os.path.join(trainer.checkpoint_dir, 'model_best.pth')
def train_base_model(config): cp.print_progress('Training base model') train_logger = Logger() # setup data_loader instances data_loader = util.get_instance(data_loaders, 'data_loader', config) valid_data_loader = data_loader.split_validation() # build model architecture model = util.get_instance(models, 'model', config) # get function handles of loss and metrics loss_fn = getattr(loss_functions, config['loss']) metrics = [getattr(metric_functions, met) for met in config['metrics']] # build optimizer, learning rate scheduler. delete every lines containing lr_scheduler for disabling scheduler trainable_params = filter(lambda p: p.requires_grad, model.parameters()) optimizer = util.get_instance(torch.optim, 'optimizer', config, trainable_params) lr_scheduler = util.get_instance(torch.optim.lr_scheduler, 'lr_scheduler', config, optimizer) util.print_setting(data_loader, valid_data_loader, model, loss_fn, metrics, optimizer, lr_scheduler) trainer = Trainer(model, loss_fn, metrics, optimizer, resume=None, config=config, data_loader=data_loader, valid_data_loader=valid_data_loader, lr_scheduler=lr_scheduler, train_logger=train_logger) cp.print_progress('TRAINER\n', trainer) trainer.train() cp.print_progress('Training base model completed') return os.path.join(trainer.checkpoint_dir, 'model_best.pth')
def main(): parser = argparse.ArgumentParser() parser.add_argument("-k", "--keyword", type=str, required=True, help="keyword for the given evaluation data list") parser.add_argument( "-f", "--summary_file", type=str, help="file containing list of evaluation data to be generated") parser.add_argument("-e", "--extractor", type=str, default="edit_distance_extractor", help="type of extraction algorithm to use") parser.add_argument("-th", "--threshold", type=float, default=0.95, help="threshold for retrieving a window") args = parser.parse_args() data_folder_path = "./kws-gen-data" if not os.path.exists(data_folder_path): cp.print_error("please clone kws-gen-data folder using git submodule") exit() keyword = args.keyword.lower() audio_dir = os.path.join(data_folder_path, "audio_data/" + keyword) if not os.path.exists(audio_dir): cp.print_error("audio data is missing - ", audio_dir) exit() total = sum( [1 for i in open(args.summary_file, "r").readlines() if i.strip()]) cp.print_progress("evaluation data file - ", args.summary_file) # load pre recorded target audios target_audios = [] target_audio_dir = os.path.join(data_folder_path, "target_audio/" + keyword) if not os.path.exists(target_audio_dir): cp.print_error("target audio data is missing - ", target_audio_dir) exit() for file_name in os.listdir(target_audio_dir): target_audios.append( librosa.core.load(os.path.join(target_audio_dir, file_name))[0]) # instantiate extractor extractor = None if args.extractor == "edit_distance_extractor": cp.print_progress("extractor type :", args.extractor, "( threshold :", args.threshold, ", number of target audios : ", len(target_audios), ")") extractor = EditDistanceExtractor(target_audios, args.threshold) # extract similar audio from each audio blocks with open(args.summary_file, "r") as file: reader = csv.reader(file, delimiter=",") for i, line in enumerate(reader): vid = line[0] start_time = line[1] end_time = line[2] wav_file = os.path.join( audio_dir, vid + "~" + start_time + "~" + end_time + ".wav") start_time = int(start_time) end_time = int(end_time) cp.print_progress(i + 1, " / ", total, " - ", wav_file) if not os.path.exists(wav_file): cp.print_warning("audio file is missing - ", wav_file) continue data = librosa.core.load(wav_file, SAMPLE_RATE)[0] extracted_audio_times = extractor.extract_keywords(data) # TODO :: count how many window has been extracted and compare it against true count # TODO :: might be good idea to update threshold if the accuracy is way too low cp.print_progress("evaluation is completed for ", keyword, " - ", total)
def evaluate_models(saved_model_dir, num_iter, step_size): cp.print_progress("< evaluate base model >") num_base_model, base_model_acc = evaluate_base_model(saved_model_dir) cp.print_progress("< base model acc >") cp.print_progress(json.dumps(base_model_acc, indent=4)) cp.print_progress("< evaluate fine tuned model >") num_fine_tuned_model, fine_tuned_model_acc = evaluate_fine_tuned_model( saved_model_dir) cp.print_progress("< fine tuned model acc >") cp.print_progress(json.dumps(fine_tuned_model_acc, indent=4)) cp.print_progress("< evaluate combined model >") num_combined_model, combined_model_acc = evaluate_combined_model( saved_model_dir, num_iter, step_size) cp.print_progress("< combined model acc >") cp.print_progress(json.dumps(combined_model_acc, indent=4)) assert num_base_model == num_combined_model assert num_base_model == num_fine_tuned_model results = {} for loss in EXP_LOSS: results[loss] = {} results[loss]['base_model'] = base_model_acc[loss] results[loss]['fine_tuned_model'] = fine_tuned_model_acc[loss] results[loss]['combined_model'] = combined_model_acc[loss] results['num_model'] = num_base_model return results
nargs='+', type=int, help="target class to fine tune (default: all classes)", default=None) parser.add_argument('-s', '--seed', default=None, type=int, help="random seed") args = parser.parse_args() latest_model = max(os.listdir(args.base_model)) base_model = os.path.join(args.base_model, latest_model, 'model_best.pth') cp.print_progress("base model : ", base_model) if not torch.cuda.is_available(): config = torch.load(base_model, map_location='cpu')['config'] else: config = torch.load(base_model)['config'] if args.device: os.environ["CUDA_VISIBLE_DEVICES"] = args.device if args.target_class is None: args.target_class = np.arange(config['n_class']).tolist() main(config, base_model, args.fine_tuned_model_dir, args.target_class, args.seed)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-k", "--keyword", type=str, required=True, help="target keyword to generate data for") parser.add_argument("-s", "--size", type=int, default=100, help="number of url to collect") parser.add_argument("-a", "--api_key", type=str, required=True, help="API key for youtube data v3 API") data_folder_path = "./kws-gen-data/evaluation_data" if not os.path.exists(data_folder_path): cp.print_error("please clone kws-gen-data folder using git submodule") exit() args = parser.parse_args() keyword = args.keyword.lower() cp.print_progress("keyword is ", keyword) url_fetcher = YoutubeSearcher(args.api_key, keyword) urls = [] plural = inflect.engine() while len(urls) < args.size: url = url_fetcher.next()[0] if not url: cp.print_warning("there are no more urls to process") if url in urls: cp.print_warning("video is already added", url) continue try: video = PyTube(util.get_youtube_url(url)) except Exception as exception: cp.print_error( "failed to generate PyTube representation for vidoe - ", url) continue caption = video.captions.get_by_language_code('en') if not caption: cp.print_warning("no caption available for video - ", url) continue try: srt_captions = caption.generate_srt_captions().lower().split( '\n\n') except Exception as exception: cp.print_error("failed to retrieve for vidoe - ", url) continue keyword_exist = False for captions in srt_captions: if keyword in captions or plural.plural(keyword) in captions: keyword_exist = True break if not keyword_exist: cp.print_warning("keywords never appear in the video - ", url) continue urls.append(url) cp.print_progress(len(urls), " / ", args.size, " - ", url) cp.print_warning(len(urls), "urls are collected for ", keyword) file_path = os.path.join(data_folder_path, keyword + "_url_" + str(args.size) + ".txt") with open(file_path, 'w') as output_file: for url in urls: output_file.write(url + "\n")
def main(config, resume): # # setup data_loader instances # data_loader = get_instance(data_loaders, 'data_loader', config) # setup data_loader instances data_loader = getattr(data_loaders, config['data_loader']['type'])( config['data_loader']['args']['data_dir'], batch_size=512, shuffle=False, validation_split=0.0, training=False, num_workers=2 ) # TODO :: use generic function for printing out model setting cp.print_progress('test DATASET\n', data_loader) # build model architecture model = util.get_instance(models, 'model', config) cp.print_progress('MODEL\n', model) # get function handles of loss and metrics loss_fn = getattr(loss_functions, config['loss']) cp.print_progress('LOSS FUNCTION\n', loss_fn.__name__) metrics = [getattr(metric_functions, met) for met in config['metrics']] cp.print_progress('METRICS\n', [metric.__name__ for metric in metrics]) # load state dict checkpoint = torch.load(resume) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) # prepare model for testing device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = model.to(device) model.eval() total_loss = 0.0 total_metrics = torch.zeros(len(metrics)) with torch.no_grad(): for i, (data, target) in enumerate(tqdm(data_loader)): data, target = data.to(device), target.to(device) output = model(data) # computing loss, metrics on test set loss = loss_fn(output, target) batch_size = data.shape[0] total_loss += loss.item() * batch_size for i, metric in enumerate(metrics): total_metrics[i] += metric(output, target) * batch_size n_samples = len(data_loader.sampler) log = {'loss': total_loss / n_samples} log.update({met.__name__ : total_metrics[i].item() / n_samples for i, met in enumerate(metrics)}) test_result_str = 'TEST RESULTS\n' for key, val in log.items(): test_result_str += ('\t' + str(key) + ' : ' + str(val) + '\n') cp.print_progress(test_result_str)