def query_directory(self, first_time): query_url = utils.gs.get_selector(self.server, utils.gs.QUERY) payload = self.payload.create_query_payload(self.pending) Logger.log(LogLevel.DEBUG, payload) try: if first_time: resp = requests.post(query_url, data=payload, headers=self.headers, verify=self.verify) self.set_cookie(resp) Logger.log(LogLevel.DEBUG, 'set cookie=', self.cookies) else: Logger.log(LogLevel.DEBUG, 'cookie=', self.cookies) resp = requests.post(query_url, data=payload, headers=self.headers, cookies=self.cookies, verify=self.verify) if not self.handle_response(resp, first_time): return False except IOError as e: Logger.log(LogLevel.ERROR, 'IO_ERROR', e) return False return True
def run(_run, _config, _log): # check args sanity _config = args_sanity_check(_config, _log) #TODO why use SimpleNamespace args = SimpleNamespace(**_config) args.device = "cuda" if args.use_cuda else "cpu" # setup loggers logger = Logger(_log) _log.info("Experiment Parameters:") experiment_params = pprint.pformat(_config, indent=4, width=1) _log.info("\n\n" + experiment_params + "\n") # configure tensorboard logger unique_token = "{}__{}".format( args.name, datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) args.unique_token = unique_token if args.use_tensorboard: tb_logs_direc = os.path.join(dirname(dirname(abspath(__file__))), "results", "tb_logs") tb_exp_direc = os.path.join(tb_logs_direc, "{}").format(unique_token) logger.setup_tb(tb_exp_direc) # sacred is on by default logger.setup_sacred(_run) # Run and train run_sequential(args=args, logger=logger) # Clean up after finishing print("Exiting Main") print("Stopping all threads") for t in threading.enumerate(): if t.name != "MainThread": print("Thread {} is alive! Is daemon: {}".format(t.name, t.daemon)) t.join(timeout=1) print("Thread joined") print("Exiting script") # Making sure framework really exits os._exit(os.EX_OK)
def main(args): np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.benchmark = True sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt')) trainset = Feeder(args.feat_path, args.knn_graph_path, args.label_path, args.seed, args.k_at_hop, args.active_connection) trainloader = DataLoader(trainset, batch_size=args.batch_size, num_workers=args.workers, shuffle=True, pin_memory=True) net = model.gcn().cuda() opt = torch.optim.SGD(net.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) criterion = nn.CrossEntropyLoss().cuda() save_checkpoint({ 'state_dict': net.state_dict(), 'epoch': 0, }, False, fpath=osp.join(args.logs_dir, 'epoch_{}.ckpt'.format(0))) for epoch in range(args.epochs): adjust_lr(opt, epoch) train(trainloader, net, criterion, opt, epoch) save_checkpoint({ 'state_dict': net.state_dict(), 'epoch': epoch + 1, }, False, fpath=osp.join(args.logs_dir, 'epoch_{}.ckpt'.format(epoch + 1)))
def my_main(_run, _config, _log): global mongo_client import datetime # arglist = parse_args() # unique_token = "{}__{}".format(arglist.name, datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) # run the framework # run(_run, _config, _log, mongo_client, unique_token) logger = Logger(_log) # configure tensorboard logger unique_token = "{}__{}".format( _config["label"], datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) use_tensorboard = False if use_tensorboard: tb_logs_direc = os.path.join(dirname(dirname(abspath(__file__))), "results", "tb_logs") tb_exp_direc = os.path.join(tb_logs_direc, "{}").format(unique_token) logger.setup_tb(tb_exp_direc) logger.setup_sacred(_run) _log.info("Experiment Parameters:") import pprint experiment_params = pprint.pformat(_config, indent=4, width=1) _log.info("\n\n" + experiment_params + "\n") # START THE TRAINING PROCESS runner = Runner(logger) runner.load(_config) runner.reset() # args = vars(arglist) runner.run(_config) # runner.run(args) # train(arglist, logger, _config) # arglist = convert(_config) # train(arglist) # force exit os._exit(0)
def run(_run, _config, _log, pymongo_client): # check args sanity _config = args_sanity_check(_config, _log) args = SN(**_config) args.device = "cuda" if args.use_cuda else "cpu" # setup loggers logger = Logger(_log) _log.info("Experiment Parameters:") experiment_params = pprint.pformat(_config, indent=4, width=1) _log.info("\n\n" + experiment_params + "\n") # configure tensorboard logger unique_token = "{}__{}".format( args.name, datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) args.unique_token = unique_token if args.use_tensorboard: tb_logs_direc = os.path.join(dirname(dirname(abspath(__file__))), "results", "tb_logs") tb_exp_direc = os.path.join(tb_logs_direc, "{}").format(unique_token) logger.setup_tb(tb_exp_direc) # sacred is on by default logger.setup_sacred(_run) # Run and train if args.cross_play and args.evaluate: run_sequential_cross(args=args, logger=logger) else: run_sequential(args=args, logger=logger) # Clean up after finishing print("Exiting Main") if pymongo_client is not None: print("Attempting to close mongodb client") pymongo_client.close() print("Mongodb client closed")
def upload_directory(self): # Use copy of the list for proper removal res = True for file_data in self.pending.values(): if not file_data.upload: continue try: session = requests.Session() json_request = self.payload.create_upload_payload(file_data) Logger.log(LogLevel.DEBUG, json_request) upload_url = utils.gs.get_selector(self.server, utils.gs.UPLOAD) with open(file_data.file_path, 'rb') as f: form = MultipartEncoder({ "request": json_request, "file": f, }) headers = self.headers headers["Content-Type"] = form.content_type resp = session.post(upload_url, headers=headers, data=form, cookies=self.cookies, verify=self.verify) Logger.log(LogLevel.DEBUG, resp) if not self.handle_response(resp): raise Exception('Failed to handle upload response') except Exception as e: Logger.log(LogLevel.ERROR, 'Uploading Error', e) res = False continue return res
def evaluate(epoch, data, model, criterion): """Main evaluation procedure. Arguments: epoch -- current epoch data -- DataLoader which can provide validation batches model -- model to be evaluated criterion -- instance of loss function to measure performance """ model.eval() # initialize counters, etc. mcd, mcd_count = 0, 0 cla, cla_count = 0, 0 eval_losses = {} # loop through epoch batches with torch.no_grad(): for i, batch in enumerate(data): # parse batch batch = list(map(to_gpu, batch)) src, src_len, trg_mel, trg_lin, trg_len, stop_trg, spkrs, langs = batch # run the model (twice, with and without teacher forcing) post_pred, pre_pred, stop_pred, alignment, spkrs_pred, enc_output = model( src, src_len, trg_mel, trg_len, spkrs, langs, 1.0) post_pred_0, _, stop_pred_0, alignment_0, _, _ = model( src, src_len, trg_mel, trg_len, spkrs, langs, 0.0) stop_pred_probs = torch.sigmoid(stop_pred_0) # evaluate loss function post_trg = trg_lin if hp.predict_linear else trg_mel classifier = model._reversal_classifier if hp.reversal_classifier else None loss, batch_losses = criterion(src_len, trg_len, pre_pred, trg_mel, post_pred, post_trg, stop_pred, stop_trg, alignment, spkrs, spkrs_pred, enc_output, classifier) # compute mel cepstral distorsion for j, (gen, ref, stop) in enumerate( zip(post_pred_0, trg_mel, stop_pred_probs)): stop_idxes = np.where(stop.cpu().numpy() > 0.5)[0] stop_idx = min( np.min(stop_idxes) + hp.stop_frames, gen.size()[1]) if len(stop_idxes) > 0 else gen.size()[1] gen = gen[:, :stop_idx].data.cpu().numpy() ref = ref[:, :trg_len[j]].data.cpu().numpy() if hp.normalize_spectrogram: gen = audio.denormalize_spectrogram( gen, not hp.predict_linear) ref = audio.denormalize_spectrogram(ref, True) if hp.predict_linear: gen = audio.linear_to_mel(gen) mcd = (mcd_count * mcd + audio.mel_cepstral_distorision( gen, ref, 'dtw')) / (mcd_count + 1) mcd_count += 1 # compute adversarial classifier accuracy if hp.reversal_classifier: input_mask = lengths_to_mask(src_len) trg_spkrs = torch.zeros_like(input_mask, dtype=torch.int64) for s in range(hp.speaker_number): speaker_mask = (spkrs == s) trg_spkrs[speaker_mask] = s matches = (trg_spkrs == torch.argmax( torch.nn.functional.softmax(spkrs_pred, dim=-1), dim=-1)) matches[~input_mask] = False cla = (cla_count * cla + torch.sum(matches).item() / torch.sum(input_mask).item()) / (cla_count + 1) cla_count += 1 # add batch losses to epoch losses for k, v in batch_losses.items(): eval_losses[k] = v + eval_losses[k] if k in eval_losses else v # normalize loss per batch for k in eval_losses.keys(): eval_losses[k] /= len(data) # log evaluation Logger.evaluation(epoch + 1, eval_losses, mcd, src_len, trg_len, src, post_trg, post_pred, post_pred_0, stop_pred_probs, stop_trg, alignment_0, cla) return sum(eval_losses.values())
def main(args): if not os.path.exists(args.logs_dir): os.mkdir(args.logs_dir) if not os.path.exists(args.tensorboard_dir): os.mkdir(args.tensorboard_dir) tensorboardWrite = SummaryWriter(log_dir = args.tensorboard_dir) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) cudnn.benchmark = True device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # log file if args.evaluate == 1: sys.stdout = Logger(osp.join(args.logs_dir, 'log_test.txt')) else: sys.stdout = Logger(osp.join(args.logs_dir, 'log_train.txt')) print("==========\nArgs:{}\n==========".format(args)) print("Initializing dataset {}".format(args.dataset)) # from reid.data import get_data , dataset, num_classes, train_loader, query_loader, gallery_loader = \ get_data(args, args.dataset, args.split, args.data_dir, args.batch_size, args.seq_len, args.seq_srd, args.workers) print('[len] train: {}, query: {}, gallery: {}'.format(*list(map(len, [train_loader, query_loader, gallery_loader])))) # create CNN model # cnn_model = models.create(args.a1, args.flow1, args.flow2, num_features=args.features, dropout=args.dropout) cnn_model_flow = [models.create(args.a1, args.flow1, num_features=args.features, dropout=args.dropout)] if any(args.flow2): cnn_model_flow.append(models.create(args.a1, args.flow2, num_features=args.features, dropout=args.dropout)) # cnn_model_flow1 = cnn_model_flow1.cuda() # cnn_model_flow2 = cnn_model_flow2.cuda() # create ATT model input_num = cnn_model_flow[0].feat.in_features # 2048 output_num = args.features # 128 att_model = models.create(args.a2, input_num, output_num) # att_model.cuda() # # ------peixian:tow attmodel------ # att_model_flow1 = models.create(args.a2, input_num, output_num) # att_model_flow2 = models.create(args.a2, input_num, output_num) # # -------------------------------- # create classifier model class_num = 2 classifier_model = models.create(args.a3, output_num, class_num) # classifier_model.cuda() # CUDA acceleration model # cnn_model = torch.nn.DataParallel(cnn_model).to(device) # # ------peixian:tow attmodel------ # for att_model in [att_model_flow1, att_model_flow2]: # att_model = att_model.to(device) # # -------------------------------- att_model = att_model.cuda() classifier_model = classifier_model.cuda() # cnn_model = torch.nn.DataParallel(cnn_model).cuda() # cnn_model_flow1 = torch.nn.DataParallel(cnn_model_flow1,device_ids=[0,1,2]) # cnn_model_flow2 = torch.nn.DataParallel(cnn_model_flow2,device_ids=[0,1,2]) # cnn_model_flow[0].cuda() cnn_model_flow[0] = torch.nn.DataParallel(cnn_model_flow[0],device_ids=[0]) if len(cnn_model_flow) > 1: cnn_model_flow[1].cuda() cnn_model_flow[1] = torch.nn.DataParallel(cnn_model_flow[1],device_ids=[0]) # att_model = torch.nn.DataParallel(att_model,device_ids=[1,2,3]) # classifier_model = torch.nn.DataParallel(classifier_model,device_ids=[1,2,3]) criterion_oim = OIMLoss(args.features, num_classes, scalar=args.oim_scalar, momentum=args.oim_momentum) criterion_veri = PairLoss(args.sampling_rate) criterion_oim.cuda() criterion_veri.cuda() # criterion_oim.cuda() # criterion_veri.cuda() # Optimizer optimizer1 = [] # cnn_model_flow = [cnn_model_flow1, cnn_model_flow2] for cnn_model in range(len(cnn_model_flow)): base_param_ids = set(map(id, cnn_model_flow[cnn_model].module.base.parameters())) new_params = [p for p in cnn_model_flow[cnn_model].module.parameters() if id(p) not in base_param_ids] param_groups1 = [ {'params': cnn_model_flow[cnn_model].module.base.parameters(), 'lr_mult': 1}, {'params': new_params, 'lr_mult': 1}] optimizer1.append(torch.optim.SGD(param_groups1, lr=args.lr1, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True)) param_groups2 = [ {'params': att_model.parameters(), 'lr_mult': 1}, {'params': classifier_model.parameters(), 'lr_mult': 1}] optimizer2 = torch.optim.SGD(param_groups2, lr=args.lr2, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) # optimizer1 = torch.optim.Adam(param_groups1, lr=args.lr1, weight_decay=args.weight_decay) # # optimizer2 = torch.optim.Adam(param_groups2, lr=args.lr2, weight_decay=args.weight_decay) # Schedule Learning rate def adjust_lr1(epoch): lr = args.lr1 * (0.1 ** (epoch/args.lr1step)) print(lr) for o in optimizer1: for g in o.param_groups: g['lr'] = lr * g.get('lr_mult', 1) def adjust_lr2(epoch): lr = args.lr2 * (0.01 ** (epoch//args.lr2step)) print(lr) for g in optimizer2.param_groups: g['lr'] = lr * g.get('lr_mult', 1) # # peixian: two attmodel: # for o in optimizer2: # for g in o.param_groups: # g['lr'] = lr * g.get('lr_mult', 1) # # def adjust_lr3(epoch): lr = args.lr3 * (0.000001 ** (epoch //args.lr3step)) print(lr) return lr # Trainer trainer = SEQTrainer(cnn_model_flow, att_model, classifier_model, criterion_veri, criterion_oim, args.lr3, args.flow1rate) # Evaluator evaluator = ATTEvaluator(cnn_model_flow, att_model, classifier_model, args.flow1rate) best_top1 = 0 if args.evaluate == 1 or args.pretrain == 1: # evaluate for cnn_model in range(len(cnn_model_flow)): checkpoint = load_checkpoint(osp.join(args.logs_dir, 'cnnmodel_best_flow' + str(cnn_model) + '.pth.tar')) cnn_model_flow[cnn_model].module.load_state_dict(checkpoint['state_dict']) checkpoint = load_checkpoint(osp.join(args.logs_dir, 'attmodel_best.pth.tar')) att_model.load_state_dict(checkpoint['state_dict']) checkpoint = load_checkpoint(osp.join(args.logs_dir, 'clsmodel_best.pth.tar')) classifier_model.load_state_dict(checkpoint['state_dict']) top1 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo) # top1 = evaluator.evaluate(query_loader, gallery_loader,dataset.num_tracklet) if args.evaluate == 0: for epoch in range(args.start_epoch, args.epochs): adjust_lr1(epoch) adjust_lr2(epoch) rate = adjust_lr3(epoch) trainer.train(epoch, train_loader, optimizer1, optimizer2, rate,tensorboardWrite) if (epoch+1) % 1 == 0 or (epoch+1) == args.epochs: top1 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo) is_best = top1 > best_top1 if is_best: best_top1 = top1 for cnn_model in range(len(cnn_model_flow)): save_cnn_checkpoint({ 'state_dict': cnn_model_flow[cnn_model].module.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, index=cnn_model, fpath=osp.join(args.logs_dir, 'cnn_checkpoint_flow'+str(cnn_model)+'.pth.tar')) save_att_checkpoint({ 'state_dict': att_model.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=osp.join(args.logs_dir, 'att_checkpoint.pth.tar')) save_cls_checkpoint({ 'state_dict': classifier_model.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=osp.join(args.logs_dir, 'cls_checkpoint.pth.tar'))
def main(args): np.random.seed(args.seed) torch.manual_seed(args.seed) # cudnn.benchmark = True # Redirect print to both console and log file if not args.evaluate: sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt')) # Create data loaders if args.height is None or args.width is None: args.height, args.width = (144, 56) if args.arch == 'inception' else \ (240, 240) dataset, num_classes, train_loader, val_loader, test_loader = \ get_data(args.dataset, args.split, args.data_dir, args.height, args.width, args.batch_size, args.workers, args.combine_trainval) # Create model img_branch = models.create(args.arch, cut_layer=args.cut_layer, num_classes=num_classes, num_features=args.features) diff_branch = models.create(args.arch, cut_layer=args.cut_layer, num_classes=num_classes, num_features=args.features) # Load from checkpoint start_epoch = best_top1 = 0 if args.resume: checkpoint = load_checkpoint(args.resume) img_branch.load_state_dict(checkpoint['state_dict_img']) diff_branch.load_state_dict(checkpoint['state_dict_diff']) start_epoch = checkpoint['epoch'] best_top1 = checkpoint['best_top1'] print("=> Start epoch {} best top1 {:.1%}".format( start_epoch, best_top1)) img_branch = nn.DataParallel(img_branch).cuda() diff_branch = nn.DataParallel(diff_branch).cuda() # img_branch = nn.DataParallel(img_branch) # diff_branch = nn.DataParallel(diff_branch) # Criterion criterion = nn.CrossEntropyLoss().cuda() # criterion = nn.CrossEntropyLoss() # Evaluator evaluator = Evaluator(img_branch, diff_branch, criterion) if args.evaluate: # print("Validation:") # top1, _ = evaluator.evaluate(val_loader) # print("Validation acc: {:.1%}".format(top1)) print("Test:") top1, (gt, pred) = evaluator.evaluate(test_loader) print("Test acc: {:.1%}".format(top1)) from confusion_matrix import plot_confusion_matrix plot_confusion_matrix(gt, pred, dataset.classes, args.logs_dir) return img_param_groups = [ { 'params': img_branch.module.low_level_modules.parameters(), 'lr_mult': 0.1 }, { 'params': img_branch.module.high_level_modules.parameters(), 'lr_mult': 0.1 }, { 'params': img_branch.module.classifier.parameters(), 'lr_mult': 1 }, ] diff_param_groups = [ { 'params': diff_branch.module.low_level_modules.parameters(), 'lr_mult': 0.1 }, { 'params': diff_branch.module.high_level_modules.parameters(), 'lr_mult': 0.1 }, { 'params': diff_branch.module.classifier.parameters(), 'lr_mult': 1 }, ] img_optimizer = torch.optim.SGD(img_param_groups, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) diff_optimizer = torch.optim.SGD(diff_param_groups, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) # Trainer trainer = Trainer(img_branch, diff_branch, criterion) # Schedule learning rate def adjust_lr(epoch): step_size = args.step_size lr = args.lr * (0.1**(epoch // step_size)) for g in img_optimizer.param_groups: g['lr'] = lr * g.get('lr_mult', 1) for g in diff_optimizer.param_groups: g['lr'] = lr * g.get('lr_mult', 1) # Start training for epoch in range(start_epoch, args.epochs): adjust_lr(epoch) trainer.train(epoch, train_loader, img_optimizer, diff_optimizer) if epoch < args.start_save: continue top1, _ = evaluator.evaluate(val_loader) is_best = top1 > best_top1 best_top1 = max(top1, best_top1) save_checkpoint( { 'state_dict_img': img_branch.module.state_dict(), 'state_dict_diff': diff_branch.module.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=osp.join(args.logs_dir, 'checkpoint.pth.tar')) print('\n * Finished epoch {:3d} top1: {:5.1%} best: {:5.1%}{}\n'. format(epoch, top1, best_top1, ' *' if is_best else '')) # Final test print('Test with best model:') checkpoint = load_checkpoint(osp.join(args.logs_dir, 'model_best.pth.tar')) img_branch.module.load_state_dict(checkpoint['state_dict_img']) diff_branch.module.load_state_dict(checkpoint['state_dict_diff']) top1, (gt, pred) = evaluator.evaluate(test_loader) from confusion_matrix import plot_confusion_matrix plot_confusion_matrix(gt, pred, dataset.classes, args.logs_dir) print('\n * Test Accuarcy: {:5.1%}\n'.format(top1))
def main(): parser = argparse.ArgumentParser( description='Threat Prevention API example') files_argument_group = parser.add_mutually_exclusive_group( required=not IS_ONLY_GENERATE_TOKEN) files_argument_group.add_argument('-D', '--directory', help='The scanning directory') files_argument_group.add_argument('-fp', '--file_path', help='Path to file') parser.add_argument('-fn', '--file_name', help='File Name, relevant when file path supplied') parser.add_argument( '-R', '--recursive', action='store_true', help= 'Emulate the files in the directory recursively, relevant when scanning directory supplied' ) server_argument_group = parser.add_mutually_exclusive_group( required=not IS_ONLY_GENERATE_TOKEN) server_argument_group.add_argument('-k', '--key', help='API key') server_argument_group.add_argument('-e', '--sandblast_appliance', help='Check Point SandBlast Appliance') server_argument_group.add_argument( '-ci', '--client_id', nargs=2, metavar=('CLIENT_ID', 'ACCESS_KEY'), help= 'Client ID and Access key, used for JWT token authenticated requests') parser.add_argument( '-gt', '--generate_token', action='store_true', help='Only create the JWT token without sending a request') parser.add_argument('-d', '--debug', action='store_true', help='Add debugging') blades_info = parser.add_argument_group('Blades info') blades_info.add_argument('-t', '--te', action='store_true', help='Activate Threat Emulation') blades_info.add_argument( '--tex', action='store_true', help='Activate Threat Extraction (supported only with cloud)') blades_info.add_argument( '--tex_folder', help= 'A folder to download the Scrubbing attachments (required when TEX is active)' ) blades_info.add_argument( '-m', '--tex_method', choices=['convert', 'clean'], default='convert', help='Scrubbing method. Convert to PDF / CleanContent') reports_section = parser.add_argument_group('Reports info', 'Download Reports') reports_section.add_argument( '-r', '--reports', help='A folder to download the reports to (required for cloud)', required=False) reports_section.add_argument( '-p', '--pdf', action='store_true', help='Download PDF reports', ) reports_section.add_argument( '-x', '--xml', action='store_true', help='Download XML reports', ) reports_section.add_argument( '-s', '--summary', action='store_true', help='Download summary reports', ) args = parser.parse_args() Logger.level = LogLevel.DEBUG if args.debug else LogLevel.INFO # Asking the API to enable features and reports according # to what was required by the user. features = [] reports = [] server = "" key = "" client_id = "" access_key = "" file_path = "" file_name = "" directory = "" args.te and features.append('te') args.tex and features.append('extraction') if (args.summary and args.pdf): parser.error( "Illegal request. Pdf reports are not available in the new Threat Emulation reports format. Requesting for pdf and summary reports simultaneously is not supported." ) exit(-1) args.xml and reports.append('xml') args.pdf and reports.append('pdf') args.summary and reports.append('summary') # Verify the user values if len(reports) and not args.reports: parser.error("Please supply a reports directory") exit(-1) if args.key: key = args.key if not args.reports: parser.error("API Key supplied, please supply a reports folder") exit(-1) elif args.client_id: client_id = args.client_id[0] access_key = args.client_id[1] if not args.generate_token and not args.reports: parser.error("API Token supplied, please supply a reports folder") exit(-1) elif args.sandblast_appliance: if args.tex: Logger.log( LogLevel.ERROR, 'TEX is not supported with Check Point SandBlast Appliance') features.remove('extraction') server = args.sandblast_appliance if args.tex: if not args.tex_folder: parser.error("TEX is active, please supply a tex folder") exit(-1) if not os.path.isdir(args.tex_folder): Logger.log(LogLevel.ERROR, 'Invalid tex folder as input') exit(-1) if not args.generate_token: if args.directory: if not os.path.isdir(args.directory): Logger.log(LogLevel.ERROR, 'Invalid scanning directory in input') exit(-1) directory = args.directory else: file_path = args.file_path.encode('utf-8') if args.file_name and args.file_name != 0: file_name = args.file_name.encode('utf-8') else: file_name = os.path.basename(file_path) if not os.path.isfile(args.file_path): Logger.log(LogLevel.ERROR, 'Invalid file path in input (%s)' % args.file_path) exit(-1) api = Run(directory, file_path, file_name, key, client_id, access_key, args.generate_token, server, args.reports, args.tex_method, args.tex_folder, features, reports, args.recursive) if not api.is_pending_files(): Logger.log(LogLevel.INFO, 'The directory is empty') exit(0) if directory: Logger.log( LogLevel.INFO, 'Querying %d files from directory: %s' % (len(api.pending), args.directory)) else: Logger.log(LogLevel.INFO, 'Querying file: %s ' % (file_path)) api.query_directory(True) api.print_arrays_status() if api.is_pending_files(): Logger.log(LogLevel.INFO, 'UPLOADING'), api.upload_directory() api.print_arrays_status() max_tries = MAX_TRIES while api.is_pending_files() and max_tries > 0: time.sleep(WAITING_SEC) api.query_directory(False) api.print_arrays_status() max_tries -= 1 api.print_arrays() ret = api.get_final_status() print("return {}".format(ret)) exit(ret)
def create_meta_file(dataset_name, dataset_root_dir, output_metafile_name, audio_sample_rate, num_fft_freqs, spectrograms=True, phonemes=True): """Create the meta-file and spectrograms (mel and linear, optionally) or phonemized utterances (optionally). Format details: Every line of the metadata file contains info about one dataset item. The line has following format 'id|speaker|language|audio_file_path|mel_spectrogram_path|linear_spectrogram_path|text|phonemized_text' And the following must hold 'audio_file_path' can be empty if loading just spectrograms 'text' should be carefully normalized and should contain interpunction 'phonemized_text' can be empty if loading just raw text Arguments: dataset_name (string): Name of the dataset, loaders.py should contain a function for loading with a corresponding name. dataset_root_dir (string): Root directory from which is the dataset build and to which are spectrograms and the meta-file saved.. output_metafile_name (string): Name of the output meta-file. audio_sample_rate (int): Sample rate of audios, used if spectrograms is set True. num_fft_freqs (int): Number of frequency bands used during spectrogram computation, used if spectrograms is set True. Keyword arguments: spectrograms (boolean, default True): If true, spetrograms (both mel and linear) are computed and saved. phonemes (boolean, default True): If true, phonemized variants of utterances are computed and saved. """ # save current sample rate and fft freqs hyperparameters, as we may process dataset with different sample rate if spectrograms: old_sample_rate = hp.sample_rate hp.sample_rate = audio_sample_rate old_fft_freqs = hp.num_fft hp.num_fft = num_fft_freqs # load metafiles, an item is a list like: [text, audiopath, speaker_id, language_code] items = loaders.get_loader_by_name(dataset_name)(dataset_root_dir) # build dictionaries for translation to IPA from source languages, see utils.text for details if phonemes: text_lang_pairs = [(i[0], hp.languages[0] if i[3] == "" else i[3]) for i in items] phoneme_dicts = text.build_phoneme_dicts(text_lang_pairs) # prepare directories which will store spectrograms if spectrograms: spectrogram_dirs = [os.path.join(dataset_root_dir, 'spectrograms'), os.path.join(dataset_root_dir, 'linear_spectrograms')] for x in spectrogram_dirs: if not os.path.exists(x): os.makedirs(x) # iterate through items and build the meta-file metafile_path = os.path.join(dataset_root_dir, output_metafile_name) with open(metafile_path, 'w', encoding='utf-8') as f: Logger.progress(0, prefix='Building metafile:') for i in range(len(items)): raw_text, audio_path, speaker, language = items[i] if language == "": language = hp.languages[0] phonemized_text = text.to_phoneme(raw_text, False, language, phoneme_dicts[language]) if phonemes else "" spectrogram_paths = "|" if spectrograms: spec_name = f'{str(i).zfill(6)}.npy' audio_data = audio.load(os.path.join(dataset_root_dir, audio_path)) np.save(os.path.join(spectrogram_dirs[0], spec_name), audio.spectrogram(audio_data, True)) np.save(os.path.join(spectrogram_dirs[1], spec_name), audio.spectrogram(audio_data, False)) spectrogram_paths = os.path.join('spectrograms', spec_name) + '|' + os.path.join('linear_spectrograms', spec_name) print(f'{str(i).zfill(6)}|{speaker}|{language}|{audio_path}|{spectrogram_paths}|{raw_text}|{phonemized_text}', file=f) Logger.progress((i + 1) / len(items), prefix='Building metafile:') # restore the original sample rate and fft freq values if spectrograms: hp.sample_rate = old_sample_rate hp.num_fft = old_fft_freqs
import os from utils.logging import Logger logger = Logger().logger class AppConfig: """ Contains the application configurations to be used. """ def __init__(self): self.MAX_CONSIDERED_SUBREDDITS = int() self.MAX_SUBMISSIONS_PER_SUBREDDIT = int() self.MAX_COMMENTS_PER_SUBMISSION = int() self.updated = False self.set_config_from_environment() def set_config_from_environment(self): """ Sets configuration from environment and sets it globally. Defaults to some values on error :return: Nothing """ self.MAX_CONSIDERED_SUBREDDITS = int( os.environ.get('MAX_CONSIDERED_SUBREDDITS', 50)) self.MAX_SUBMISSIONS_PER_SUBREDDIT = int( os.environ.get('MAX_SUBMISSIONS_PER_SUBREDDIT', 10)) self.MAX_COMMENTS_PER_SUBMISSION = int( os.environ.get('MAX_COMMENTS_PER_SUBMISSION', 5))
from utils.logging import Logger from utils.timing import Timer from math import isinf from multiprocessing import Process, Value import gelpia import gelpia_logging import time logger = Logger(level=Logger.HIGH, color=Logger.green) timer = Timer() class GelpiaInfError(Exception): def __init__(self, query): self.query = query CACHE = dict() class Result(): # Setup logging to avoid runtime error gelpia_logging.set_log_filename(None) # Silence gelpia gelpia_logging.set_log_level(-10)
parser.add_argument('--features', type=int, default=128) parser.add_argument('--dropout', type=float, default=0) # ====================================================================== args = parser.parse_args() np.random.seed(args.seed) torch.manual_seed(args.seed) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices use_gpu = torch.cuda.is_available() if args.use_cpu: use_gpu = False cudnn.benchmark = True # Redirect print to both console and log file if not args.evaluate: sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt')) else: sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt')) print("==========\nArgs:{}\n==========".format(args)) if use_gpu: print("Currently using GPU {}".format(args.gpu_devices)) cudnn.benchmark = True torch.cuda.manual_seed_all(args.seed) else: print("Currently using CPU (GPU is highly recommended)") # 建立网络 net = ResNet.resnet50(num_classes=8631, include_top=False) weights_path = 'weight_file/resnet50_ft_weight.pkl' with open(weights_path, 'rb') as f:
from utils.various import parse_float from utils.logging import Logger logger = Logger() # todo: this should probably be arbitrary precision or rationals class Interval(): def __init__(self, inf, sup): self.inf = parse_float(inf) self.sup = parse_float(sup) assert (self.inf <= self.sup) def __str__(self): return "[{},{}]".format(self.inf, self.sup) def __repr__(self): return "Interval({}, {})".format(self.inf, self.sup) def __abs__(self): # 0 # <---------------+---------------> # [********] if self.inf >= 0.0: return Interval(self.inf, self.sup) # 0 # <---------------+---------------> # [********] if self.inf <= 0.0 and 0.0 <= self.sup:
return tr_nll opt.step(closure) tr_nll = crf.neg_log_likelihood(e, b) return tr_nll if config['optim']['type'] == 'rmsprop': optim_step = rmsprop_step if config['optim']['type'] == 'lbfgs': optim_step = lbfgs_step ################################################## # Train for several epochs; if Ctrl-C interrupts, # print message and deallocate. ################################################## logger = Logger(save_dir) step = 0 try: for _ in range(num_epochs): dataset = NPZLoader(train_set, test_set, batch_size=batch_size) for events, bases in dataset.training_batches: # preprocess events and bases batches: _events = Variable(events.transpose(0, 1).contiguous()) _bases = Variable(bases.transpose(0, 1).contiguous().long()) if cuda: _events = _events.cuda() _bases = _bases.cuda() # compute loss, grads, updates: tr_nll = optim_step(_events, _bases)
def run_simulation(**kwargs): kp = KwargsParser(kwargs, DEFAULTS) folder = Path(kp.folder).expanduser() folder.mkdir(exist_ok=True, parents=True) file_str = f'L_{kp.L}_g_{kp.g}_chi_{kp.chi}_dt_{kp.dt}_quench_{kp.quench}' if kp.task_id: file_str += f'_{kp.task_id}' logger = Logger(folder.joinpath(file_str + '.log'), True) opt_logger = Logger(folder.joinpath(file_str + '.opt.log'), True) outfile = folder.joinpath(file_str + '.pkl') kp.log(logger) opt_opts = dict(display_fun=get_display_fun(opt_logger), line_search_fn='strong_wolfe', max_iter=kp.max_iter, tolerance_grad=kp.tolerance_grad) cont_opts = dict(contraction_method='brute') model = TFIM(kp.g, bc='obc', lx=kp.L, ly=kp.L, dtype_hamiltonian=np.float64) evolver = TimeEvolution(kp.g, kp.dt, 'obc', real_time=True, lx=kp.L, ly=kp.L, pepo_dtype=np.complex128) logger.log(f'Starting with groundstate of g={kp.g} TFIM') # Prepare groundstate gs = None gs_energy = None if kp.gs_file: logger.log('GS file specified, loading GS from file') try: with open(kp.gs_file, 'rb') as f: res = pickle.load(f) gs_tensors = res['gs_tensors'] gs = Peps(gs_tensors, 'obc') gs_energy = res['gs_energy'] assert np.allclose(kp.g, res['kwargs']['g']) assert gs.lx == kp.L assert gs.ly == kp.L except Exception as e: logger.log('Failed to load GS from file. Error: ' + str(e)) if (gs is None) or (gs_energy is None): logger.log('No GS file specified, optimising gs...') gs, gs_energy = model.groundstate(kp.chi, (kp.L, kp.L), 'ps', 0.05, cont_opts, opt_opts) logger.log('Saving GS to ' + str(folder.joinpath(file_str + '.gs.pkl'))) results = dict(kwargs=kp.kwargs(), gs=gs, gs_energy=gs_energy, g=kp.g) with open(folder.joinpath(file_str + '.gs.pkl'), 'wb') as f: pickle.dump(results, f) # Prepare quench if kp.quench == 'X': # <Sx(r,t) Sx(center,0)> quench_operator = sx measure_operator = sx elif kp.quench == 'Y': # <Sy(r,t) Sy(center,0)> quench_operator = sy measure_operator = sy elif kp.quench == 'Z': # <Sz(r,t) Sz(center,0)> quench_operator = sz measure_operator = sz elif kp.quench == '+': # <S+(r,t) S-(center,0)> quench_operator = sm measure_operator = sp else: raise ValueError(f'Illegal quench code {kp.quench}') logger.log(f'Quench: Applying quench operator to center site') quenched = SingleSiteOperator(quench_operator, kp.L // 2, kp.L // 2).apply_to_peps(gs) # Time evolution x_snapshot_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L]) y_snapshot_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L]) z_snapshot_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L]) correlator_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L], dtype=onp.complex) t_data = onp.zeros([kp.n_steps + 1]) state = quenched opt_opts['dtype'] = np.complex128 opt_opts['max_grad_evals_ls'] = 100 for n in range(kp.n_steps): logger.log('Computing Observables') t = n * kp.dt x_snapshot_data[n, :, :] = x_snapshot(state, cont_opts) y_snapshot_data[n, :, :] = y_snapshot(state, cont_opts) z_snapshot_data[n, :, :] = z_snapshot(state, cont_opts) correlator_data[n, :, :] = correlator_timeslice( gs, state, measure_operator, gs_energy, t, **cont_opts) t_data[n] = t logger.log(f'Evolving to t={(n + 1) * kp.dt}') state = evolver.evolve(state, contraction_options=cont_opts, optimisation_options=opt_opts, random_dev=None, initial=kp.initial) # save results (will be overwritten), (in case process dies before it finishes) results = dict(kwargs=kp.kwargs(), quench=kp.quench, x_snapshot=x_snapshot_data, y_snapshot=y_snapshot_data, z_snapshot=z_snapshot_data, correlator=correlator_data, t=t_data, state_tensors=state.get_tensors()) with open(outfile, 'wb') as f: pickle.dump(results, f) if kp.save_all_peps: results = dict(kwargs=kp.kwargs(), t=t, state_tensors=state.get_tensors()) with open(folder.joinpath(file_str + f'state_t_{t}.pkl'), 'wb') as f: pickle.dump(results, f) logger.log('Computing Observables') t = kp.n_steps * kp.dt x_snapshot_data[kp.n_steps, :, :] = x_snapshot(state, cont_opts) y_snapshot_data[kp.n_steps, :, :] = y_snapshot(state, cont_opts) z_snapshot_data[kp.n_steps, :, :] = z_snapshot(state, cont_opts) correlator_data[kp.n_steps, :, :] = correlator_timeslice( gs, state, measure_operator, gs_energy, t, **cont_opts) t_data[kp.n_steps] = t # save results logger.log(f'saving results to {outfile}') results = dict(kwargs=kp.kwargs(), quench=kp.quench, x_snapshot=x_snapshot_data, y_snapshot=y_snapshot_data, z_snapshot=z_snapshot_data, correlator=correlator_data, t=t_data, state_tensors=state.get_tensors()) with open(outfile, 'wb') as f: pickle.dump(results, f) if kp.save_all_peps: results = dict(kwargs=kp.kwargs(), t=t, state_tensors=state.get_tensors()) with open(folder.joinpath(file_str + f'state_t_{t}.pkl'), 'wb') as f: pickle.dump(results, f)
from domain.ebay import EbayPhone from domain.phone import PhoneDetails import pandas as pd import numpy as np from utils.text_utils import match_word, tokenize, update_vocabulary from utils.logging import Logger from nlp.embeddings import WordEmbeddings from nlp.training import DataSet from config import Config logger = Logger.of('DataPreparation') def create_embeddings(): titles = pd.read_csv(Config.get_filepath('train-data'))['title'].tolist() titles = ['EMP ' + title + ' EMP' for title in titles] embeddings = WordEmbeddings.from_sentences(titles) embeddings.save(Config.get_filepath('word2vec')) def clean_phones(): logger.info('cleaning phones') EbayPhone.objects().update(details=PhoneDetails()) for phone in EbayPhone.objects(title__contains='*'): phone.update(title=phone.title.replace('*', ' ')) for phone in EbayPhone.objects(title__contains='/'): phone.update(title=phone.title.replace('/', ' ')) for phone in EbayPhone.objects(title__contains='+'): phone.update(title=phone.title.replace('+', ' plus '))
if torch.cuda.is_available(): if args.cuda: torch.set_default_tensor_type('torch.cuda.FloatTensor') if not args.cuda: print("WARNING: It looks like you have a CUDA device, but aren't " + "using CUDA.\nRun with --cuda for optimal training speed.") torch.set_default_tensor_type('torch.FloatTensor') else: torch.set_default_tensor_type('torch.FloatTensor') if not os.path.exists(args.save_folder): os.mkdir(args.save_folder) sys.stdout = Logger(os.path.join(args.save_folder, 'log.txt')) def train(): if args.dataset == 'COCO': '''if args.dataset_root == VOC_ROOT: if not os.path.exists(COCO_ROOT): parser.error('Must specify dataset_root if specifying dataset') print("WARNING: Using default COCO dataset_root because " + "--dataset_root was not specified.") args.dataset_root = COCO_ROOT cfg = coco dataset = COCODetection(root=args.dataset_root, transform=SSDAugmentation(cfg['min_dim'], MEANS))''' elif args.dataset == 'VOC': '''if args.dataset_root == COCO_ROOT:
parser.add_argument( "--max_pool", type=int, default=None, help="Use max pooling after the initial convolution layer.", ) ARGS = parser.parse_args() ARGS.cuda = torch.cuda.is_available() and ARGS.cuda ARGS.device = torch.device("cuda") if ARGS.cuda else torch.device("cpu") if ARGS.conv_kernel_size is not None and len(ARGS.conv_kernel_size): ARGS.conv_kernel_size = ARGS.conv_kernel_size[0] log = Logger("TRAIN", ARGS.debug, ARGS.log_dir) """ Get audio all audio files from the given data directory except they are broken. """ def get_audio_files(): audio_files = None if input_data.can_load_from_csv(): log.info("Found csv files in {}".format(ARGS.data_dir)) else: log.debug("Searching for audio files in {}".format(ARGS.data_dir)) if ARGS.filter_broken_audio: data_dir_ = pathlib.Path(ARGS.data_dir) audio_files = get_audio_files_from_dir(ARGS.data_dir) log.debug("Moving possibly broken audio files to .bkp:")
def main(): global best_prec global opt if opt['id'] != '': model_id = opt['id'] else: model_id = time.strftime("%m_%d_%H-%M-%S") sys.stdout = Logger(osp.join(opt['log_dir'], 'log.' + model_id + '.txt')) # initialize checkpoint_dir = osp.join(opt['checkpoint_dir'], model_id) mkdir_if_missing(checkpoint_dir) # check gpu assert opt['gpus'] is not None # set random seed cudnn.benchmark = False cudnn.deterministic = True random.seed(opt['seed']) np.random.seed(opt['seed']) torch.manual_seed(opt['seed']) torch.cuda.manual_seed_all(opt['seed']) # load imdb train_refdb = get_db('refvg_train_' + opt['model_method']) vocab = train_refdb.load_dictionary() opt['vocab_size'] = len(vocab) val_refdb = get_db('refvg_val_' + opt['model_method']) # model, criterion, optimizer model = SGReason(opt) model = torch.nn.DataParallel(model).cuda() criterion = SoftmaxLoss().cuda() optimizer = torch.optim.Adam(list(model.parameters()) + list(criterion.parameters()), lr=opt['learning_rate'], betas=(opt['optim_alpha'], opt['optim_beta']), eps=opt['optim_epsilon']) scheduler = ReduceLROnPlateau(optimizer, factor=0.1, patience=3, mode='max') if opt['evaluate']: if osp.isfile(opt['model']): model, criterion = load_checkpoint(model, criterion, opt['model']) test_refdb = get_db('refvg_test_' + opt['model_method']) test_dataset = RefDataset(test_refdb, vocab, opt) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=opt['batch_size'], shuffle=False, num_workers=opt['workers'], pin_memory=True) test_loss, test_prec = validate(test_loader, model, criterion) print(test_loss, test_prec) else: print("=> no checkpoint found at '{}'".format(opt['model'])) return # start training epoch_cur = 0 train_dataset = RefDataset(train_refdb, vocab, opt) val_dataset = RefDataset(val_refdb, vocab, opt) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=opt['batch_size'], shuffle=True, num_workers=opt['workers'], pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=opt['batch_size'], shuffle=False, num_workers=opt['workers'], pin_memory=True) for epoch in range(epoch_cur, opt['max_epochs']): train(train_loader, model, criterion, optimizer, epoch) val_loss, prec = validate(val_loader, model, criterion, epoch) scheduler.step(prec) for i, param_group in enumerate(optimizer.param_groups): print(float(param_group['lr'])) is_best = prec >= best_prec best_prec = max(best_prec, prec) save_checkpoint( { 'model_state_dict': model.state_dict(), 'crit_state_dict': criterion.state_dict(), 'optimizer': optimizer.state_dict() }, is_best, checkpoint_dir, str(epoch))
def main(): global args, best_prec1, t ,loss_weight, Identity_num t = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') loss_weight = 0.001 Identity_num = 'Null' args = parser.parse_args() sys.stdout = Logger(os.path.join('.', 'logs', args.arch + t + '.txt')) # create model print("=> creating model '{}'".format(args.arch)) print("=> --learning-rate '{}'".format(args.lr)) print("=> --momentum '{}'".format(args.momentum)) print("=> --weight_decay '{}'".format(args.weight_decay)) print("=> --batch-size '{}'".format(args.batch_size)) print("=> --pretrained '{}'".format(args.pretrained)) print("=> --loss_weight '{}'".format(loss_weight)) print("=> --Identity_num '{}'".format(Identity_num)) print("=> --model_path '{}'".format(args.model_path)) print("=> --result_path '{}'".format(args.result_path)) model = XTmodelzoo.__dict__[args.arch](num_classes=20, pretrained=args.pretrained) # optionally resume from a checkpoint if os.path.isfile(args.model_path): print("=> loading checkpoint '{}'".format(args.model_path)) checkpoint = torch.load(args.model_path) best_prec1 = checkpoint['best_prec1'] new_dict = {k[7:] : v for k,v in checkpoint['state_dict'].items()} model.load_state_dict(new_dict) print("=> loaded checkpoint '{}' (epoch {})" .format(args.model_path, checkpoint['epoch'])) else: print("=> no model found at '{}'".format(args.model_path)) cudnn.benchmark = True # Data loading code testdir = os.path.join(args.data) scale = 0.875 print('Images transformed from size {} to {}'.format( int(round(max(model.input_size) / scale)), model.input_size)) test_tf = utils.TransformImage(model, scale=scale) test_loader = torch.utils.data.DataLoader( MyTestDataset(data_root=testdir,csv_path=os.path.join('..','DataProcess','testB','list.csv'), transform=test_tf), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) nllloss = nn.NLLLoss().cuda() # Center loss loss_weight = 0.001 center_loss = CenterLoss(20, 20).cuda() criterion = [nllloss, center_loss] optimizer4nn = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) optimzer4center = torch.optim.SGD(center_loss.parameters(), lr=0.5) optimizer = [optimizer4nn, optimzer4center] model = torch.nn.DataParallel(model).cuda() test(test_loader, model, criterion, args.result_path)
def main(args): ## fix random_seed fixRandomSeed(1) ## cuda setting cudnn.benchmark = True cudnn.enabled = True device = torch.device('cuda:' + str(args.gpuid)) torch.cuda.set_device(device) ## Logger setting if not args.evaluate: sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt')) print('logs_dir=', args.logs_dir) print('args : ', args) ## get dataset & dataloader: dataset, source_num_classes, source_train_loader, \ target_train_loader, query_loader, gallery_loader = get_data(args.data_dir, args.source,args.target, args.source_train_path, args.target_train_path, args.source_extension,args.target_extension, args.height, args.width, args.batch_size, args.re, args.workers) h, w = map(int, [args.height, args.width]) input_size_source = (h, w) input_size_target = (h, w) # cudnn.enabled = True # Create Network # model = Res_Deeplab(num_classes=args.num_classes) model = Res_Deeplab(num_classes=source_num_classes) if args.restore_from[:4] == 'http': saved_state_dict = model_zoo.load_url(args.restore_from) else: saved_state_dict = torch.load(args.restore_from) new_params = model.state_dict().copy() ## adapte new_params's layers / classes to saved_state_dict for i in saved_state_dict: i_parts = i.split('.') if not args.num_classes == 19 or not i_parts[1] == 'layer5': new_params['.'.join(i_parts[1:])] = saved_state_dict[i] if args.restore_from[:4] == './mo': model.load_state_dict(new_params) else: model.load_state_dict(saved_state_dict) ## set mode = train and moves the params of model to GPU model.train() model.cuda(args.gpu) # cudnn.benchmark = True # Init D model_D = FCDiscriminator(num_classes=args.num_classes) # ============================================================================= # #for retrain # saved_state_dict_D = torch.load(RESTORE_FROM_D) # model_D.load_state_dict(saved_state_dict_D) # ============================================================================= model_D.train() model_D.cuda(args.gpu) # if not os.path.exists(args.snapshot_dir): # os.makedirs(args.snapshot_dir) if args.source == 'GTA5': trainloader = data.DataLoader(GTA5DataSet( args.data_dir, args.data_list, max_iters=args.num_steps * args.iter_size * args.batch_size, crop_size=input_size_source, scale=True, mirror=True, mean=IMG_MEAN), batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) else: trainloader = data.DataLoader(SYNTHIADataSet( args.data_dir, args.data_list, max_iters=args.num_steps * args.iter_size * args.batch_size, crop_size=input_size_source, scale=True, mirror=True, mean=IMG_MEAN), batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) trainloader_iter = enumerate(trainloader) targetloader = data.DataLoader(cityscapesDataSet( args.data_dir_target, args.data_list_target, max_iters=args.num_steps * args.iter_size * args.batch_size, crop_size=input_size_target, scale=True, mirror=True, mean=IMG_MEAN, set=args.set), batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) targetloader_iter = enumerate(targetloader) optimizer = optim.SGD(model.optim_parameters(args), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) optimizer.zero_grad() optimizer_D = optim.Adam(model_D.parameters(), lr=args.learning_rate_D, betas=(0.9, 0.99)) optimizer_D.zero_grad() bce_loss = torch.nn.BCEWithLogitsLoss() weighted_bce_loss = WeightedBCEWithLogitsLoss() interp_source = nn.Upsample(size=(input_size_source[1], input_size_source[0]), mode='bilinear', align_corners=True) interp_target = nn.Upsample(size=(input_size_target[1], input_size_target[0]), mode='bilinear', align_corners=True) # Labels for Adversarial Training source_label = 0 target_label = 1 for i_iter in range(args.num_steps): optimizer.zero_grad() adjust_learning_rate(optimizer, i_iter) optimizer_D.zero_grad() adjust_learning_rate_D(optimizer_D, i_iter) damping = (1 - i_iter / NUM_STEPS) # ====================================================================================== # train G # ====================================================================================== # Remove Grads in D for param in model_D.parameters(): param.requires_grad = False # Train with Source _, batch = next(trainloader_iter) images_s, labels_s, _, _, _ = batch images_s = Variable(images_s).cuda(args.gpu) pred_source1, pred_source2 = model(images_s) pred_source1 = interp_source(pred_source1) pred_source2 = interp_source(pred_source2) # Segmentation Loss loss_seg = (loss_calc(pred_source1, labels_s, args.gpu) + loss_calc(pred_source2, labels_s, args.gpu)) loss_seg.backward() # Train with Target _, batch = next(targetloader_iter) images_t, _, _, _ = batch images_t = Variable(images_t).cuda(args.gpu) pred_target1, pred_target2 = model(images_t) pred_target1 = interp_target(pred_target1) pred_target2 = interp_target(pred_target2) weight_map = weightmap(F.softmax(pred_target1, dim=1), F.softmax(pred_target2, dim=1)) D_out = interp_target( model_D(F.softmax(pred_target1 + pred_target2, dim=1))) # Adaptive Adversarial Loss if (i_iter > PREHEAT_STEPS): loss_adv = weighted_bce_loss( D_out, Variable( torch.FloatTensor( D_out.data.size()).fill_(source_label)).cuda(args.gpu), weight_map, Epsilon, Lambda_local) else: loss_adv = bce_loss( D_out, Variable( torch.FloatTensor( D_out.data.size()).fill_(source_label)).cuda(args.gpu)) loss_adv = loss_adv * Lambda_adv * damping loss_adv.backward() # Weight Discrepancy Loss W5 = None W6 = None if args.model == 'ResNet': for (w5, w6) in zip(model.layer5.parameters(), model.layer6.parameters()): if W5 is None and W6 is None: W5 = w5.view(-1) W6 = w6.view(-1) else: W5 = torch.cat((W5, w5.view(-1)), 0) W6 = torch.cat((W6, w6.view(-1)), 0) loss_weight = (torch.matmul(W5, W6) / (torch.norm(W5) * torch.norm(W6)) + 1 ) # +1 is for a positive loss loss_weight = loss_weight * Lambda_weight * damping * 2 loss_weight.backward() # ====================================================================================== # train D # ====================================================================================== # Bring back Grads in D for param in model_D.parameters(): param.requires_grad = True # Train with Source pred_source1 = pred_source1.detach() pred_source2 = pred_source2.detach() D_out_s = interp_source( model_D(F.softmax(pred_source1 + pred_source2, dim=1))) loss_D_s = bce_loss( D_out_s, Variable( torch.FloatTensor( D_out_s.data.size()).fill_(source_label)).cuda(args.gpu)) loss_D_s.backward() # Train with Target pred_target1 = pred_target1.detach() pred_target2 = pred_target2.detach() weight_map = weight_map.detach() D_out_t = interp_target( model_D(F.softmax(pred_target1 + pred_target2, dim=1))) # Adaptive Adversarial Loss if (i_iter > PREHEAT_STEPS): loss_D_t = weighted_bce_loss( D_out_t, Variable( torch.FloatTensor( D_out_t.data.size()).fill_(target_label)).cuda( args.gpu), weight_map, Epsilon, Lambda_local) else: loss_D_t = bce_loss( D_out_t, Variable( torch.FloatTensor( D_out_t.data.size()).fill_(target_label)).cuda( args.gpu)) loss_D_t.backward() optimizer.step() optimizer_D.step() print('exp = {}'.format(args.snapshot_dir)) print( 'iter = {0:6d}/{1:6d}, loss_seg = {2:.4f} loss_adv = {3:.4f}, loss_weight = {4:.4f}, loss_D_s = {5:.4f} loss_D_t = {6:.4f}' .format(i_iter, args.num_steps, loss_seg, loss_adv, loss_weight, loss_D_s, loss_D_t)) f_loss = open(osp.join(args.snapshot_dir, 'loss.txt'), 'a') f_loss.write('{0:.4f} {1:.4f} {2:.4f} {3:.4f} {4:.4f}\n'.format( loss_seg, loss_adv, loss_weight, loss_D_s, loss_D_t)) f_loss.close() if i_iter >= args.num_steps_stop - 1: print('save model ...') torch.save( model.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(args.num_steps) + '.pth')) torch.save( model_D.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(args.num_steps) + '_D.pth')) break if i_iter % args.save_pred_every == 0 and i_iter != 0: print('taking snapshot ...') torch.save( model.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(i_iter) + '.pth')) torch.save( model_D.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(i_iter) + '_D.pth')) ## create dataloader dataset, source_num_classes, source_train_loader, target_train_loader, query_loader, gallery_loader = get_data( args.data_dir, args.source, args.target, args.source_train_path, args.target_train_path, args.source_extension, args.target_extension, args.height, args.width, args.batch_size, args.re, args.workers) h, w = map(int, args.input_size_source.split(',')) input_size_source = (h, w) input_size_target = (h, w)
for epoch in range(start_epoch, 210): pj_model.two_classifier(epoch, train_loader_s, train_loader_t, writer, logger, rand_src_1, rand_src_2) if ((epoch+1) % 2 == 0): evaluator.run(val_loader) cmc, mAP = evaluator.state.metrics['r1_mAP'] logger.info("Validation Results - Epoch: {}".format(epoch)) logger.info("mAP: {:.1%}".format(mAP)) for r in [1, 5, 10, 20]: logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(r, cmc[r - 1])) is_best = cmc[0] > best_top1 best_top1 = max(cmc[0], best_top1) save_checkpoint({ 'Content_encoder': pj_model.Content_Encoder.module.state_dict(), 'Content_optimizer': pj_model.Content_optimizer.state_dict(), 'Content_optimizer_fix': pj_model.Content_optimizer_fix.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=cfg.OUTPUT_DIR + 'checkpoint.pth.tar', info=ckpt+'.pth.tar') logger.info('\n * Finished epoch {:3d} top1: {:5.1%} best: {:5.1%}{}\n'. format(epoch, cmc[0], best_top1, ' *' if is_best else '')) writer.close() if __name__ == '__main__': info = 'market_duke_stage1' sys.stdout = Logger(osp.join(cfg.OUTPUT_DIR, info+'.txt')) print(info) print('--------------------------------------------------') logger = setup_logger("reid_baseline", cfg.OUTPUT_DIR, 0, info+'.txt') main('two', info, logger)
k: v for k, v in checkpoint_state['model'].items() if k in model_dict } model_dict.update(pretrained_dict) model.load_state_dict(model_dict) # other states from checkpoint -- optimizer, scheduler, loss, epoch initial_epoch = checkpoint_state['epoch'] + 1 optimizer.load_state_dict(checkpoint_state['optimizer']) scheduler.load_state_dict(checkpoint_state['scheduler']) criterion.load_state_dict(checkpoint_state['criterion']) # initialize logger log_dir = os.path.join( args.base_directory, "logs", f'{hp.version}-{datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")}') Logger.initialize(log_dir, args.flush_seconds) # training loop best_eval = float('inf') for epoch in range(initial_epoch, hp.epochs): train(args.logging_start, epoch, train_data, model, criterion, optimizer) if hp.learning_rate_decay_start - hp.learning_rate_decay_each < epoch * len( train_data): scheduler.step() eval_loss = evaluate(epoch, eval_data, model, criterion) if (epoch + 1) % hp.checkpoint_each_epochs == 0: # save checkpoint together with hyper-parameters, optimizer and scheduler states checkpoint_file = f'{checkpoint_dir}/{hp.version}_loss-{epoch}-{eval_loss:2.3f}' state_dict = { 'epoch': epoch,
def main(args): np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) cudnn.benchmark = True # log file sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt')) dataset, num_classes, train_loader, query_loader, gallery_loader = \ get_data(args.dataset, args.split, args.data_dir, args.batch_size, args.seq_len, args.seq_srd, args.workers, args.train_mode) # create CNN model cnn_model = models.create(args.a1, num_features=args.features, dropout=args.dropout) # create ATT model input_num = cnn_model.feat.in_features output_num = args.features att_model = models.create(args.a2, input_num, output_num) # create classifier model class_num = 2 classifier_model = models.create(args.a3, output_num, class_num) # CUDA acceleration model cnn_model = torch.nn.DataParallel(cnn_model).cuda() att_model = att_model.cuda() classifier_model = classifier_model.cuda() # Loss function criterion_oim = OIMLoss(args.features, num_classes, scalar=args.oim_scalar, momentum=args.oim_momentum) criterion_veri = PairLoss(args.sampling_rate) criterion_oim.cuda() criterion_veri.cuda() # Optimizer base_param_ids = set(map(id, cnn_model.module.base.parameters())) new_params = [p for p in cnn_model.parameters() if id(p) not in base_param_ids] param_groups1 = [ {'params': cnn_model.module.base.parameters(), 'lr_mult': 1}, {'params': new_params, 'lr_mult': 1}] param_groups2 = [ {'params': att_model.parameters(), 'lr_mult': 1}, {'params': classifier_model.parameters(), 'lr_mult': 1}] optimizer1 = torch.optim.SGD(param_groups1, lr=args.lr1, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) optimizer2 = torch.optim.SGD(param_groups2, lr=args.lr2, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) # Schedule Learning rate def adjust_lr1(epoch): lr = args.lr1 * (0.1 ** (epoch/args.lr1step)) print(lr) for g in optimizer1.param_groups: g['lr'] = lr * g.get('lr_mult', 1) def adjust_lr2(epoch): lr = args.lr2 * (0.01 ** (epoch//args.lr2step)) print(lr) for g in optimizer2.param_groups: g['lr'] = lr * g.get('lr_mult', 1) def adjust_lr3(epoch): lr = args.lr3 * (0.000001 ** (epoch //args.lr3step)) print(lr) return lr best_top1 = 0 start_epoch = args.start_epoch if args.evaluate == 1: print('Evaluate:') evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri) top1, mAP = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo) elif args.test == 1: print('Test:') checkpoint1 = load_checkpoint(osp.join(args.logs_dir, 'cnnmodel_best.pth.tar')) cnn_model.load_state_dict(checkpoint1['state_dict']) checkpoint2 = load_checkpoint(osp.join(args.logs_dir, 'attmodel_best.pth.tar')) att_model.load_state_dict(checkpoint2['state_dict']) checkpoint3 = load_checkpoint(osp.join(args.logs_dir, 'clsmodel_best.pth.tar')) classifier_model.load_state_dict(checkpoint3['state_dict']) evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri) mAP, top1, top5, top10, top20 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo) else: tensorboard_test_logdir = osp.join(args.logs_dir, 'test_log') writer = SummaryWriter(log_dir=tensorboard_test_logdir) if args.resume == 1: checkpoint1 = load_checkpoint(osp.join(args.logs_dir, 'cnn_checkpoint.pth.tar')) cnn_model.load_state_dict(checkpoint1['state_dict']) checkpoint2 = load_checkpoint(osp.join(args.logs_dir, 'att_checkpoint.pth.tar')) att_model.load_state_dict(checkpoint2['state_dict']) checkpoint3 = load_checkpoint(osp.join(args.logs_dir, 'cls_checkpoint.pth.tar')) classifier_model.load_state_dict(checkpoint3['state_dict']) start_epoch = checkpoint1['epoch'] best_top1 = checkpoint1['best_top1'] print("=> Start epoch {} best top1 {:.1%}" .format(start_epoch, best_top1)) # Trainer tensorboard_train_logdir = osp.join(args.logs_dir, 'train_log') trainer = SEQTrainer(cnn_model, att_model, classifier_model, criterion_veri, criterion_oim, args.train_mode, args.lr3, tensorboard_train_logdir) # Evaluator if args.train_mode == 'cnn': evaluator = CNNEvaluator(cnn_model, args.train_mode) elif args.train_mode == 'cnn_rnn': evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri) else: raise RuntimeError('Yes, Evaluator is necessary') for epoch in range(start_epoch, args.epochs): adjust_lr1(epoch) adjust_lr2(epoch) rate = adjust_lr3(epoch) trainer.train(epoch, train_loader, optimizer1, optimizer2, rate) if epoch % 1 == 0: mAP, top1, top5, top10, top20 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo) writer.add_scalar('test/mAP', mAP, epoch+1) writer.add_scalar('test/top1', top1, epoch+1) writer.add_scalar('test/top5', top5, epoch+1) writer.add_scalar('test/top10', top10, epoch+1) writer.add_scalar('test/top20', top20, epoch+1) is_best = top1 > best_top1 if is_best: best_top1 = top1 save_cnn_checkpoint({ 'state_dict': cnn_model.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=osp.join(args.logs_dir, 'cnn_checkpoint.pth.tar')) if args.train_mode == 'cnn_rnn': save_att_checkpoint({ 'state_dict': att_model.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=osp.join(args.logs_dir, 'att_checkpoint.pth.tar')) save_cls_checkpoint({ 'state_dict': classifier_model.state_dict(), 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=osp.join(args.logs_dir, 'cls_checkpoint.pth.tar')) print('Test: ') checkpoint1 = load_checkpoint(osp.join(args.logs_dir, 'cnnmodel_best.pth.tar')) cnn_model.load_state_dict(checkpoint1['state_dict']) checkpoint2 = load_checkpoint(osp.join(args.logs_dir, 'attmodel_best.pth.tar')) att_model.load_state_dict(checkpoint2['state_dict']) checkpoint3 = load_checkpoint(osp.join(args.logs_dir, 'clsmodel_best.pth.tar')) classifier_model.load_state_dict(checkpoint3['state_dict']) evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri) mAP, top1, top5, top10, top20 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo)
def train(logging_start_epoch, epoch, data, model, criterion, optimizer): """Main training procedure. Arguments: logging_start_epoch -- number of the first epoch to be logged epoch -- current epoch data -- DataLoader which can provide batches for an epoch model -- model to be trained criterion -- instance of loss function to be optimized optimizer -- instance of optimizer which will be used for parameter updates """ model.train() # initialize counters, etc. learning_rate = optimizer.param_groups[0]['lr'] cla = 0 done, start_time = 0, time.time() # loop through epoch batches for i, batch in enumerate(data): global_step = done + epoch * len(data) optimizer.zero_grad() # parse batch batch = list(map(to_gpu, batch)) src, src_len, trg_mel, trg_lin, trg_len, stop_trg, spkrs, langs = batch # get teacher forcing ratio if hp.constant_teacher_forcing: tf = hp.teacher_forcing else: tf = cos_decay( max(global_step - hp.teacher_forcing_start_steps, 0), hp.teacher_forcing_steps) # run the model post_pred, pre_pred, stop_pred, alignment, spkrs_pred, enc_output = model( src, src_len, trg_mel, trg_len, spkrs, langs, tf) # evaluate loss function post_trg = trg_lin if hp.predict_linear else trg_mel classifier = model._reversal_classifier if hp.reversal_classifier else None loss, batch_losses = criterion(src_len, trg_len, pre_pred, trg_mel, post_pred, post_trg, stop_pred, stop_trg, alignment, spkrs, spkrs_pred, enc_output, classifier) # evaluate adversarial classifier accuracy, if present if hp.reversal_classifier: input_mask = lengths_to_mask(src_len) trg_spkrs = torch.zeros_like(input_mask, dtype=torch.int64) for s in range(hp.speaker_number): speaker_mask = (spkrs == s) trg_spkrs[speaker_mask] = s matches = (trg_spkrs == torch.argmax(torch.nn.functional.softmax( spkrs_pred, dim=-1), dim=-1)) matches[~input_mask] = False cla = torch.sum(matches).item() / torch.sum(input_mask).item() # comptute gradients and make a step loss.backward() gradient = torch.nn.utils.clip_grad_norm_(model.parameters(), hp.gradient_clipping) optimizer.step() # log training progress if epoch >= logging_start_epoch: Logger.training(global_step, batch_losses, gradient, learning_rate, time.time() - start_time, cla) # update criterion states (params and decay of the loss and so on ...) criterion.update_states() start_time = time.time() done += 1
class CsvSplit(object): def __init__( self, split_fracs: Dict[str, float], working_dir: (str) = None, seed: (int) = None, split_per_dir=False, ): if not np.isclose(np.sum([p for _, p in split_fracs.items()]), 1.): raise ValueError("Split probabilities have to sum up to 1.") self.split_fracs = split_fracs self.working_dir = working_dir self.seed = seed self.split_per_dir = split_per_dir self.splits = defaultdict(list) self._logger = Logger("CSVSPLIT") """ Return split for given partition. If there is already an existing CSV split return this split if it is valid or in case there exist not a split yet generate a new CSV split """ def load(self, split: str, files: List[Any] = None): if split not in self.split_fracs: raise ValueError( "Provided split '{}' is not in `self.split_fracs`.".format( split)) if self.splits[split]: return self.splits[split] if self.working_dir is None: self.splits = self._split_with_seed(files) return self.splits[split] if self.can_load_from_csv(): if not self.split_per_dir: csv_split_files = { split_: (os.path.join(self.working_dir, split_ + ".csv"), ) for split_ in self.split_fracs.keys() } else: csv_split_files = {} for split_ in self.split_fracs.keys(): split_file = os.path.join(self.working_dir, split_) csv_split_files[split_] = [] with open(split_file, "r") as f: for line in f.readlines(): csv_split_files[split_].append(line.strip()) for split_ in self.split_fracs.keys(): for csv_file in csv_split_files[split_]: if not csv_file or csv_file.startswith(r"#"): continue csv_file_path = os.path.join(self.working_dir, csv_file) with open(csv_file_path, "r") as f: reader = csv.reader(f) for item in reader: file_ = os.path.basename(item[0]) file_ = os.path.join(os.path.dirname(csv_file), file_) self.splits[split_].append(file_) return self.splits[split] if not self.split_per_dir: working_dirs = (self.working_dir, ) else: f_d_map = self._get_f_d_map(files) working_dirs = [ os.path.join(self.working_dir, p) for p in f_d_map.keys() ] for working_dir in working_dirs: splits = self._split_with_seed( files if not self.split_per_dir else f_d_map[working_dir]) for split_ in splits.keys(): csv_file = os.path.join(working_dir, split_ + ".csv") self._logger.debug("Generating {}".format(csv_file)) if self.split_per_dir: with open(os.path.join(self.working_dir, split_), "a") as f: p = pathlib.Path(csv_file).relative_to( self.working_dir) f.write(str(p) + "\n") if len(splits[split_]) == 0: raise ValueError( "Error splitting dataset. Split '{}' has 0 entries". format(split_)) with open(csv_file, "w", newline="") as fh: writer = csv.writer(fh) for item in splits[split_]: writer.writerow([item]) self.splits[split_].extend(splits[split_]) return self.splits[split] """ Check whether it is possible to correctly load information from existing csv files """ def can_load_from_csv(self): if not self.working_dir: return False if self.split_per_dir: for split in self.split_fracs.keys(): split_file = os.path.join(self.working_dir, split) if not os.path.isfile(split_file): return False self._logger.debug( "Found dataset split file {}".format(split_file)) with open(split_file, "r") as f: for line in f.readlines(): csv_file = line.strip() if not csv_file or csv_file.startswith(r"#"): continue if not os.path.isfile( os.path.join(self.working_dir, csv_file)): self._logger.error( "File not found: {}".format(csv_file)) raise ValueError( "Split file found, but csv files are missing. " "Aborting...") else: for split in self.split_fracs.keys(): csv_file = os.path.join(self.working_dir, split + ".csv") if not os.path.isfile(csv_file): return False self._logger.debug("Found csv file {}".format(csv_file)) return True """ Create a mapping from directory to containing files. """ def _get_f_d_map(self, files: List[Any]): f_d_map = defaultdict(list) if self.working_dir is not None: for f in files: f_d_map[str(pathlib.Path( self.working_dir).joinpath(f).parent)].append(f) else: for f in files: f_d_map[str( pathlib.Path(".").resolve().joinpath(f).parent)].append(f) return f_d_map """ Randomly splits the dataset using given seed """ def _split_with_seed(self, files: List[Any]): if not files: raise ValueError("Provided list `files` is `None`.") if self.seed: random.seed(self.seed) return self.split_fn(files) """ A generator function that returns all values for the given `split`. """ def split_fn(self, files: List[Any]): _splits = np.split( ary=random.sample(files, len(files)), indices_or_sections=[ int(p * len(files)) for _, p in self.split_fracs.items() ], ) splits = dict() for i, key in enumerate(self.splits.keys()): splits[key] = _splits[i] return splits
dest="jit_load", action="store_true", help="Load model via torch jit (otherwise via torch load).", ) parser.add_argument( "--input_file", type=str, default=None, help= "Input file could either be a directory with multiple audio files or just one single audio file" ) ARGS = parser.parse_args() log = Logger("PREDICT", ARGS.debug, ARGS.log_dir) """ Main function to compute prediction by using a trained model together with the given input """ if __name__ == "__main__": if ARGS.checkpoint_path is not None: log.info("Restoring checkpoint from {} instead of using a model file.". format(ARGS.checkpoint_path)) checkpoint = torch.load(ARGS.checkpoint_path) model = UNet(1, 1, bilinear=False) model.load_state_dict(checkpoint["modelState"]) log.warning( "Using default preprocessing options. Provide Model file if they are changed" ) dataOpts = DefaultSpecDatasetOps
def main(): args = cli.parse_args() if not args.no_det: np.random.seed(args.env_seed) random.seed(args.env_seed) torch.manual_seed(args.env_seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False ### Logger setup ### logger = Logger(args.fileloc, args.load_loc) logger.init_config(args) logger.load_results() ### If we're making a visualisation, branch here ### if args.task == 'vis': if args.ranking not in logger.data['scores'][0]: print("\nNeed to compute ranking {} before doing visualisation!". format(args.ranking)) exit() run_and_save(logger) exit() ### Get or update all results ### counts_do = { 'redo': args.redo_all or not logger.is_done('counts'), 'update': args.more_count is not None } scores_do = { 'redo': counts_do['redo'] or counts_do['update'] or not logger.is_done('scores'), 'update': args.more_scoretypes is not None } interpol_do = { 'redo': scores_do['redo'] or not logger.is_done('interpol') or args.redo_interpol is not None, 'update': scores_do['update'] } ### Counts if counts_do['redo']: print("\n----- Counting -----\n") counts = count(logger) logger.update_counts(counts) if counts_do['update']: N = args.more_count print("\n----- Additional Counts ({} more runs) -----\n".format(N)) # If we're adding more counts without having run before, then we need to reset the # env or we would be revisiting the same states because of the seed. if not counts_do['redo']: for _ in range(logger.config['n_runs']): logger.config['env'].reset() counts = count(logger, n_runs=N) logger.update_counts(counts, addn=N) if counts_do['redo'] or counts_do['update']: logger.dump_results() logger.dump_config() ### Scores if scores_do['redo']: print("\n----- Scoring -----\n") scores = score(logger) logger.update_scores(scores) if scores_do['update']: already_done = [ st for st in args.more_scoretypes if st in logger.config['score_types'] ] if len(already_done) != 0: raise Exception( "Scoretypes", ",".join(already_done), "already done! Remove them from --more_scoretypes") print("\n----- Additional Scores ({}) -----\n".format( args.more_scoretypes)) scores = score(logger, score_types=args.more_scoretypes) logger.update_scores(scores) if scores_do['redo'] or scores_do['update']: logger.dump_results() logger.dump_config() ### Interpolation if interpol_do['redo']: print("\n----- Interpolating -----\n") if args.redo_interpol is not None: i, t = args.redo_interpol logger.config['n_inc'] = i if i >= 0 else logger.config['n_inc'] logger.config['n_test'] = t if t >= 0 else logger.config['n_test'] elif logger.config['n_inc'] == -1: logger.config['n_inc'] = int( logger.data['logs'][0]['counting_abs_states'] / 10) interpol = interpolate(logger) logger.update_interpolation(interpol) if interpol_do['update']: print("\n----- Additional Interpolations ({}) -----\n".format( args.more_scoretypes)) interpol = interpolate(logger, score_types=args.more_scoretypes) logger.update_interpolation(interpol) if interpol_do['redo'] or interpol_do['update']: logger.dump_results() logger.dump_config() ### Display results ### draw_interpol_results(logger, logger.config['score_types'], 0, [1], x_fracs=True, y_fracs=True, smooth=False, x_name='States Restored (%)', y_names=['Original Reward (%)'], combine_sbfl=True) draw_interpol_results(logger, logger.config['score_types'], 4, [1], y_fracs=True, trans_x=lambda x: 1 - x, x_name="Policy's Action Taken (% of Steps)", y_names=['Original Reward (%)'], smooth=False, combine_sbfl=True)