def preloader(): current_path = os.path.dirname(os.path.abspath(__file__)) config_path = f'{current_path}/config/hrnet_plate.yaml' seg_weights = f'{current_path}/weights/hrnetv2_hrnet_plate_199.pth' output_dir = f'{current_path}/' data_name = 'plate' cfg = get_cfg_defaults() cfg.defrost() cfg.merge_from_file(config_path) cfg.merge_from_list([ "train.config_path", config_path, "train.output_dir", output_dir, "dataset.data_name", data_name, ]) print( torch.load(os.path.join(f"{current_path}/data", data_name + ".pth"), map_location=torch.device('cpu'))) model = create_model(cfg) # if torch.cuda.is_available(): # model.cuda() model = nn.DataParallel(model) print(torch.load(seg_weights, map_location=torch.device('cpu')).keys()) model.load_state_dict( torch.load(seg_weights, map_location=torch.device('cpu'))["state_dict"]) model.eval() return model, cfg,
def __init__(self): self.cfg = get_cfg_defaults() self.model = build_model(self.cfg) self.device = torch.device(self.cfg.MODEL.DEVICE) self.model.to(self.device) self.model.train(False) self.model.load_state_dict(torch.load(self.cfg.MODEL.WEIGHTS))
def __init__(self, width): super(HRNet, self).__init__() cfg = get_cfg_defaults() cfg.merge_from_file("./experiments/hrnet.yaml") self.hr0 = HighResolutionNet(cfg) self.hr1 = HighResolutionNet(cfg) self.head = Regressor(in_channel=540, width=int(width / 4), n_class=5)
def __init__(self, model_path): self.cfg = get_cfg_defaults() self.model = build_model(self.cfg) self.device = torch.device(self.cfg.MODEL.DEVICE) self.model.to(self.device) self.model.train(False) self.state_dict = torch.load(model_path)
def load_model(model_path, cfg_path): import h5py import json cfg = get_cfg_defaults() cfg.merge_from_file(cfg_path) cfg.freeze() logging.info(cfg) with h5py.File(model_path, "r") as f: mappings = json.loads(f.attrs["mappings"]) label_key = f.attrs["label_key"] vocab_size = f.attrs["vocab_size"] n_class_labels = f.attrs["n_class_labels"] word_length = f.attrs["word_length"] if cfg.MODEL.CLASSIFIER == ["crf"]: from .keraslayers.ChainCRF import create_custom_objects custom_objects = create_custom_objects() model = keras.models.load_model(model_path, custom_objects=custom_objects) bilstm = BiLSTM(cfg) bilstm.set_vocab(vocab_size, n_class_labels, word_length, mappings) bilstm.model = model bilstm.label_key = label_key return bilstm
def main(): args = arg_parse() # ---- set configs, logger and device ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() tb_logger = TensorBoardLogger("tb_logs", name=cfg.DATASET.NAME) device = "cuda" if torch.cuda.is_available() else "cpu" # ---- set dataset ---- train_dataset = BindingDBDataset(name=cfg.DATASET.NAME, split="train", path=cfg.DATASET.PATH) val_dataset = BindingDBDataset(name=cfg.DATASET.NAME, split="valid", path=cfg.DATASET.PATH) test_dataset = BindingDBDataset(name=cfg.DATASET.NAME, split="test", path=cfg.DATASET.PATH) train_loader = DataLoader(dataset=train_dataset, shuffle=True, batch_size=cfg.SOLVER.TRAIN_BATCH_SIZE) val_loader = DataLoader(dataset=val_dataset, shuffle=True, batch_size=cfg.SOLVER.TEST_BATCH_SIZE) test_loader = DataLoader(dataset=test_dataset, shuffle=True, batch_size=cfg.SOLVER.TEST_BATCH_SIZE) # ---- set model ---- model = get_model(cfg) # ---- training and evaluation ---- gpus = 1 if device == "cuda" else 0 checkpoint_callback = ModelCheckpoint(monitor='val_loss', mode="min") trainer = pl.Trainer(max_epochs=cfg.SOLVER.MAX_EPOCHS, gpus=gpus, logger=tb_logger, callbacks=[checkpoint_callback]) trainer.fit(model, train_dataloader=train_loader, val_dataloaders=val_loader) trainer.test(test_dataloaders=test_loader)
def main(): args = arg_parse() # ---- setup device ---- device = 'cuda' if torch.cuda.is_available() else 'cpu' print('==> Using device ' + device) # ---- setup config ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() seed.set_seed(cfg.SOLVER.SEED) # ---- setup logger ---- os.makedirs(cfg.OUTPUT_DIR, exist_ok=True) logger = logging.construct_logger('context_cnns', cfg.OUTPUT_DIR) logger.info(f'Using {device}') logger.info('\n' + cfg.dump()) # ---- setup dataset ---- train_loader, val_loader = get_cifar(cfg) # ---- setup model ---- print('==> Building model..') net = get_model(cfg) net = net.to(device) model_stats = summary(net, (3, 32, 32), depth=10) logger.info('\n'+str(model_stats)) if device=='cuda': net = torch.nn.DataParallel(net) # ---- setup trainers ---- optim = torch.optim.SGD(net.parameters(), lr=cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM, weight_decay=cfg.SOLVER.WEIGHT_DECAY) trainer = Trainer( device, train_loader, val_loader, net, optim, logger, cfg ) if args.resume: # Load checkpoint print('==> Resuming from checkpoint..') cp = torch.load(args.resume) trainer.model.load_state_dict(cp['net']) trainer.optim.load_state_dict(cp['optim']) trainer.epochs = cp['epoch'] trainer.train_acc = cp['train_accuracy'] trainer.val_acc = cp['test_accuracy'] trainer.train()
def main(): """The main for this domain adapation example, showing the workflow""" args = arg_parse() # ---- setup device ---- device = "cuda" if torch.cuda.is_available() else "cpu" print("==> Using device " + device) # ---- setup configs ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() set_seed(cfg.SOLVER.SEED) # ---- setup logger and output ---- output_dir = os.path.join(cfg.OUTPUT_DIR, cfg.DATASET.NAME, args.output) os.makedirs(output_dir, exist_ok=True) logger = construct_logger("isonet", output_dir) logger.info("Using " + device) logger.info("\n" + cfg.dump()) # ---- setup dataset ---- train_loader, val_loader = get_cifar(cfg) print("==> Building model..") net = get_model(cfg) # print(net) net = net.to(device) # model_stats = summary(net, (3, 32, 32)) # logger.info('\n'+str(model_stats)) # Needed even for single GPU https://discuss.pytorch.org/t/attributeerror-net-object-has-no-attribute-module/45652 if device == "cuda": net = torch.nn.DataParallel(net) optim = torch.optim.SGD( net.parameters(), lr=cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM, weight_decay=cfg.SOLVER.WEIGHT_DECAY, dampening=cfg.SOLVER.DAMPENING, nesterov=cfg.SOLVER.NESTEROV, ) trainer = Trainer(device, train_loader, val_loader, net, optim, logger, output_dir, cfg) if args.resume: # Load checkpoint print("==> Resuming from checkpoint..") cp = torch.load(args.resume) trainer.model.load_state_dict(cp["net"]) trainer.optim.load_state_dict(cp["optim"]) trainer.epochs = cp["epoch"] trainer.train_acc = cp["train_accuracy"] trainer.val_acc = cp["test_accuracy"] trainer.train()
def main(): # For now hard coding the values for all methods below, change to read it using config file print("Start..") args = arg_parse() # ---- set configs, logger and device ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() device = "cuda" if torch.cuda.is_available() else "cpu" # Fetch the bindingDB dataset based on the name defined in config file bindingdb_dataset = DTI(name=cfg.DATASET.NAME) # Apply cluster based on the 3 types of cluster # can define cluster_type -> k_means, agglomerative, dbscan # bindingdb_dataset = apply_clustering(bindingdb_dataset, num_of_clusters=cfg.SOLVER.NUM_OF_CLUSTERS, # cluster_type=cfg.MODEL.CLUSTER_TYPE) bindingdb_dataset_cluster = apply_clustering( bindingdb_dataset, num_of_clusters=cfg.SOLVER.NUM_OF_CLUSTERS, cluster_type=cfg.MODEL.CLUSTER_TYPE) # Split the data based on the clusters formed by specifying the split in fraction train_dataset, val_dataset, test_dataset = get_split_by_clusters( bindingdb_dataset_cluster, num_of_clusters=cfg.SOLVER.NUM_OF_CLUSTERS) train_dataset = DTADataset(ds=train_dataset) val_dataset = DTADataset(ds=val_dataset) test_dataset = DTADataset(ds=test_dataset) train_loader = DataLoader(dataset=train_dataset, shuffle=True, batch_size=cfg.SOLVER.TRAIN_BATCH_SIZE) val_loader = DataLoader(dataset=val_dataset, shuffle=True, batch_size=cfg.SOLVER.TEST_BATCH_SIZE) test_loader = DataLoader(dataset=test_dataset, shuffle=True, batch_size=cfg.SOLVER.TEST_BATCH_SIZE) # ---- set model ---- model = get_model(cfg) # ---- training and evaluation ---- gpus = 1 if device == "cuda" else 0 checkpoint_callback = ModelCheckpoint(monitor='val_loss', mode="min") trainer = pl.Trainer(max_epochs=cfg.SOLVER.MAX_EPOCHS, gpus=gpus, callbacks=[checkpoint_callback]) trainer.fit(model, train_dataloader=train_loader, val_dataloaders=val_loader) trainer.test(test_dataloaders=test_loader) print("Done!!!")
def main(): """The main for this multi-source domain adaptation example, showing the workflow""" args = arg_parse() # ---- setup configs ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() print(cfg) # ---- setup output ---- format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)" logging.basicConfig(format=format_str) # ---- setup dataset ---- if type(cfg.DATASET.SOURCE) == list: sub_domain_set = cfg.DATASET.SOURCE + [cfg.DATASET.TARGET] else: sub_domain_set = None num_channels = cfg.DATASET.NUM_CHANNELS if cfg.DATASET.NAME.upper() == "DIGITS": kwargs = {"return_domain_label": True} else: kwargs = {"download": True, "return_domain_label": True} data_access = ImageAccess.get_multi_domain_images( cfg.DATASET.NAME.upper(), cfg.DATASET.ROOT, sub_domain_set=sub_domain_set, **kwargs ) # Repeat multiple times to get std for i in range(0, cfg.DATASET.NUM_REPEAT): seed = cfg.SOLVER.SEED + i * 10 dataset = MultiDomainAdapDataset(data_access, random_state=seed) set_seed(seed) # seed_everything in pytorch_lightning did not set torch.backends.cudnn print(f"==> Building model for seed {seed} ......") # ---- setup model and logger ---- model, train_params = get_model(cfg, dataset, num_channels) tb_logger = TensorBoardLogger(cfg.OUTPUT.TB_DIR, name="seed{}".format(seed)) checkpoint_callback = ModelCheckpoint( filename="{epoch}-{step}-{valid_loss:.4f}", monitor="valid_loss", mode="min", ) progress_bar = TQDMProgressBar(cfg.OUTPUT.PB_FRESH) trainer = pl.Trainer( min_epochs=cfg.SOLVER.MIN_EPOCHS, max_epochs=cfg.SOLVER.MAX_EPOCHS, callbacks=[checkpoint_callback, progress_bar], gpus=args.gpus, auto_select_gpus=True, logger=tb_logger, # logger, # weights_summary='full', fast_dev_run=False, # True, ) trainer.fit(model) trainer.test()
def __init__(self, width): super(HRNet, self).__init__() cfg = get_cfg_defaults() cfg.merge_from_file("./experiments/exp.yaml") self.hr = HighResolutionNet(cfg) self.add_coords = AddCoords(rank=2) self.conv5 = nn.Conv2d(7, 7, 3, padding=1) self.conv6 = nn.Conv2d(7, 7, 3, padding=1) self.conv7 = nn.Conv2d(7, 4, 1) self.pool = nn.MaxPool2d(width, stride=width)
def main(): N = 256 H = 32 W = 32 x = np.random.randn(N, H, W, 3) y = np.random.randn(N, H, W, 3) cfg = get_cfg_defaults() dnet = GenericKernel(kernel_cfg=cfg.KERNEL) start = timer() K = generate_kernel(dnet, x, y) end = timer() print(f"{N} x {N} kernel took {end - start} seconds")
def main(config_file: str): cfg = get_cfg_defaults() cfg.merge_from_file(config_file) cfg.TRAIN.N_PARALLEL = 1 cfg.freeze() metrics = eval_model(cfg) out_file = os.path.join(cfg.TEST.OUTDIR, "performance_score.json") json.dump(metrics, open(out_file, "w")) for k, v in metrics.items(): print(f"{k}: {v:.2}")
def __init__(self): self.this_dir = osp.abspath(osp.dirname(__file__)) yaml_path = osp.join(self.this_dir, 'face_lib.yaml') self.cfg = get_cfg_defaults() self.cfg.merge_from_file(yaml_path) self.cfg.freeze() if self.cfg.PET_ENGINE.USE_ENGINE: self.init_pet_engine() self.cos_dis = torch.nn.CosineSimilarity() self.face_db_dict = {}
def main(): """The main for this domain adaptation example, showing the workflow""" args = arg_parse() # ---- setup configs ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() print(cfg) # ---- setup output ---- format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)" logging.basicConfig(format=format_str) # ---- setup dataset ---- source, target, num_channels = DigitDataset.get_source_target( DigitDataset(cfg.DATASET.SOURCE.upper()), DigitDataset(cfg.DATASET.TARGET.upper()), cfg.DATASET.ROOT) dataset = MultiDomainDatasets( source, target, config_weight_type=cfg.DATASET.WEIGHT_TYPE, config_size_type=cfg.DATASET.SIZE_TYPE, valid_split_ratio=cfg.DATASET.VALID_SPLIT_RATIO, ) # Repeat multiple times to get std for i in range(0, cfg.DATASET.NUM_REPEAT): seed = cfg.SOLVER.SEED + i * 10 # seed_everything in pytorch_lightning did not set torch.backends.cudnn set_seed(seed) print(f"==> Building model for seed {seed} ......") # ---- setup model and logger ---- model, train_params = get_model(cfg, dataset, num_channels) tb_logger = pl_loggers.TensorBoardLogger(cfg.OUTPUT.TB_DIR, name="seed{}".format(seed)) checkpoint_callback = ModelCheckpoint( filename="{epoch}-{step}-{valid_loss:.4f}", monitor="valid_loss", mode="min", ) progress_bar = TQDMProgressBar(cfg.OUTPUT.PB_FRESH) trainer = pl.Trainer( min_epochs=cfg.SOLVER.MIN_EPOCHS, max_epochs=cfg.SOLVER.MAX_EPOCHS, callbacks=[checkpoint_callback, progress_bar], logger=tb_logger, gpus=args.gpus, ) trainer.fit(model) trainer.test()
def __init__(self, model_path): config = self._get_tf_config() args = get_cfg_defaults() self.tf_graph = tf.Graph() with self.tf_graph.as_default(): self.policy = CMPPolicy(args, is_training=False, batch_norm_is_training=False, only_single_step_graph=True) self.sess = tf.Session(config=config) self.sess.run(self.policy.init_op) self.policy.saver_op.restore(self.sess, model_path)
def main(): args = arg_parse() # ---- setup device ---- device = "cuda" if torch.cuda.is_available() else "cpu" print("==> Using device " + device) # ---- setup config ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() seed.set_seed(cfg.SOLVER.SEED) # ---- setup logger ---- os.makedirs(cfg.OUTPUT_DIR, exist_ok=True) logger = logging.construct_logger("context_cnns", cfg.OUTPUT_DIR) logger.info(f"Using {device}") logger.info("\n" + cfg.dump()) # ---- setup dataset ---- train_loader, valid_loader = get_cifar(cfg) # ---- setup model ---- print("==> Building model..") net = get_model(cfg) net = net.to(device) model_stats = summary(net, (3, 32, 32)) logger.info("\n" + str(model_stats)) if device == "cuda": net = torch.nn.DataParallel(net) # ---- setup trainers ---- optim = torch.optim.SGD( net.parameters(), lr=cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM, weight_decay=cfg.SOLVER.WEIGHT_DECAY ) trainer = Trainer(device, train_loader, valid_loader, net, optim, logger, cfg) if args.resume: # Load checkpoint print("==> Resuming from checkpoint..") cp = torch.load(args.resume) trainer.model.load_state_dict(cp["net"]) trainer.optim.load_state_dict(cp["optim"]) trainer.epochs = cp["epoch"] trainer.train_acc = cp["train_accuracy"] trainer.valid_acc = cp["test_accuracy"] trainer.train()
def run(): cfg = get_cfg_defaults() cfg.merge_from_file("config/base.yaml") cfg.freeze() log = ezlog.EZLOG(os.path.basename(__file__)) log.logger.info("=================restarting=========================") word_counter = Counter() ############################################ #SCAN TXT with open(cfg.SEGMENTED_TXT, 'rb') as f: #f = codecs.open(cfg['subword']['output'], 'r', encoding='utf8') for line in tqdm(f): try: line = line.decode('utf-8').strip() except Exception as e: print(line) print(e) continue article_one = line.strip().split(' ') if article_one == []: continue conter_one = Counter(article_one) word_counter += conter_one log.logger.info("process lines done") ############################################# #GET TOP-K WORDS N = len(word_counter.keys()) words_counter = word_counter.most_common(N) words = [] for word, freq in words_counter: if freq < cfg.MIN_WORD_FREQ: break words.append(word) ############################################## ##WORD2INT AND INT2WORD word2int, int2word = {}, {} words = list(set(words)) for index, word in enumerate(words): word2int[word] = index int2word[index] = word with open(cfg.WORD_DICT, 'wb') as f: pickle.dump((word2int, int2word, words_counter), f) log.logger.info("create file :{}".format(cfg.WORD_DICT))
def main(config_file: str): cfg = get_cfg_defaults() cfg.merge_from_file(config_file) cfg.freeze() if not os.path.exists(cfg.LOG.OUTDIR): os.makedirs(cfg.LOG.OUTDIR) filename = os.path.join(cfg.LOG.OUTDIR, "config.yaml") with open(filename, "w") as f: f.write(cfg.dump()) if cfg.LOG.NEPTUNE: setup_neptune(cfg) train(cfg)
def main(): args = arg_parse() # ---- setup device ---- device = "cuda" if torch.cuda.is_available() else "cpu" print("==> Using device " + device) # ---- setup configs ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() seed.set_seed(cfg.SOLVER.SEED) # ---- setup logger and output ---- output_dir = os.path.join(cfg.OUTPUT_DIR, cfg.DATASET.NAME, args.output) os.makedirs(output_dir, exist_ok=True) logger = lu.construct_logger("gripnet", output_dir) logger.info("Using " + device) logger.info(cfg.dump()) # ---- setup dataset ---- download_file_by_url(cfg.DATASET.URL, cfg.DATASET.ROOT, "pose.pt", "pt") data = torch.load(os.path.join(cfg.DATASET.ROOT, "pose.pt")) device = torch.device(device) data = data.to(device) # ---- setup model ---- print("==> Building model..") model = GripNet( cfg.GRIPN.GG_LAYERS, cfg.GRIPN.GD_LAYERS, cfg.GRIPN.DD_LAYERS, data.n_d_node, data.n_g_node, data.n_dd_edge_type ).to(device) # TODO Visualize model # ---- setup trainers ---- optimizer = torch.optim.Adam(model.parameters(), lr=cfg.SOLVER.BASE_LR) # TODO trainer = Trainer(cfg, device, data, model, optimizer, logger, output_dir) if args.resume: # Load checkpoint print("==> Resuming from checkpoint..") cp = torch.load(args.resume) trainer.model.load_state_dict(cp["net"]) trainer.optim.load_state_dict(cp["optim"]) trainer.epochs = cp["epoch"] trainer.train_auprc = cp["train_auprc"] trainer.valid_auprc = cp["valid_auprc"] trainer.train_auroc = cp["train_auroc"] trainer.valid_auroc = cp["valid_auroc"] trainer.train_ap = cp["train_ap"] trainer.valid_ap = cp["valid_ap"] trainer.train()
def run(): custom_punc = '------------ ,。、【 】 “”:;()《》‘’{}?!⑦()、%^>℃:.”“^-——=&#@¥… …\n\r ' all_punc = set(punctuation + custom_punc) cfg = get_cfg_defaults() cfg.merge_from_file("config/base.yaml") cfg.freeze() #working_folder = os.path.dirname(__file__) #raw_output = os.path.join(working_folder, cfg.SEGMENTED_TXT) raw_output = cfg.SEGMENTED_TXT if not os.path.exists(os.path.dirname(raw_output)): os.makedirs(os.path.dirname(raw_output)) program = os.path.basename(__file__) # 得到文件名 logger = logging.getLogger(program) logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s') logging.root.setLevel(level=logging.INFO) ################################################# #LOADING STOPWORDS stopwords = [] f = codecs.open(cfg.STOPWORDS, 'r', encoding='utf8') for line in f: stopwords.append(line.strip()) stopwords = set(stopwords) all_punc = all_punc | stopwords bar = tqdm(os.listdir(os.path.join(cfg.RAW_TEXT_DIR))) lines = [] for txt in bar: bar.set_description(txt) raw_input = os.path.join(cfg.RAW_TEXT_DIR, txt) with open(raw_input, 'r', encoding='utf-8') as f: for line in f: words = jieba.lcut(line) #分词 words = filter(lambda x: x not in all_punc, words) #停用词和标点符号 words = filter(lambda x: len(x) >= cfg.MIN_WORD_LEN, words) #单词最小长度 words = list(words) if words == []: continue lines.append(' '.join(words)) with open(raw_output, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) logger.info("Finished Saved articles.")
def main(): parser = argparse.ArgumentParser(description="PyTorch Self-driving Car Training and Inference.") parser.add_argument( "--config-file", default="", metavar="file", help="path to config file", type=str, ) parser.add_argument( "--mode", default="test", metavar="mode", help="'train' or 'test'", type=str, ) parser.add_argument( "opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER, ) args = parser.parse_args() # build the config cfg = get_cfg_defaults() # cfg.merge_from_file(args.config_file) # cfg.merge_from_list(args.opts) cfg.freeze() # setup the logger if not os.path.isdir(cfg.OUTPUT.DIR): os.mkdir(cfg.OUTPUT.DIR) logger = setup_logger("balad-mobile.train", cfg.OUTPUT.DIR, '{0:%Y-%m-%d %H:%M:%S}_log'.format(datetime.now())) logger.info(args) logger.info("Running with config:\n{}".format(cfg)) # TRAIN train(cfg) # Visualize visualization(cfg)
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('expert_policy_file', type=str) parser.add_argument('envname', type=str) parser.add_argument('data_file', type=str) parser.add_argument('--render', action='store_true') parser.add_argument('--max_timesteps', type=int) parser.add_argument('--num_rollouts', type=int, default=20, help='Number of expert roll outs') parser.add_argument('--test', action='store_true') args = parser.parse_args() # build the config cfg = get_cfg_defaults() cfg.freeze() if args.test: test(args, cfg) else: train(args, cfg)
def parse_args(): parser = argparse.ArgumentParser(description='Train network') parser.add_argument('--cfg', help='experiment configure file name', required=True, type=str) parser.add_argument("--local_rank", type=int, default=-1) parser.add_argument('opts', help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER) args = parser.parse_args() cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.merge_from_list(args.opts) cfg.freeze() return args, cfg
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('envname', type=str) parser.add_argument('--dagger', action='store_true') parser.add_argument('--num_rollouts', type=int, default=20, help='Number of expert roll outs in dagger mode') args = parser.parse_args() # build the config cfg = get_cfg_defaults() cfg.freeze() data = load_data(cfg, args.envname) X_train, X_test, y_train, y_test = split_data(cfg, data) input_size = np.prod(X_train[0].shape) output_size = np.prod(y_train[0].shape) model = build_model(cfg, input_size, output_size) if args.dagger: model_file_path = get_model_file_path(cfg, cfg.MODEL.NN.DAGGER_FILE_NAME, args.envname) rewards_mean, rewards_std = train_dagger(cfg, model, model_file_path, args.envname, X_train, y_train, args.num_rollouts) print('mean of rewards per iterations is', rewards_mean) print('std of rewards per iterations is', rewards_std) else: model_file_path = get_model_file_path(cfg, cfg.MODEL.NN.BC_FILE_NAME, args.envname) train(cfg, model, model_file_path, X_train, y_train) test(model_file_path, X_test, y_test)
def main(): """The main for this domain adaptation example, showing the workflow""" args = arg_parse() # ---- setup configs ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() print(cfg) # ---- setup output ---- format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)" logging.basicConfig(format=format_str) # ---- setup dataset ---- seed = cfg.SOLVER.SEED source, target, num_classes = VideoDataset.get_source_target( VideoDataset(cfg.DATASET.SOURCE.upper()), VideoDataset(cfg.DATASET.TARGET.upper()), seed, cfg) dataset = VideoMultiDomainDatasets( source, target, image_modality=cfg.DATASET.IMAGE_MODALITY, seed=seed, config_weight_type=cfg.DATASET.WEIGHT_TYPE, config_size_type=cfg.DATASET.SIZE_TYPE, ) # ---- setup model and logger ---- model, train_params = get_model(cfg, dataset, num_classes) trainer = pl.Trainer( logger=False, resume_from_checkpoint=args.ckpt, gpus=args.gpus, ) model_test = weights_update(model=model, checkpoint=torch.load(args.ckpt)) # test scores trainer.test(model=model_test)
def main(): parser = argparse.ArgumentParser("Train + evaluate kernel model") parser.add_argument("--config", default=None) parser.add_argument('--config-updates', default=[], nargs='*') args = parser.parse_args() cfg = get_cfg_defaults() if args.config is not None: cfg.merge_from_file(args.config) cfg.merge_from_list(args.config_updates) if cfg.SYSTEM.NUM_GPUS == 0: cfg.SYSTEM.NUM_GPUS = torch.cuda.device_count() cfg.freeze() print(cfg) # load dataset random.seed(cfg.DATASET.RAND_SEED) X_train, y_train, X_test, y_test = load_dataset(cfg) print(X_train.shape) print(X_test.shape) K_train, K_test = generate_kernels(cfg, X_train, X_test) best_model_result, best_train_result, best_test_result = solve_kernel(cfg, K_train, y_train, K_test, y_test)
def main(): parser = argparse.ArgumentParser( description="Deep Neural Networks for 3D Anaglyph Image Generation") parser.add_argument( "--config-file", default="", metavar="file", help="path to config file", type=str, ) parser.add_argument( "--mode", default="test", metavar="mode", help="'train' or 'test'", type=str, ) parser.add_argument( "opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER, ) args = parser.parse_args() # build the config cfg = get_cfg_defaults() cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() dataset = build_dataset(cfg) model, optimizer = build_model(cfg) train(cfg, optimizer, dataset)
def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--cfg', required=True) parser.add_argument( '--load', help='resume from a checkpoint with optimizer parameter attached') parser.add_argument('--n_threads', default=4) parser.add_argument('--save', required=True) parser.add_argument('--max_memory', type=int, default=None) parser.add_argument('--mem_every', type=int, default=5) parser.add_argument('--gpu', type=int, default=0) parser.add_argument('opts', help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER) args = parser.parse_args() cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.merge_from_list(args.opts) cfg.freeze() return args, cfg
def main(): """The main for this domain adaptation example, showing the workflow""" args = arg_parse() # ---- setup configs ---- cfg = get_cfg_defaults() cfg.merge_from_file(args.cfg) cfg.freeze() print(cfg) # ---- setup output ---- os.makedirs(cfg.OUTPUT.DIR, exist_ok=True) format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)" logging.basicConfig(format=format_str) # ---- setup dataset ---- source, target, num_channels = DigitDataset.get_source_target( DigitDataset(cfg.DATASET.SOURCE.upper()), DigitDataset(cfg.DATASET.TARGET.upper()), cfg.DATASET.ROOT) dataset = MultiDomainDatasets(source, target, config_weight_type=cfg.DATASET.WEIGHT_TYPE, config_size_type=cfg.DATASET.SIZE_TYPE) # Repeat multiple times to get std for i in range(0, cfg.DATASET.NUM_REPEAT): seed = cfg.SOLVER.SEED + i * 10 # seed_everything in pytorch_lightning did not set torch.backends.cudnn set_seed(seed) print(f"==> Building model for seed {seed} ......") # ---- setup model and logger ---- model, train_params = get_model(cfg, dataset, num_channels) logger, results, checkpoint_callback, test_csv_file = setup_logger( train_params, cfg.OUTPUT.DIR, cfg.DAN.METHOD, seed) trainer = pl.Trainer( progress_bar_refresh_rate=cfg.OUTPUT.PB_FRESH, # in steps min_epochs=cfg.SOLVER.MIN_EPOCHS, max_epochs=cfg.SOLVER.MAX_EPOCHS, checkpoint_callback=checkpoint_callback, # resume_from_checkpoint=last_checkpoint_file, gpus=args.gpus, logger=False, # logger, # weights_summary='full', fast_dev_run=True, # True, ) trainer.fit(model) results.update( is_validation=True, method_name=cfg.DAN.METHOD, seed=seed, metric_values=trainer.callback_metrics, ) # test scores trainer.test() results.update( is_validation=False, method_name=cfg.DAN.METHOD, seed=seed, metric_values=trainer.callback_metrics, ) results.to_csv(test_csv_file) results.print_scores(cfg.DAN.METHOD)