def main(): parser = argparse.ArgumentParser(description='Runs a shell command in the parent shell') parser.add_argument('cmd', metavar='COMMAND', type=str, help='Command to execute') parser.add_argument('section', metavar='SECTION', type=str, help='Section where the Command is executed') args = parser.parse_args() pcmd = get_parser() if not is_config_there(): msg = "No Config found in %s\n" % get_config() parser.exit(status=1, message=msg) if not pcmd.has_section(args.section): msg = "Section %s not in config\n" % args.section parser.exit(status=1, message=msg) if not pcmd.has_option(args.section, args.cmd): msg = "Command %s not in Section %s\n" % (args.cmd, args.section) parser.exit(status=1, message=msg) # run the shell command in the parent terminal do_script(pcmd.get(args.section, args.cmd))
import optimizer_util import config as configs from util import Snapshot, Summary, InitNodes, Metric from job_function_util import get_train_config, get_val_config import resnet_model import resnext_model import vgg_model import alexnet_model import inception_model import mobilenet_v2_model <<<<<<< HEAD ======= >>>>>>> tianshu parser = configs.get_parser() args = parser.parse_args() configs.print_args(args) total_device_num = args.num_nodes * args.gpu_num_per_node train_batch_size = total_device_num * args.batch_size_per_device val_batch_size = total_device_num * args.val_batch_size_per_device (C, H, W) = args.image_shape epoch_size = math.ceil(args.num_examples / train_batch_size) num_val_steps = int(args.num_val_examples / val_batch_size) model_dict = { "resnet50": resnet_model.resnet50, "vgg": vgg_model.vgg16bn, "alexnet": alexnet_model.alexnet, "inceptionv3": inception_model.inceptionv3,
def build_parser(self): parser = get_parser("Augment config") parser.add_argument('--name', default='') parser.add_argument( '--dataset', default='ImageNet', help='imagenet / ImageNet56 / ImageNet112 / cifar10') parser.add_argument('--data_path', default='/gdata/ImageNet2012', help='data path') parser.add_argument('--data_loader_type', default='torch', help='torch/dali') parser.add_argument('--grad_clip', type=float, default=0, help='gradient clipping for weights') parser.add_argument( '--model_method', default='my_model_collection', ) parser.add_argument( '--model_name', default='my_model_collection', ) parser.add_argument('--model_init', type=str, default='he_fout', choices=['he_fin', 'he_fout']) parser.add_argument('--batch_size', type=int, default=256, help='batch size') # parser.add_argument('--lr', type=float, default=0.05, help='lr for weights') # parser.add_argument('--momentum', type=float, default=0.9, help='momentum') # parser.add_argument('--weight_decay', type=float, default=4e-5, help='weight decay') parser.add_argument('--label_smoothing', type=float, default=0.1) parser.add_argument('--no_decay_keys', type=str, default='bn', choices=['None', 'bn', 'bn#bias']) parser.add_argument('--print_freq', type=int, default=1, help='print frequency') parser.add_argument('--gpus', default='0', help='gpu device ids separated by comma. ' '`all` indicates use all gpus.') # parser.add_argument('--epochs', type=int, default=150, help='# of training epochs') parser.add_argument('--init_channels', type=int, default=36) parser.add_argument('--layers', type=int, default=20, help='# of layers') parser.add_argument('--seed', type=int, default=2, help='random seed') parser.add_argument('--workers', type=int, default=4, help='# of workers') parser.add_argument('--aux_weight', type=float, default=0, help='auxiliary loss weight') parser.add_argument('--cutout_length', type=int, default=0, help='cutout length') parser.add_argument('--auto_augmentation', action='store_true', default=False, help='using autoaugmentation') parser.add_argument('--bn_momentum', type=float, default=0.1) parser.add_argument('--bn_eps', type=float, default=1e-3) parser.add_argument('--sync_bn', action='store_true', default=False, help='using sync_bn model') parser.add_argument('--dropout_rate', type=float, default=0) # parser.add_argument('--drop_path_prob', type=float, default=0.2, help='drop path prob') parser.add_argument('--drop_path_prob', type=float, default=0, help='drop path prob') parser.add_argument('--genotype', default='', help='Cell genotype') parser.add_argument('--structure_path', default=None, type=str, help='Config path') parser.add_argument('--deterministic', action='store_true', default=False, help='using deterministic model') parser.add_argument('--pretrained', type=str, default=False, help='load pretrained module') return parser
rho_best = rho epoch_best = epoch log_and_print(base_logger, '-----New best found!-----') if args.save: torch.save( { 'epoch': epoch, 'i3d': i3d.state_dict(), 'evaluator': evaluator.state_dict(), 'optimizer': optimizer.state_dict(), 'rho_best': rho_best }, f'ckpts/{args.type}.pt') if __name__ == '__main__': args = get_parser().parse_args() if not os.path.exists('./exp'): os.mkdir('./exp') if not os.path.exists('./ckpts'): os.mkdir('./ckpts') init_seed(args) base_logger = get_logger(f'exp/{args.type}.log', args.log_info) i3d, evaluator = get_models(args) dataloaders = get_dataloaders(args) main(dataloaders, i3d, evaluator, base_logger, args)
if e.config.summarize: writer.add_scalar( "dev/best_result", best_dev_res, it) for n, v in test_perf.items(): writer.add_scalar( "test/" + n, v, it) e.log.info("best dev result: {:.4f}, " "test result: {:.4f}, " .format(best_dev_res, test_res)) label_stats.reset() unlabel_stats.reset() if __name__ == '__main__': args = config.get_parser().parse_args() args.use_cuda = torch.cuda.is_available() def exit_handler(*args): print(args) print("best dev result: {:.4f}, " "test result: {:.4f}" .format(best_dev_res, test_res)) exit() train_helper.register_exit_handler(exit_handler) np.random.seed(args.random_seed) torch.manual_seed(args.random_seed) with train_helper.experiment(args, args.prefix) as e:
def main(): args = get_parser().parse_args() print(args)
segmentation = Segmentation(segment) section_segmented = segmentation.execute() sections_segmented.append(section_segmented) self.save_solution(sections_segmented, original_image.image) elapsed_time = int(time.time() - start_time) minutes = elapsed_time % 3600 // 60 seconds = elapsed_time % 60 print("Finished extracting in {:02d}:{:02d}".format(minutes, seconds)) print("Finished extracting.") if __name__ == "__main__": parser = get_parser() no_parser = False try: args = parser.parse_args() pass except: args = sys.argv[1:] no_parser = True if no_parser: if args[0] != '-h': solution = Solution(args=args, no_parser=no_parser) solution.run() else: solution = Solution(args=args, no_parser=no_parser)
def main(): #获取参数 vars将parser对象转换为字典 option=vars(get_parser().parse_args()) #首先判断保存结果的文件夹是否存在,并写入参数到json文件中保存 if not os.path.isdir(option['log_file']): os.makedirs(option['log_file']) with open(os.path.join(option['log_file'],'opt.json'),'w') as f: json.dump(option,f) f.write('\n') #定义trace_file文件路径 trace_file=os.path.join(option['log_file'],'trace.txt') #设置随机数种子 torch.manual_seed(2018) if option['data_cuda']: torch.cuda.manual_seed(2018) #step 1 加载数据集 if option['run_mode']=='train_val': trainLoader,valLoader=init_dataset(option,mode=option['run_mode']) #step2 加载模型 model=MatchingNetwork(keep_prob=0.0,num_channles=1,fce=False, image_size=28,batch_size=1,use_cuda=option['data_cuda']) if option['data_cuda']: model.cuda() #step3 目标函数和优化器 optimizer=torch.optim.Adam(model.parameters(),lr=option['train_learningrate']) lr_schedule=torch.optim.lr_scheduler.StepLR(optimizer=optimizer,gamma=0.5,step_size=option['train_decay_every']) #训练 if os.path.isfile(trace_file): os.remove(trace_file) best_acc=0.0 for epoch in range(option['train_epoches']): train_loss=[] train_acc=[] print("------epoch: %2d --------"%epoch) model.train() for data,label in tqdm(trainLoader): data=Variable(data) optimizer.zero_grad() if(option['data_cuda']): data=data.cuda() acc,loss=model(data,option['data_way'],option['data_shot'],option['data_query']) loss.backward() optimizer.step() train_loss.append(float(loss)) train_acc.append(float(acc)) avg_loss=np.mean(train_loss) avg_acc=np.mean(train_acc) lr_schedule.step() print("epoch %2d 训练结束 : avg_loss:%.4f , avg_acc:%.4f"%(epoch,avg_loss,avg_acc)) #下面进入validation阶段 val_acc=[] print("开始进行validation:") model.eval() for data,label in tqdm(valLoader): data=Variable(data) if option['data_cuda']: data=data.cuda() acc_val,_=model(data,option['data_test_way'],option['data_test_shot'],option['data_test_query']) val_acc.append(float(acc_val)) avg_acc_val=np.mean(val_acc) print("validation结束 : avg_acc:%.4f"%avg_acc_val) if (best_acc<avg_acc_val): print("产生目前最佳模型,正在保存......") name=model.save(option['log_file']) best_acc = avg_acc_val print("保存成功,保存在: ",name) with open(trace_file,'a') as f: f.write('epoch:{:2d} 训练结束:avg_loss:{:.4f} , avg_acc:{:.4f} , validation_acc:{:.4f}'.format(epoch,avg_loss,avg_acc,avg_acc_val)) f.write('\n') print("训练结束,最佳模型的精度为:",best_acc)
images = torch.from_numpy(images).type(torch.FloatTensor).to(device) # 当使用CPU时,跳过GPU相关指令 if device != torch.device("cpu"): torch.cuda.synchronize(device) # model_time = time.time() outputs = model(images) outputs = [{k: v.to(cpu_device).numpy() for k, v in t.items()} for t in outputs] infos.append([outputs, targets, shapes]) if draw: for x in range(len(outputs)): os.makedirs('result1', exist_ok=True) savename = os.path.join('result1', os.path.split(shapes[x][2])[-1]) plot_boxes_cv2(images_src[x], outputs[x]['boxes'], savename=savename, class_names=None) torch.set_num_threads(n_threads) Map = make_labels_and_compute_map(infos, parser_data.input_dir, save_err_miss=parser_data.save_err_miss) return Map if __name__ == "__main__": from config import get_parser args = get_parser() lines, num_train, num_val = get_train_lines(args.data_path) args.pth_path = './save_weights/Epoch_022_Loss_0.0736_lr_0.000059.pth' main(args, lines, draw=True)
parser.add_argument('--eval', type=ast.literal_eval, default=True, help='Evaluate the interpretation results or not') parser.add_argument('--path', type=str, default="data/", help='the input data path, can be a single csv ' 'or a data folder') parser.add_argument('--w2s_ratio', type=str, default='real_len', help='\'real-len\', \'auto\', \'pn\', or a ratio.') parser.add_argument('--runs', type=int, default=1) parser.add_argument('--record_name', type=str, default='') parser = get_parser(algorithm_name, parser) args = parser.parse_args() input_root_list = [root + args.path] w2s_ratio = args.w2s_ratio od_eval_model = [ "iforest", "copod", "hbos" ] # we obtain ground-truth annotations using three outlier detection methods runs = args.runs record_name = args.record_name # ------------------- record ----------------- # if not os.path.exists("record/" + algorithm_name): os.makedirs("record/" + algorithm_name) if not os.path.exists("checkpoints"): os.makedirs("checkpoints/")
smiles = [] for i in df.itertuples(index=False): if len(i < max_len): smiles.append(i) vocab = OneHotVocab.from_data(smiles) experiment = Experiment(project_name='pytorch') train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=8 * 6, pin_memory=True) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=True, num_workers=8 * 6, pin_memory=True) model = VAE(vocab, get_parser()) optimizer = optim.Adam((p for p in model.vae.parameters() if p.requires_grad), lr=0.0003) def train(): return def test(): return def sample(): return for epoch in range(1, epochs + 1):
import sounddevice as sd import numpy as np import comm import huffman import rec import config import os sd.default.samplerate = fs = 48000 sd.default.channels = 1 start_time = 0 end_time = 0 # args = get_args() parser = config.get_parser() args = parser.parse_args() if __name__ == '__main__': print("Hi~ Welcome to project check script") if args.send: huffman.getHuffFile() print('You are sending signal...') while True: part = input("Press any key to send\n") # if part == 'e': # exit comm.play_file() print('Done\n') else: print('You are receiving signal...') if os.path.exists('record.wav'): os.remove('record.wav')
def get_config(): parser = config.get_parser() # pretrain bert config parser.add_argument( "--ofrecord_path", type=str, default="/dataset/bert/of_wiki_seq_len_128", help="Path to ofrecord dataset", ) parser.add_argument( "--train-dataset-size", type=int, default=10000000, help="dataset size of ofrecord", ) parser.add_argument( "--train-data-part", type=int, default=64, help="data part num of ofrecord" ) parser.add_argument( "--train-batch-size", type=int, default=8, help="Training batch size" ) parser.add_argument( "--val-batch-size", type=int, default=32, help="Validation batch size" ) parser.add_argument( "--train-global-batch-size", type=int, default=None, dest="train_global_batch_size", help="train batch size", ) parser.add_argument( "--val-global-batch-size", type=int, default=None, dest="val_global_batch_size", help="val batch size", ) parser.add_argument("-e", "--epochs", type=int, default=1, help="Number of epochs") parser.add_argument( "--with-cuda", type=bool, default=True, help="Training with CUDA: true, or false", ) parser.add_argument( "--cuda_devices", type=int, nargs="+", default=None, help="CUDA device ids" ) parser.add_argument( "--optim_name", type=str, default="adamw", help="optimizer name" ) parser.add_argument("--lr", type=float, default=1e-3, help="Learning rate of adam") parser.add_argument( "--weight_decay", type=float, default=0.01, help="Weight_decay of adam" ) parser.add_argument( "--loss_print_every_n_iters", type=int, default=20, help="Interval of training loss printing", ) parser.add_argument( "--val_print_every_n_iters", type=int, default=20, help="Interval of evaluation printing", ) parser.add_argument( "--checkpoint_path", type=str, default="checkpoints", help="Path to model saving", ) parser.add_argument( "--grad-acc-steps", type=int, default=1, help="Steps for gradient accumulation" ) parser.add_argument( "--nccl-fusion-threshold-mb", type=int, default=16, dest="nccl_fusion_threshold_mb", help="NCCL fusion threshold megabytes, set to 0 to compatible with previous version of OneFlow.", ) parser.add_argument( "--nccl-fusion-max-ops", type=int, default=24, dest="nccl_fusion_max_ops", help="Maximum number of ops of NCCL fusion, set to 0 to compatible with previous version of OneFlow.", ) parser.add_argument( "--use_ddp", type=str2bool, nargs="?", const=True, help="Whether to use use fp16", ) parser.add_argument( "--use_consistent", type=str2bool, nargs="?", const=True, help="Whether to use use consistent", ) parser.add_argument( "--metric-local", type=str2bool, default=False, nargs="?", const=True, dest="metric_local", ) args = parser.parse_args() return args