def __init__(self, seq, max_scan=45): """ cs_probs_all = probability of finding a cleavage site at all positions. Scored by 5 random forests. The position here is 0 based. shape=(5, num_subseqs) cleavage_sites = Max of cs_probs_all for each random forest (length 5) """ self.max_scan = functions.validate_scan(seq, max_scan) self.seq = functions.validate(seq, self.max_scan) self.s_scores = None self.c_scores = None self.y_scores = None self.cs_probs_all = None self.cleavage_sites = None self.preds = None self.final_cleavage_prob = None self.final_cleavage = 0 self.final_score_sp = None self.fungi_scores = functions.np.zeros(5) self.fungi_preds = functions.np.zeros(5, dtype=bool) self.final_score_fungi = None self.toxin_scores = functions.np.zeros(5) self.toxin_preds = functions.np.zeros(5, dtype=bool) self.final_score_toxin = None self.run_fungi = False self.run_toxin = False
def main(): # 사용자가 종료할 때까지 무한 루프 while True: equation = input('Please input equation(quit: q): ') if equation == 'q': print('Bye') break x_value = input('Please input x: ') # 입력값이 유효한지를 체크 is_validated = validate(equation, x_value) if is_validated is False: continue # 방정식에 x값을 대입하고 우변만 쪼갠다 expression = make_expression(equation, x_value) # 입력된 계산이 전위/중위/후위 표기인지 판단 # (오류를 발견해 기능을 숨깁니다) # notation = select_notations(expression) # # # 판단된 표기법에 따라 계산 진행 # # (전위, 후위 표기법에 따른 계산은 해결하지 못함) # if notation is 'prefix': # answer = 'prefix' # elif notation is 'infix': # answer = evaluate_infix(expression) # elif notation is 'postfix': # answer = 'postfix' answer = evaluate_infix(expression) # 해답 출력 및 1초간 휴식 후 루프 재개 print(f'y: {answer}') time.sleep(1)
def main(): args = cfg.parse_args() torch.manual_seed(args.random_seed) random.seed(args.random_seed) torch.cuda.manual_seed(args.random_seed) assert args.exp_name assert args.load_path.endswith('.pth') assert os.path.exists(args.load_path) args.path_helper = set_log_dir('logs_eval', args.exp_name) logger = create_logger(args.path_helper['log_path'], phase='test') # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # import network gen_net = eval('models.' + args.model + '.Generator')(args=args).cuda() # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' else: raise NotImplementedError(f'no fid stat for {args.dataset.lower()}') assert os.path.exists(fid_stat) # initial np.random.seed(args.random_seed) fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) if args.percent < 0.9: pruning_generate(gen_net, (1 - args.percent)) see_remain_rate(gen_net) # set writer logger.info(f'=> resuming from {args.load_path}') checkpoint_file = args.load_path assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) if 'avg_gen_state_dict' in checkpoint: gen_net.load_state_dict(checkpoint['avg_gen_state_dict']) epoch = checkpoint['epoch'] logger.info(f'=> loaded checkpoint {checkpoint_file} (epoch {epoch})') else: gen_net.load_state_dict(checkpoint) logger.info(f'=> loaded checkpoint {checkpoint_file}') logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'valid_global_steps': 0, } inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict, epoch) logger.info(f'Inception score: {inception_score}, FID score: {fid_score}.') writer_dict['writer'].close()
def main(): args = cfg.parse_args() torch.cuda.manual_seed(args.random_seed) assert args.exp_name assert args.load_path.endswith(".pth") assert os.path.exists(args.load_path) args.path_helper = set_log_dir("logs_eval", args.exp_name) logger = create_logger(args.path_helper["log_path"], phase="test") # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # import network gen_net = eval("models." + args.gen_model + ".Generator")(args=args).cuda() # fid stat if args.dataset.lower() == "cifar10": fid_stat = "fid_stat/fid_stats_cifar10_train.npz" elif args.dataset.lower() == "stl10": fid_stat = "fid_stat/stl10_train_unlabeled_fid_stats_48.npz" else: raise NotImplementedError(f"no fid stat for {args.dataset.lower()}") assert os.path.exists(fid_stat) # initial fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) # set writer logger.info(f"=> resuming from {args.load_path}") checkpoint_file = args.load_path assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) if "avg_gen_state_dict" in checkpoint: gen_net.load_state_dict(checkpoint["avg_gen_state_dict"]) epoch = checkpoint["epoch"] logger.info(f"=> loaded checkpoint {checkpoint_file} (epoch {epoch})") else: gen_net.load_state_dict(checkpoint) logger.info(f"=> loaded checkpoint {checkpoint_file}") logger.info(args) writer_dict = { "writer": SummaryWriter(args.path_helper["log_path"]), "valid_global_steps": 0, } inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict, clean_dir=False) logger.info(f"Inception score: {inception_score}, FID score: {fid_score}.")
def main(args): logging.info(args) if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) cudnn.benchmark = True model = get_network_func(args.norm_module) model.cuda() print('load checkpoint') assert os.path.exists(args.weight_path), print(f"Cannot find {args.weight_path}") checkpoint = torch.load(args.weight_path) model.load_state_dict(checkpoint['state_dict']) # dataset process test_dataset = datasets.CIFAR10(root=args.data, train=False, transform=transforms.Compose([ transforms.ToTensor() ]), download=True) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=2, pin_memory=True) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() # evaluate on test set # clean branch print(f"=>test on clean branch") test_tacc_clean, test_tloss_clean = validate(args, test_loader, model, criterion, 0) print(f"Test accuracy: {test_tacc_clean}, Test loss: {test_tloss_clean}") # test_atacc_clean, test_atloss_clean = validate_adv(args, test_loader, model, criterion, 0) # print(f"Adversarial test accuracy: {test_atacc_clean}, Adversarial test loss: {test_atloss_clean}") # adv branch print(f"=>test on adv branch") test_tacc_adv, test_tloss_adv = validate(args, test_loader, model, criterion, 1) print(f"Test accuracy: {test_tacc_adv}, Test loss: {test_tloss_adv}") test_atacc_adv, test_atloss_adv = validate_adv(args, test_loader, model, criterion, 1) print(f"Adversarial test accuracy: {test_atacc_adv}, Adversarial test loss: {test_atloss_adv}")
def login(): error = None if request.method == 'POST': username = request.form['username'] password = request.form['password'] completion = validate(username, password) if completion == False: error = 'Invalid Credentials. Please try again.' else: messages = json.dumps({"user": username}) session['messages'] = messages return redirect(url_for('search')) return render_template('login.html', error=error)
def main(args): assert args.exp_name logger, final_output_dir, _ = create_logger(args, args.exp_name, 'test') args.sample_path = final_output_dir torch.cuda.manual_seed(args.random_seed) # set tf env init_inception() # fid stat if args.dataset.lower() == 'cifar10': args.n_classes = 10 fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'imagenet': # TODO: Support intra-FID args.n_classes = 143 fid_stat = None else: raise NotImplementedError(f'no fid stat for {args.dataset.lower()}') if fid_stat: assert os.path.exists(fid_stat), f"{fid_stat} not found" # get network gen_net, _ = get_network_func(args) gen_net.cuda() # load checkpoint checkpoint_file = args.load_path assert os.path.exists(checkpoint_file), print( f"checkpoint file {checkpoint_file} not found.") logger.info("=> loading checkpoint '{}'".format(checkpoint_file)) checkpoint = torch.load(checkpoint_file, map_location=lambda storage, loc: storage) gen_net.load_state_dict(checkpoint) logger.info(f"=> loaded checkpoint '{checkpoint_file}' ") # evaluation torch.cuda.empty_cache() inception_score, fid_score = validate(args, fid_stat, gen_net, None) logger.info(f'Inception score: {inception_score}, FID score: {fid_score} ')
def login(): error = None if request.method == 'POST': username = request.form['username'] password = request.form['password'] completion = validate(username, password) if completion is False: error = 'Niepoprawny login lub hasło' else: session['username'] = request.form['username'] username = session['username'] if check_grupa(username) == 'admin': info = "Witaj" + " " + check_grupa(username)+"ie" flash(info) if check_grupa(username) == 'nauczyciel': info = "Witaj" + " " + check_grupa(username) + "u" flash(info) if check_grupa(username) == 'rodzic': info = "Witaj" + " " + check_grupa(username) + "u" flash(info) return render_template('base.html', error=error, info=username, grupa=check_grupa(username)) return render_template('login.html', error=error)
def main(): args = cfg.parse_args() torch.cuda.manual_seed(args.random_seed) # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # import network gen_net = eval('models.' + args.model + '.Generator')(args=args).cuda() dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda() # weight init def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv2d') != -1: if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format( args.init_type)) elif classname.find('BatchNorm2d') != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net.apply(weights_init) dis_net.apply(weights_init) # set optimizer gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # set up data_loader dataset = datasets.ImageDataset(args) train_loader = dataset.train # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz' else: raise NotImplementedError(f'no fid stat for {args.dataset.lower()}') assert os.path.exists(fid_stat) # epoch number for dis_net args.max_epoch = args.max_epoch * args.n_critic if args.max_iter: args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader)) # initial fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) gen_avg_param = copy_params(gen_net) start_epoch = 0 best_fid = 1e4 # set writer if args.load_path: print(f'=> resuming from {args.load_path}') assert os.path.exists(args.load_path) checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) start_epoch = checkpoint['epoch'] best_fid = checkpoint['best_fid'] gen_net.load_state_dict(checkpoint['gen_state_dict']) dis_net.load_state_dict(checkpoint['dis_state_dict']) gen_optimizer.load_state_dict(checkpoint['gen_optimizer']) dis_optimizer.load_state_dict(checkpoint['dis_optimizer']) avg_gen_net = deepcopy(gen_net) avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict']) gen_avg_param = copy_params(avg_gen_net) del avg_gen_net args.path_helper = checkpoint['path_helper'] logger = create_logger(args.path_helper['log_path']) logger.info( f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})') else: # create new log dir assert args.exp_name args.path_helper = set_log_dir('logs', args.exp_name) logger = create_logger(args.path_helper['log_path']) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } # train loop for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)), desc='total progress'): lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None train(args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers) if epoch and epoch % args.val_freq == 0 or epoch == int( args.max_epoch) - 1: backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict) logger.info( f'Inception score: {inception_score}, FID score: {fid_score} || @ epoch {epoch}.' ) load_params(gen_net, backup_param) if fid_score < best_fid: best_fid = fid_score is_best = True else: is_best = False else: is_best = False avg_gen_net = deepcopy(gen_net) load_params(avg_gen_net, gen_avg_param) save_checkpoint( { 'epoch': epoch + 1, 'model': args.model, 'gen_state_dict': gen_net.state_dict(), 'dis_state_dict': dis_net.state_dict(), 'avg_gen_state_dict': avg_gen_net.state_dict(), 'gen_optimizer': gen_optimizer.state_dict(), 'dis_optimizer': dis_optimizer.state_dict(), 'best_fid': best_fid, 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path']) del avg_gen_net
def getHistoricalData(country, indicator, initDate=None, endDate=None, output_type=None): """ Return historical information for specific country and indicator. ================================================================= Parameters: ----------- country: string or list. String to get data for one country. List of strings to get data for several countries. For example, country = ['United States', 'Australia']. indicator: string or list. String to get data for one category. List of strings to get data for several calendar events. For example, category = 'GDP Growth Rate' or category = ['Exports', 'Imports'] initDate: string with format: YYYY-MM-DD. For example: '2011-01-01' endDate: string with format: YYYY-MM-DD. output_type: string. 'dict'(default) for dictionary format output, 'raw' for list of dictionaries without any parsing. Notes ----- Without credentials only sample data will be provided. Example ------- getHistoricalData(country = 'United States', indicator = 'Imports', initDate = '2011-01-01', endDate = '2016-01-01') getHistoricalData(country = ['United States', 'United Kingdom'], indicator = ['Imports','Exports'], initDate = '2011-01-01', endDate = '2016-01-01') """ if type(country) is not str or type(indicator) is not str: linkAPI = paramCheck(country, indicator) else: linkAPI = 'http://api.tradingeconomics.com/historical/country/' + urllib.quote( country) + '/indicator/' + urllib.quote(indicator) if initDate == None and endDate == None: minDate = [ (datetime.now() - relativedelta(years=10)).strftime('%Y-%m-%d') ] linkAPI = fn.finalLink(linkAPI, minDate) if initDate == None and (endDate is not None): iDate = (datetime.now() - relativedelta(years=10)).strftime('%Y-%m-%d') try: fn.validate(endDate) except ValueError: raise DateError( 'Incorrect endDate format, should be YYYY-MM-DD or MM-DD-YYYY.' ) try: fn.validatePeriod(iDate, endDate) except ValueError: raise DateError('Incorrect time period.') param = [iDate, endDate] linkAPI = fn.finalLink(linkAPI, param) if (initDate is not None) and (endDate is not None): try: fn.validate(initDate) except ValueError: raise DateError( 'Incorrect initDate format, should be YYYY-MM-DD or MM-DD-YYYY.' ) try: fn.validate(endDate) except ValueError: raise DateError( 'Incorrect endDate format, should be YYYY-MM-DD or MM-DD-YYYY.' ) try: fn.validatePeriod(initDate, endDate) except ValueError: raise DateError('Invalid time period.') param = [initDate, endDate] linkAPI = fn.finalLink(linkAPI, param) if (initDate is not None) and endDate == None: try: fn.validate(initDate) except ValueError: raise DateError( 'Incorrect initDate format, should be YYYY-MM-DD or MM-DD-YYYY.' ) if initDate > str(date.today()): raise DateError('Initial date out of range.') myDate = [initDate] linkAPI = fn.finalLink(linkAPI, myDate) try: linkAPI = linkAPI + '?c=' + glob.apikey except AttributeError: raise LoginError('You need to do login before making any request') try: webResults = json.load(urllib.urlopen(linkAPI)) except ValueError: raise CredentialsError('Invalid credentials') if len(webResults) > 0: date = [d['DateTime'] for d in webResults] value = [d[u'Value'] for d in webResults] results = {'dates': date, 'values': value} if (type(country) == str and type(indicator) == str): results = parseData(results) else: results = multiParams(webResults) else: raise ParametersError('No data available for the provided parameters.') if output_type == None or output_type == 'dict': output = results elif output_type == 'raw': output = webResults else: raise ParametersError( 'output_type options : dict(defoult) for dictionary or raw for unparsed results.' ) return output
def getCalendarData(country = None, category = None, initDate = None, endDate = None, output_type = None): """ Return calendar events. =========================================================== Parameters: ----------- country: string or list. String to get data for one country. List of strings to get data for several countries. For example, country = ['United States', 'Australia']. category: string or list. String to get data for one category. List of strings to get data for several calendar events. For example, category = 'GDP Growth Rate' or category = ['Exports', 'Imports'] initDate: string with format: YYYY-MM-DD. For example: '2011-01-01' endDate: string with format: YYYY-MM-DD. output_type: string. 'dict'(default) for dictionary format output, 'df' for data frame, 'raw' for list of dictionaries without any parsing. Notes ----- All parameters are optional. When not supplying parameters, data for all countries and indicators will be provided. Example ------- getCalendarData(country = 'United States', category = 'Imports', initDate = '2011-01-01', endDate = '2016-01-01') getCalendarData(country = ['United States', 'India'], category = ['Imports','Exports'], initDate = '2011-01-01', endDate = '2016-01-01') """ if country == None and category == None: linkAPI = 'http://api.tradingeconomics.com/calendar' elif country == None and category != None: country_all = 'all' linkAPI = paramCheck(country_all, category) elif type(country) is str and type(category) is str: linkAPI = 'http://api.tradingeconomics.com/calendar/country/' + urllib.quote(country) + '/indicator/' + urllib.quote(category) else: linkAPI = paramCheck(country, category) if initDate == None and endDate == None: linkAPI = linkAPI elif endDate > str(datetime.now()): raise DateError ('End date could not be greater than actual date') else: try: fn.validate(initDate) except ValueError: raise DateError ('Incorrect initial date format, should be YYYY-MM-DD ') try: fn.validate(endDate) except ValueError: raise DateError ('Incorrect end date format, should be YYYY-MM-DD ') try: fn.validatePeriod(initDate, endDate) except ValueError: raise DateError ('Invalid time period.') param=[initDate, endDate] linkAPI = fn.finalLink(linkAPI, param) try: linkAPI = linkAPI + '?c=' + glob.apikey except AttributeError: raise LoginError('You need to do login before making any request') try: webResults = json.load(urllib.urlopen(linkAPI)) except ValueError: raise CredentialsError ('Invalid credentials') if len(webResults) > 0: names = ['date', 'country', 'category', 'event', 'reference', 'unit', 'source', 'actual', 'previous', 'forecast', 'teforecast'] names2 = ['Date', 'Country', 'Category', 'Event', 'Reference', 'Unit', 'Source', 'Actual', 'Previous', 'Forecast', 'TEForecast'] maindf = pd.DataFrame() for i in range(len(names)): names[i] = [d[names2[i]] for d in webResults] maindf = pd.concat([maindf, pd.DataFrame(names[i], columns = [names2[i]])], axis = 1) else: raise ParametersError ('No data available for the provided parameters.') if output_type == None or output_type =='dict': output = fn.out_type(maindf) elif output_type == 'df': output = maindf elif output_type == 'raw': output = webResults else: raise ParametersError ('output_type options : df for data frame, dict(defoult) for dictionary by country, raw for unparsed results.') return output
def main(): args = cfg.parse_args() random.seed(args.random_seed) torch.manual_seed(args.random_seed) torch.cuda.manual_seed(args.random_seed) np.random.seed(args.random_seed) torch.backends.cudnn.deterministic = False torch.backends.cudnn.benchmark = True os.environ['PYTHONHASHSEED'] = str(args.random_seed) # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # import network gen_net = eval('models.'+args.model+'.Generator')(args=args) dis_net = eval('models.'+args.model+'.Discriminator')(args=args) initial_gen_net_weight = torch.load(os.path.join(args.init_path, 'initial_gen_net.pth'), map_location="cpu") initial_dis_net_weight = torch.load(os.path.join(args.init_path, 'initial_dis_net.pth'), map_location="cpu") gen_net = gen_net.cuda() dis_net = dis_net.cuda() gen_net.load_state_dict(initial_gen_net_weight) dis_net.load_state_dict(initial_dis_net_weight) # set optimizer gen_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # set up data_loader dataset = datasets.ImageDataset(args) train_loader = dataset.train # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/fid_stats_stl10_train.npz' else: raise NotImplementedError('no fid stat for %s' % args.dataset.lower()) assert os.path.exists(fid_stat) # epoch number for dis_net args.max_epoch = args.max_epoch * args.n_critic if args.max_iter: args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader)) # initial fixed_z = torch.cuda.FloatTensor(np.random.normal(0, 1, (25, args.latent_dim))) gen_avg_param = copy_params(gen_net) start_epoch = 0 best_fid = 1e4 # set writer if args.load_path: print('=> resuming from %s' % args.load_path) assert os.path.exists(args.load_path) checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) start_epoch = checkpoint['epoch'] best_fid = checkpoint['best_fid'] gen_net.load_state_dict(checkpoint['gen_state_dict']) dis_net.load_state_dict(checkpoint['dis_state_dict']) gen_optimizer.load_state_dict(checkpoint['gen_optimizer']) dis_optimizer.load_state_dict(checkpoint['dis_optimizer']) avg_gen_net = deepcopy(gen_net) avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict']) gen_avg_param = copy_params(avg_gen_net) del avg_gen_net args.path_helper = checkpoint['path_helper'] logger = create_logger(args.path_helper['log_path']) logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch)) else: # create new log dir assert args.exp_name args.path_helper = set_log_dir('logs', args.exp_name) logger = create_logger(args.path_helper['log_path']) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } # train loop switch = False for epoch in range(int(start_epoch), int(args.max_epoch)): lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None train(args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers) if epoch and epoch % args.val_freq == 0 or epoch == int(args.max_epoch)-1: backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict, epoch) logger.info('Inception score: %.4f, FID score: %.4f || @ epoch %d.' % (inception_score, fid_score, epoch)) load_params(gen_net, backup_param) if fid_score < best_fid: best_fid = fid_score is_best = True else: is_best = False else: is_best = False avg_gen_net = deepcopy(gen_net) load_params(avg_gen_net, gen_avg_param) save_checkpoint({ 'epoch': epoch + 1, 'model': args.model, 'gen_state_dict': gen_net.state_dict(), 'dis_state_dict': dis_net.state_dict(), 'avg_gen_state_dict': avg_gen_net.state_dict(), 'gen_optimizer': gen_optimizer.state_dict(), 'dis_optimizer': dis_optimizer.state_dict(), 'best_fid': best_fid, 'path_helper': args.path_helper, 'seed': args.random_seed }, is_best, args.path_helper['ckpt_path']) del avg_gen_net
def validate(task_info, data_type): """ check if the data_type can be correctly calculated in the task :param task_info: task_info dict, e.g. { 'input_data_type_id': 121, 'time-range': '1m', 'functions': { '1': { 'name': 'filter', 'target': 'status', 'conditions': [ { 'target': 'cost', 'operator': 'bt', 'param1': 100, 'param2': 1000 }, {...} ] }, '2': { 'name': 'average', 'target': 'cost', 'tag': 'avg_cost', 'group_by': ['function', 'status'] } } } :param data_type: data_type dict, e.g. {'userid': 'string', 'function': 'string', 'status': 'integer', 'cost': 'real'} :return: the result dict, in which 'result' is a boolean indicate if it is validate, if not, the 'info' would contain some information of the reason """ result = {'succ': True} if not isinstance(task_info, dict): result['succ'] = False result['info'] = 'task desc should be dict' return result # check title if len(task_info['title']) == 0: result['succ'] = False result['info'] = 'title is empty.' return result # check basic attributes for key in task_structure: if key not in task_info: result['succ'] = False result['info'] = 'key not found: {key}'.format(key=key) return result if not task_structure[key](task_info[key]): result['succ'] = False result['info'] = 'type error: {key}'.format(key=key) return result # check time range pattern = re.compile("\A[1-9][0-9]{0,2}[dhms]\Z") if not pattern.match(task_info['time_range']): result['succ'] = False result['info'] = 'time range format not valid'.format( func_name=task_info['name']) return result # check each function temp_type = data_type for i in xrange(0, len(task_info['functions'])): if str(i + 1) not in task_info['functions']: result['succ'] = False result['info'] = 'function key error: expected {key}'.format( key=i + 1) return result temp_result = functions.validate(task_info['functions'][str(i + 1)], temp_type) if temp_result['succ']: temp_type = temp_result['result_type'] else: result['succ'] = False result['info'] = temp_result['info'] result['result_type'] = temp_type return result
def main(): args = cfg.parse_args() random.seed(args.random_seed) torch.manual_seed(args.random_seed) torch.cuda.manual_seed(args.random_seed) np.random.seed(args.random_seed) # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # import netwo # weight init def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv2d') != -1: if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format( args.init_type)) elif classname.find('BatchNorm2d') != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net = eval('models.' + args.model + '.Generator')(args=args).cuda() dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda() gen_net.apply(weights_init) dis_net.apply(weights_init) avg_gen_net = deepcopy(gen_net) initial_gen_net_weight = torch.load(os.path.join(args.init_path, 'initial_gen_net.pth'), map_location="cpu") initial_dis_net_weight = torch.load(os.path.join(args.init_path, 'initial_dis_net.pth'), map_location="cpu") assert id(initial_dis_net_weight) != id(dis_net.state_dict()) assert id(initial_gen_net_weight) != id(gen_net.state_dict()) # set optimizer gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # set up data_loader dataset = datasets.ImageDataset(args) train_loader = dataset.train # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/fid_stats_stl10_train.npz' else: raise NotImplementedError('no fid stat for %s' % args.dataset.lower()) assert os.path.exists(fid_stat) # epoch number for dis_net args.max_epoch = args.max_epoch * args.n_critic if args.max_iter: args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader)) # initial fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) start_epoch = 0 best_fid = 1e4 print('=> resuming from %s' % args.load_path) assert os.path.exists(args.load_path) checkpoint_file = args.load_path assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) pruning_generate(gen_net, checkpoint['gen_state_dict']) dis_net.load_state_dict(checkpoint['dis_state_dict']) total = 0 total_nonzero = 0 for m in dis_net.modules(): if isinstance(m, nn.Conv2d): total += m.weight_orig.data.numel() mask = m.weight_orig.data.abs().clone().gt(0).float().cuda() total_nonzero += torch.sum(mask) conv_weights = torch.zeros(total) index = 0 for m in dis_net.modules(): if isinstance(m, nn.Conv2d): size = m.weight_orig.data.numel() conv_weights[index:( index + size)] = m.weight_orig.data.view(-1).abs().clone() index += size y, i = torch.sort(conv_weights) # thre_index = int(total * args.percent) # only care about the non zero weights # e.g: total = 100, total_nonzero = 80, percent = 0.2, thre_index = 36, that means keep 64 thre_index = total - total_nonzero thre = y[int(thre_index)] pruned = 0 print('Pruning threshold: {}'.format(thre)) zero_flag = False masks = OrderedDict() for k, m in enumerate(dis_net.modules()): if isinstance(m, nn.Conv2d): weight_copy = m.weight_orig.data.abs().clone() mask = weight_copy.gt(thre).float() masks[k] = mask pruned = pruned + mask.numel() - torch.sum(mask) m.weight_orig.data.mul_(mask) if int(torch.sum(mask)) == 0: zero_flag = True print( 'layer index: {:d} \t total params: {:d} \t remaining params: {:d}' .format(k, mask.numel(), int(torch.sum(mask)))) print('Total conv params: {}, Pruned conv params: {}, Pruned ratio: {}'. format(total, pruned, pruned / total)) pruning_generate(avg_gen_net, checkpoint['gen_state_dict']) see_remain_rate(gen_net) if not args.finetune_G: gen_weight = gen_net.state_dict() gen_orig_weight = rewind_weight(initial_gen_net_weight, gen_weight.keys()) gen_weight.update(gen_orig_weight) gen_net.load_state_dict(gen_weight) gen_avg_param = copy_params(gen_net) if args.finetune_D: dis_net.load_state_dict(checkpoint['dis_state_dict']) else: dis_net.load_state_dict(initial_dis_net_weight) for k, m in enumerate(dis_net.modules()): if isinstance(m, nn.Conv2d): m.weight_orig.data.mul_(masks[k]) orig_dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda() orig_dis_net.load_state_dict(checkpoint['dis_state_dict']) orig_dis_net.eval() args.path_helper = set_log_dir('logs', args.exp_name + "_{}".format(args.percent)) logger = create_logger(args.path_helper['log_path']) #logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch)) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } # train loop for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)), desc='total progress'): lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None see_remain_rate(gen_net) see_remain_rate_orig(dis_net) if not args.use_kd_D: train_with_mask(args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, masks, lr_schedulers) else: train_with_mask_kd(args, gen_net, dis_net, orig_dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, masks, lr_schedulers) if epoch and epoch % args.val_freq == 0 or epoch == int( args.max_epoch) - 1: backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict, epoch) logger.info( 'Inception score: %.4f, FID score: %.4f || @ epoch %d.' % (inception_score, fid_score, epoch)) load_params(gen_net, backup_param) if fid_score < best_fid: best_fid = fid_score is_best = True else: is_best = False else: is_best = False avg_gen_net.load_state_dict(gen_net.state_dict()) load_params(avg_gen_net, gen_avg_param) save_checkpoint( { 'epoch': epoch + 1, 'model': args.model, 'gen_state_dict': gen_net.state_dict(), 'dis_state_dict': dis_net.state_dict(), 'avg_gen_state_dict': avg_gen_net.state_dict(), 'gen_optimizer': gen_optimizer.state_dict(), 'dis_optimizer': dis_optimizer.state_dict(), 'best_fid': best_fid, 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path'])
def main(): functions.clear() if not functions.exists(): try: functions.intro() except KeyboardInterrupt: functions.clear() import sys sys.exit(0) functions.clear() try: password = getpass.getpass("Password: "******"Invalid password") import sys sys.exit(1) while True: functions.clear() print("Vault") print( "Use Ctrl+C to return back to this menu, or use it here to exit.\n" ) try: action = input( "1: Encrypt a file\n2: Download and encrypt a file from a URL\n3: Decrypt a file\n4: Change password\nSelect an option: " ) except: break action = action.strip().lstrip() try: action = int(action) except: continue if action not in [1, 2, 3, 4]: continue if action == 1: functions.clear() try: path = input("Please specify a file to encrypt: ") except KeyboardInterrupt: continue if functions.validate(path) and not path.endswith(".png"): functions.clear() try: response = input( "This file may be a valid image file. Convert to .png (y/n)? " ) except KeyboardInterrupt: continue if response.lower().lstrip().strip() == "y": temp = path path = functions.convert(path) delete = input("Delete original file (y/n)? ") if delete.lower().lstrip().strip() == "y": if not windows: temp = temp.replace("\\", "") try: directory = os.path.dirname(os.path.realpath(temp)) os.chdir(directory) os.remove(temp) except: pass functions.clear() try: save = functions.encodepath(path) if save == None: continue except Exception as e: print(str(e)) try: input("Invalid file path. Press enter to continue. ") except KeyboardInterrupt: continue continue else: try: input( "File encrypted.\nEncrypted file path: {}\nPress enter to continue. " .format(save)) except KeyboardInterrupt: continue continue elif action == 2: functions.clear() try: url = input("Please specify a file URL to download: ") except KeyboardInterrupt: continue name = os.path.split(url)[-1] base = os.path.dirname(os.path.realpath(__file__)) path = os.path.join(base, name) functions.clear() try: r = requests.get(url, stream=True) with open(path, 'wb') as out_file: shutil.copyfileobj(r.raw, out_file) except: try: input("Invalid URL: {}\nPress enter to continue. ".format( url)) except KeyboardInterrupt: continue continue if functions.validate(path) and not path.endswith(".png"): functions.clear() try: response = input( "This file may be a valid image file. Convert to .png (y/n)? " ) except KeyboardInterrupt: continue if response.lower().lstrip().strip() == "y": temp = path path = functions.convert(path) delete = input("Delete original file (y/n)? ") if delete.lower().lstrip().strip() == "y": if not windows: temp = temp.replace("\\", "") try: directory = os.path.dirname(os.path.realpath(temp)) os.chdir(directory) os.remove(temp) except: pass functions.clear() try: save = functions.encodepath(path) except: try: input("Invalid file path. Press enter to continue. ") except KeyboardInterrupt: continue continue else: try: input( "File encrypted.\nEncrypted file path: {}\nPress enter to continue. " .format(save)) except KeyboardInterrupt: continue continue elif action == 3: functions.clear() try: path = input("Please specify a .enc file to decrypt: ") except KeyboardInterrupt: continue functions.clear() try: save = functions.decodepath(path) except Exception as e: print(str(e)) try: input("Invalid file path. Press enter to continue. ") except KeyboardInterrupt: continue continue else: try: input( "File decrypted.\nDecrypted file path: {}\nPress enter to continue. " .format(save)) except KeyboardInterrupt: continue continue elif action == 4: try: functions.clear() current = input("Current password: "******"Passwords do not match. Press enter to continue. ") continue new = input("New password: "******"You will need to decrypt all files with your old password. Your new password will not be able to decrypt already encrypted files.\nRetype your password to confirm: " ) if confirm != new: input("Passwords do not match. Press enter to continue. ") continue functions.writepw(new) functions.clear() input("Password changed to: {}.\nPress enter to continue. ". format(new)) except KeyboardInterrupt: continue functions.clear()
def main(): args = cfg_train.parse_args() torch.cuda.manual_seed(args.random_seed) # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # import network # gen_net = eval('models.' + args.gen_model + '.' + args.gen)(args=args).cuda() genotype_gen = eval('genotypes.%s' % args.arch_gen) gen_net = eval('models.' + args.gen_model + '.' + args.gen)( args, genotype_gen).cuda() # gen_net = eval('models.' + args.gen_model + '.' + args.gen)(args = args).cuda() if 'Discriminator' not in args.dis: genotype_dis = eval('genotypes.%s' % args.arch_dis) dis_net = eval('models.' + args.dis_model + '.' + args.dis)( args, genotype_dis).cuda() else: dis_net = eval('models.' + args.dis_model + '.' + args.dis)(args=args).cuda() # weight init def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv2d') != -1: if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format( args.init_type)) elif classname.find('BatchNorm2d') != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net.apply(weights_init) dis_net.apply(weights_init) # set up data_loader dataset = datasets.ImageDataset(args) train_loader = dataset.train val_loader = dataset.valid # set optimizer gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, args.g_lr * 0.01, 260 * len(train_loader), args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, args.d_lr * 0.01, 260 * len(train_loader), args.max_iter * args.n_critic) # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz' elif args.dataset.lower() == 'mnist': fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz' else: raise NotImplementedError(f'no fid stat for {args.dataset.lower()}') assert os.path.exists(fid_stat) # epoch number for dis_net args.max_epoch = args.max_epoch * args.n_critic if args.max_iter: args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader)) # initial fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) fixed_z_sample = torch.cuda.FloatTensor( np.random.normal(0, 1, (args.eval_batch_size, args.latent_dim))) gen_avg_param = copy_params(gen_net) start_epoch = 0 best_fid = 1e4 best_fid_epoch = 0 is_with_fid = 0 std_with_fid = 0. best_is = 0 best_is_epoch = 0 fid_with_is = 0 best_dts = 0 # set writer if args.load_path: print(f'=> resuming from {args.load_path}') assert os.path.exists(args.load_path) checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) start_epoch = checkpoint['epoch'] best_fid = checkpoint['best_fid'] gen_net.load_state_dict(checkpoint['gen_state_dict']) dis_net.load_state_dict(checkpoint['dis_state_dict']) gen_optimizer.load_state_dict(checkpoint['gen_optimizer']) dis_optimizer.load_state_dict(checkpoint['dis_optimizer']) avg_gen_net = deepcopy(gen_net) avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict']) gen_avg_param = copy_params(avg_gen_net) del avg_gen_net args.path_helper = checkpoint['path_helper'] logger = create_logger(args.path_helper['log_path']) logger.info( f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})') else: # create new log dir assert args.exp_name args.path_helper = set_log_dir('logs', args.exp_name) logger = create_logger(args.path_helper['log_path']) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } # calculate the FLOPs and param count of G input = torch.randn(args.gen_batch_size, args.latent_dim).cuda() flops, params = profile(gen_net, inputs=(input, )) flops, params = clever_format([flops, params], "%.3f") logger.info('FLOPs is {}, param count is {}'.format(flops, params)) # train loop dg_list = [] worst_lr = 1e-5 for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)), desc='total progress'): lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None train(args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, args.consistent, lr_schedulers) if epoch and epoch % args.val_freq == 0 or epoch == int( args.max_epoch) - 1: backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) inception_score, std, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict, args.path_helper, search=False) logger.info( f'Inception score: {inception_score}, FID score: {fid_score}+-{std} || @ epoch {epoch}.' ) load_params(gen_net, backup_param) if fid_score < best_fid: best_fid = fid_score best_fid_epoch = epoch is_with_fid = inception_score std_with_fid = std is_best = True else: is_best = False if inception_score > best_is: best_is = inception_score best_std = std fid_with_is = fid_score best_is_epoch = epoch else: is_best = False # save generated images if epoch % args.image_every == 0: gen_noise = torch.cuda.FloatTensor( np.random.normal(0, 1, (args.eval_batch_size, args.latent_dim))) # gen_images = gen_net(fixed_z_sample) # gen_images = gen_images.reshape(args.eval_batch_size, 32, 32, 3) # gen_images = gen_images.cpu().detach() gen_images = gen_net(fixed_z_sample).mul_(127.5).add_( 127.5).clamp_(0.0, 255.0).permute(0, 2, 3, 1).to('cpu', torch.uint8).numpy() fig = plt.figure() grid = ImageGrid(fig, 111, nrows_ncols=(10, 10), axes_pad=0) for x in range(args.eval_batch_size): grid[x].imshow(gen_images[x]) # cmap="gray") grid[x].set_xticks([]) grid[x].set_yticks([]) plt.savefig( os.path.join(args.path_helper['sample_path'], "epoch_{}.png".format(epoch))) plt.close() avg_gen_net = deepcopy(gen_net) # avg_gen_net = eval('models.'+args.gen_model+'.' + args.gen)(args, genotype_gen).cuda() # avg_gen_net = eval('models.' + args.gen_model + '.' + args.gen)(args=args).cuda() load_params(avg_gen_net, gen_avg_param) save_checkpoint( { 'epoch': epoch + 1, 'gen_model': args.gen_model, 'dis_model': args.dis_model, 'gen_state_dict': gen_net.state_dict(), 'dis_state_dict': dis_net.state_dict(), 'avg_gen_state_dict': avg_gen_net.state_dict(), 'gen_optimizer': gen_optimizer.state_dict(), 'dis_optimizer': dis_optimizer.state_dict(), 'best_fid': best_fid, 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path']) del avg_gen_net logger.info( 'best_is is {}+-{}@{} epoch, fid is {}, best_fid is {}@{}, is is {}+-{}' .format(best_is, best_std, best_is_epoch, fid_with_is, best_fid, best_fid_epoch, is_with_fid, std_with_fid))
def main(): args = cfg.parse_args() torch.cuda.manual_seed(args.random_seed) torch.cuda.manual_seed_all(args.random_seed) np.random.seed(args.random_seed) random.seed(args.random_seed) torch.backends.cudnn.deterministic = True # import network # args.gen_model is TransGAN_8_8_1 for example gen_net = eval('models.'+args.gen_model+'.Generator')(args=args).cuda() dis_net = eval('models.'+args.dis_model+'.Discriminator')(args=args).cuda() gen_net.set_arch(args.arch, cur_stage=2) print("The shit!") # weight init: Xavier Uniform def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv2d') != -1: if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format(args.init_type)) elif classname.find('BatchNorm2d') != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net.apply(weights_init) dis_net.apply(weights_init) gpu_ids = [i for i in range(int(torch.cuda.device_count()))] gen_net = torch.nn.DataParallel(gen_net.to("cuda:0"), device_ids=gpu_ids) dis_net = torch.nn.DataParallel(dis_net.to("cuda:0"), device_ids=gpu_ids) # print(gen_net.module.cur_stage) if args.optimizer == "adam": gen_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) elif args.optimizer == "adamw": gen_optimizer = AdamW(filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, weight_decay=args.wd) dis_optimizer = AdamW(filter(lambda p: p.requires_grad, dis_net.parameters()), args.g_lr, weight_decay=args.wd) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz' elif args.fid_stat is not None: fid_stat = args.fid_stat else: raise NotImplementedError # (f"no fid stat for %s"%args.dataset.lower()") assert os.path.exists(fid_stat) dataset = datasets.ImageDataset(args, cur_img_size=8) train_loader = dataset.train writer=SummaryWriter() writer_dict = {'writer':writer} writer_dict["train_global_steps"]=0 writer_dict["valid_global_steps"]=0 best = 1e4 for epoch in range(args.max_epoch): train(args, gen_net = gen_net, dis_net = dis_net, gen_optimizer = gen_optimizer, dis_optimizer = dis_optimizer, gen_avg_param = None, train_loader = train_loader, epoch = epoch, writer_dict = writer_dict, fixed_z = None, schedulers=[gen_scheduler, dis_scheduler]) checkpoint = {'epoch':epoch, 'best_fid':best} checkpoint['gen_state_dict'] = gen_net.state_dict() checkpoint['dis_state_dict'] = dis_net.state_dict() score = validate(args, None, fid_stat, epoch, gen_net, writer_dict, clean_dir=True) # print these scores, is it really the latest print(f'FID score: {score} - best ID score: {best} || @ epoch {epoch}.') if epoch == 0 or epoch > 30: if score < best: save_checkpoint(checkpoint, is_best=(score<best), output_dir=args.output_dir) print("Saved Latest Model!") best = score checkpoint = {'epoch':epoch, 'best_fid':best} checkpoint['gen_state_dict'] = gen_net.state_dict() checkpoint['dis_state_dict'] = dis_net.state_dict() score = validate(args, None, fid_stat, epoch, gen_net, writer_dict, clean_dir=True) save_checkpoint(checkpoint, is_best=(score<best), output_dir=args.output_dir)
def validate(task_info, data_type): """ check if the data_type can be correctly calculated in the task :param task_info: task_info dict, e.g. { 'input_data_type_id': 121, 'time-range': '1m', 'functions': { '1': { 'name': 'filter', 'target': 'status', 'conditions': [ { 'target': 'cost', 'operator': 'bt', 'param1': 100, 'param2': 1000 }, {...} ] }, '2': { 'name': 'average', 'target': 'cost', 'tag': 'avg_cost', 'group_by': ['function', 'status'] } } } :param data_type: data_type dict, e.g. {'userid': 'string', 'function': 'string', 'status': 'integer', 'cost': 'real'} :return: the result dict, in which 'result' is a boolean indicate if it is validate, if not, the 'info' would contain some information of the reason """ result = { 'succ': True } if not isinstance(task_info, dict): result['succ'] = False result['info'] = 'task desc should be dict' return result # check title if len(task_info['title']) == 0: result['succ'] = False result['info'] = 'title is empty.' return result # check basic attributes for key in task_structure: if key not in task_info: result['succ'] = False result['info'] = 'key not found: {key}'.format(key=key) return result if not task_structure[key](task_info[key]): result['succ'] = False result['info'] = 'type error: {key}'.format(key=key) return result # check time range pattern = re.compile("\A[1-9][0-9]{0,2}[dhms]\Z") if not pattern.match(task_info['time_range']): result['succ'] = False result['info'] = 'time range format not valid'.format( func_name=task_info['name']) return result # check each function temp_type = data_type for i in xrange(0, len(task_info['functions'])): if str(i + 1) not in task_info['functions']: result['succ'] = False result['info'] = 'function key error: expected {key}'.format(key=i + 1) return result temp_result = functions.validate(task_info['functions'][str(i + 1)], temp_type) if temp_result['succ']: temp_type = temp_result['result_type'] else: result['succ'] = False result['info'] = temp_result['info'] result['result_type'] = temp_type return result
def main(): args = cfg.parse_args() random.seed(args.random_seed) torch.manual_seed(args.random_seed) torch.cuda.manual_seed(args.random_seed) # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # weight init def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv2d') != -1: if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format( args.init_type)) elif classname.find('BatchNorm2d') != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net = Generator(bottom_width=args.bottom_width, gf_dim=args.gf_dim, latent_dim=args.latent_dim).cuda() dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda() gen_net.apply(weights_init) dis_net.apply(weights_init) initial_gen_net_weight = torch.load(os.path.join(args.init_path, 'initial_gen_net.pth'), map_location="cpu") initial_dis_net_weight = torch.load(os.path.join(args.init_path, 'initial_dis_net.pth'), map_location="cpu") os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu exp_str = args.dir args.load_path = os.path.join('output', exp_str, 'pth', 'epoch{}.pth'.format(args.load_epoch)) # state dict: assert os.path.exists(args.load_path) checkpoint = torch.load(args.load_path) print('=> loaded checkpoint %s' % args.load_path) state_dict = checkpoint['generator'] gen_net = load_subnet(args, state_dict, initial_gen_net_weight).cuda() avg_gen_net = deepcopy(gen_net) # set optimizer gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # set up data_loader dataset = datasets.ImageDataset(args) train_loader = dataset.train # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' else: raise NotImplementedError('no fid stat for %s' % args.dataset.lower()) assert os.path.exists(fid_stat) # epoch number for dis_net args.max_epoch = args.max_epoch * args.n_critic if args.max_iter: args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader)) # initial np.random.seed(args.random_seed) fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) start_epoch = 0 best_fid = 1e4 args.path_helper = set_log_dir('logs', args.exp_name) logger = create_logger(args.path_helper['log_path']) #logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch)) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } gen_avg_param = copy_params(gen_net) # train loop for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)), desc='total progress'): lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None train(args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers) if epoch and epoch % args.val_freq == 0 or epoch == int( args.max_epoch) - 1: backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict) logger.info( 'Inception score: %.4f, FID score: %.4f || @ epoch %d.' % (inception_score, fid_score, epoch)) load_params(gen_net, backup_param) if fid_score < best_fid: best_fid = fid_score is_best = True else: is_best = False else: is_best = False avg_gen_net.load_state_dict(gen_net.state_dict()) load_params(avg_gen_net, gen_avg_param) save_checkpoint( { 'epoch': epoch + 1, 'model': args.model, 'gen_state_dict': gen_net.state_dict(), 'dis_state_dict': dis_net.state_dict(), 'avg_gen_state_dict': avg_gen_net.state_dict(), 'gen_optimizer': gen_optimizer.state_dict(), 'dis_optimizer': dis_optimizer.state_dict(), 'best_fid': best_fid, 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path'])
def main(): args = cfg.parse_args() torch.cuda.manual_seed(args.random_seed) torch.cuda.manual_seed_all(args.random_seed) np.random.seed(args.random_seed) random.seed(args.random_seed) torch.backends.cudnn.deterministic = True # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # epoch number for dis_net dataset = datasets.ImageDataset(args, cur_img_size=8) train_loader = dataset.train if args.max_iter: args.max_epoch = np.ceil(args.max_iter / len(train_loader)) else: args.max_iter = args.max_epoch * len(train_loader) args.max_epoch = args.max_epoch * args.n_critic # import network gen_net = eval('models.' + args.gen_model + '.Generator')(args=args).cuda() dis_net = eval('models.' + args.dis_model + '.Discriminator')(args=args).cuda() gen_net.set_arch(args.arch, cur_stage=2) # weight init def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv2d') != -1: if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform_(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format( args.init_type)) elif classname.find('BatchNorm2d') != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net.apply(weights_init) dis_net.apply(weights_init) gpu_ids = [i for i in range(int(torch.cuda.device_count()))] gen_net = torch.nn.DataParallel(gen_net.to("cuda:0"), device_ids=gpu_ids) dis_net = torch.nn.DataParallel(dis_net.to("cuda:0"), device_ids=gpu_ids) gen_net.module.cur_stage = 0 dis_net.module.cur_stage = 0 gen_net.module.alpha = 1. dis_net.module.alpha = 1. # set optimizer if args.optimizer == "adam": gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) elif args.optimizer == "adamw": gen_optimizer = AdamW(filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, weight_decay=args.wd) dis_optimizer = AdamW(filter(lambda p: p.requires_grad, dis_net.parameters()), args.g_lr, weight_decay=args.wd) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz' elif args.fid_stat is not None: fid_stat = args.fid_stat else: raise NotImplementedError(f'no fid stat for {args.dataset.lower()}') assert os.path.exists(fid_stat) # initial fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (64, args.latent_dim))) gen_avg_param = copy_params(gen_net) start_epoch = 0 best_fid = 1e4 # set writer if args.load_path: print(f'=> resuming from {args.load_path}') assert os.path.exists(args.load_path) checkpoint_file = os.path.join(args.load_path) assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) start_epoch = checkpoint['epoch'] best_fid = checkpoint['best_fid'] gen_net.load_state_dict(checkpoint['gen_state_dict']) dis_net.load_state_dict(checkpoint['dis_state_dict']) gen_optimizer.load_state_dict(checkpoint['gen_optimizer']) dis_optimizer.load_state_dict(checkpoint['dis_optimizer']) # avg_gen_net = deepcopy(gen_net) # avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict']) gen_avg_param = checkpoint['gen_avg_param'] # del avg_gen_net cur_stage = cur_stages(start_epoch, args) gen_net.module.cur_stage = cur_stage dis_net.module.cur_stage = cur_stage gen_net.module.alpha = 1. dis_net.module.alpha = 1. args.path_helper = checkpoint['path_helper'] else: # create new log dir assert args.exp_name args.path_helper = set_log_dir('logs', args.exp_name) logger = create_logger(args.path_helper['log_path']) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } def return_states(): states = {} states['epoch'] = epoch states['best_fid'] = best_fid_score states['gen_state_dict'] = gen_net.state_dict() states['dis_state_dict'] = dis_net.state_dict() states['gen_optimizer'] = gen_optimizer.state_dict() states['dis_optimizer'] = dis_optimizer.state_dict() states['gen_avg_param'] = gen_avg_param states['path_helper'] = args.path_helper return states # train loop for epoch in range(start_epoch + 1, args.max_epoch): train( args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, fixed_z, ) backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) fid_score = validate( args, fixed_z, fid_stat, epoch, gen_net, writer_dict, ) logger.info(f'FID score: {fid_score} || @ epoch {epoch}.') load_params(gen_net, backup_param) is_best = False if epoch == 1 or fid_score < best_fid_score: best_fid_score = fid_score is_best = True if is_best or epoch % 1 == 0: states = return_states() save_checkpoint(states, is_best, args.path_helper['ckpt_path'], filename=f'checkpoint_epoch_{epoch}.pth')
def main(args): logging.info(args) writer = SummaryWriter(args.save_dir) best_prec1, best_ata, cln_best_prec1 = 0, 0, 0 if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) cudnn.benchmark = True setup_seed(args.seed) model = get_network_func(args.norm_module) model.cuda() start_epoch = 0 if args.resume: print('resume from checkpoint') checkpoint = torch.load(os.path.join(args.save_dir, 'model.pt')) start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) # dataset process train_datasets = datasets.CIFAR10(root=args.data, train=True, transform=transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, 4), transforms.ToTensor() ]), download=True) test_dataset = datasets.CIFAR10(root=args.data, train=False, transform=transforms.Compose( [transforms.ToTensor()]), download=True) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=2, pin_memory=True) valid_size = 0.1 indices = list(range(len(train_datasets))) split = int(np.floor(valid_size * len(train_datasets))) np.random.seed(args.seed) np.random.shuffle(indices) train_idx, valid_idx = indices[split:], indices[:split] train_sampler = SubsetRandomSampler(train_idx) valid_sampler = SubsetRandomSampler(valid_idx) train_loader = torch.utils.data.DataLoader(train_datasets, batch_size=args.batch_size, sampler=train_sampler) val_loader = torch.utils.data.DataLoader(train_datasets, batch_size=args.batch_size, sampler=valid_sampler) decreasing_lr = list(map(int, args.decreasing_lr.split(','))) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=decreasing_lr, gamma=0.1) print('starting adv training') all_result = {} train_acc = [] ta0 = [] ata0 = [] ta1 = [] ata1 = [] test_ta0 = [] test_ata0 = [] test_ta1 = [] test_ata1 = [] if os.path.exists(args.save_dir) is not True: os.mkdir(args.save_dir) for epoch in tqdm(range(args.epochs)): if epoch < start_epoch: scheduler.step() continue print(optimizer.state_dict()['param_groups'][0]['lr']) acc, train_loss = train(args, train_loader, model, criterion, optimizer, epoch) writer.add_scalar('acc/train_acc', acc, epoch) writer.add_scalar('loss/train_loss', train_loss, epoch) # evaluate on validation set tacc_clean, tloss_clean = validate(args, val_loader, model, criterion, 0) writer.add_scalar('acc/tacc_clean', tacc_clean, epoch) writer.add_scalar('loss/tloss_clean', tloss_clean, epoch) atacc_clean, atloss_clean = validate_adv(args, val_loader, model, criterion, 0) writer.add_scalar('acc/atacc_clean', atacc_clean, epoch) writer.add_scalar('loss/atloss_clean', atloss_clean, epoch) tacc_adv, tloss_adv = validate(args, val_loader, model, criterion, 1) writer.add_scalar('acc/tacc_adv', tacc_adv, epoch) writer.add_scalar('loss/tloss_adv', tloss_adv, epoch) atacc_adv, atloss_adv = validate_adv(args, val_loader, model, criterion, 1) writer.add_scalar('acc/atacc_adv', atacc_adv, epoch) writer.add_scalar('loss/atloss_adv', atloss_adv, epoch) # evaluate on test set # clean branch test_tacc_clean, test_tloss_clean = validate(args, test_loader, model, criterion, 0) writer.add_scalar('acc/test_tacc_clean', test_tacc_clean, epoch) writer.add_scalar('loss/test_tloss_clean', test_tloss_clean, epoch) test_atacc_clean, test_atloss_clean = validate_adv( args, test_loader, model, criterion, 0) writer.add_scalar('acc/test_atacc_clean', test_atacc_clean, epoch) writer.add_scalar('loss/test_atloss_clean', test_atloss_clean, epoch) # adv branch test_tacc_adv, test_tloss_adv = validate(args, test_loader, model, criterion, 1) writer.add_scalar('acc/test_tacc_adv', test_tacc_adv, epoch) writer.add_scalar('loss/test_tloss_adv', test_tloss_adv, epoch) test_atacc_adv, test_atloss_adv = validate_adv(args, test_loader, model, criterion, 1) writer.add_scalar('acc/test_atacc_adv', test_atacc_adv, epoch) writer.add_scalar('loss/test_atloss_adv', test_atloss_adv, epoch) scheduler.step() train_acc.append(acc) ta0.append(tacc_clean) ata0.append(atacc_clean) ta1.append(tacc_adv) ata1.append(atacc_adv) test_ta0.append(test_tacc_clean) test_ata0.append(test_atacc_clean) test_ta1.append(test_tacc_adv) test_ata1.append(test_atacc_adv) # remember best prec@1 and save checkpoint is_best = tacc_adv > best_prec1 best_prec1 = max(tacc_adv, best_prec1) ata_is_best = atacc_adv > best_ata best_ata = max(atacc_adv, best_ata) cln_is_best = tacc_clean > cln_best_prec1 cln_best_prec1 = max(tacc_clean, cln_best_prec1) if is_best: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'ata_best_prec1': best_ata, }, is_best, filename=os.path.join(args.save_dir, 'best_model.pt')) if cln_is_best: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'ata_best_prec1': best_ata, }, is_best, filename=os.path.join(args.save_dir, 'clean_best_model.pt')) if ata_is_best: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'ata_best_prec1': best_ata, }, is_best, filename=os.path.join(args.save_dir, 'ata_best_model.pt')) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'ata_best_prec1': best_ata, }, is_best, filename=os.path.join(args.save_dir, 'model.pt')) if epoch and epoch % 10 == 0: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'ata_best_prec1': best_ata, }, is_best, filename=os.path.join(args.save_dir, f'model_epoch{epoch}.pt')) all_result['train'] = train_acc all_result['test_ta0'] = test_ta0 all_result['test_ata0'] = test_ata0 all_result['test_ta1'] = test_ta1 all_result['test_ata1'] = test_ata1 all_result['ta0'] = ta0 all_result['ata0'] = ata0 all_result['ta1'] = ta1 all_result['ata1'] = ata1 pickle.dump(all_result, open(os.path.join(args.save_dir, 'result_c.pkl'), 'wb'))
def main(): args = cfg.parse_args() torch.cuda.manual_seed(args.random_seed) # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # import network gen_net = eval("models_search." + args.gen_model + ".Generator")(args=args).cuda() dis_net = eval("models_search." + args.dis_model + ".Discriminator")(args=args).cuda() gen_net.set_arch(args.arch, cur_stage=2) dis_net.cur_stage = 2 # weight init def weights_init(m): classname = m.__class__.__name__ if classname.find("Conv2d") != -1: if args.init_type == "normal": nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == "orth": nn.init.orthogonal_(m.weight.data) elif args.init_type == "xavier_uniform": nn.init.xavier_uniform(m.weight.data, 1.0) else: raise NotImplementedError("{} unknown inital type".format( args.init_type)) elif classname.find("BatchNorm2d") != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net.apply(weights_init) dis_net.apply(weights_init) # set optimizer gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2), ) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2), ) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # set up data_loader dataset = datasets.ImageDataset(args) train_loader = dataset.train # fid stat if args.dataset.lower() == "cifar10": fid_stat = "fid_stat/fid_stats_cifar10_train.npz" elif args.dataset.lower() == "stl10": fid_stat = "fid_stat/stl10_train_unlabeled_fid_stats_48.npz" else: raise NotImplementedError(f"no fid stat for {args.dataset.lower()}") assert os.path.exists(fid_stat) # epoch number for dis_net args.max_epoch = args.max_epoch * args.n_critic if args.max_iter: args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader)) # initial fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) gen_avg_param = copy_params(gen_net) start_epoch = 0 best_fid = 1e4 # set writer if args.load_path: print(f"=> resuming from {args.load_path}") assert os.path.exists(args.load_path) checkpoint_file = os.path.join(args.load_path, "Model", "checkpoint.pth") assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) start_epoch = checkpoint["epoch"] best_fid = checkpoint["best_fid"] gen_net.load_state_dict(checkpoint["gen_state_dict"]) dis_net.load_state_dict(checkpoint["dis_state_dict"]) gen_optimizer.load_state_dict(checkpoint["gen_optimizer"]) dis_optimizer.load_state_dict(checkpoint["dis_optimizer"]) avg_gen_net = deepcopy(gen_net) avg_gen_net.load_state_dict(checkpoint["avg_gen_state_dict"]) gen_avg_param = copy_params(avg_gen_net) del avg_gen_net args.path_helper = checkpoint["path_helper"] logger = create_logger(args.path_helper["log_path"]) logger.info( f"=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})") else: # create new log dir assert args.exp_name args.path_helper = set_log_dir("logs", args.exp_name) logger = create_logger(args.path_helper["log_path"]) logger.info(args) writer_dict = { "writer": SummaryWriter(args.path_helper["log_path"]), "train_global_steps": start_epoch * len(train_loader), "valid_global_steps": start_epoch // args.val_freq, } # train loop for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)), desc="total progress"): lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None train( args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers, ) if epoch and epoch % args.val_freq == 0 or epoch == int( args.max_epoch) - 1: backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict) logger.info( f"Inception score: {inception_score}, FID score: {fid_score} || @ epoch {epoch}." ) load_params(gen_net, backup_param) if fid_score < best_fid: best_fid = fid_score is_best = True else: is_best = False else: is_best = False avg_gen_net = deepcopy(gen_net) load_params(avg_gen_net, gen_avg_param) save_checkpoint( { "epoch": epoch + 1, "gen_model": args.gen_model, "dis_model": args.dis_model, "gen_state_dict": gen_net.state_dict(), "dis_state_dict": dis_net.state_dict(), "avg_gen_state_dict": avg_gen_net.state_dict(), "gen_optimizer": gen_optimizer.state_dict(), "dis_optimizer": dis_optimizer.state_dict(), "best_fid": best_fid, "path_helper": args.path_helper, }, is_best, args.path_helper["ckpt_path"], ) del avg_gen_net
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # Create tensorboard logger writer_dict = { 'writer': SummaryWriter(path_helper['log']), 'inner_steps': 0, 'val_steps': 0, 'valid_global_steps': 0 } # set tf env if args.eval: _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # fid_stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz' elif args.dataset.lower() == 'mnist': fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz' else: raise NotImplementedError(f'no fid stat for {args.dataset.lower()}') assert os.path.exists(fid_stat) # initial fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) FID_best = 1e+4 IS_best = 0. FID_best_epoch = 0 IS_best_epoch = 0 # build gen and dis gen = eval('model_search_gan.' + args.gen)(args) gen = gen.cuda() dis = eval('model_search_gan.' + args.dis)(args) dis = dis.cuda() logging.info("generator param size = %fMB", utils.count_parameters_in_MB(gen)) logging.info("discriminator param size = %fMB", utils.count_parameters_in_MB(dis)) if args.parallel: gen = nn.DataParallel(gen) dis = nn.DataParallel(dis) # resume training if args.load_path != '': gen.load_state_dict( torch.load( os.path.join(args.load_path, 'model', 'weights_gen_' + 'last' + '.pt'))) dis.load_state_dict( torch.load( os.path.join(args.load_path, 'model', 'weights_dis_' + 'last' + '.pt'))) # set optimizer for parameters W of gen and dis gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis.parameters()), args.d_lr, (args.beta1, args.beta2)) # set moving average parameters for generator gen_avg_param = copy_params(gen) img_size = 8 if args.grow else args.img_size train_transform, valid_transform = eval('utils.' + '_data_transforms_' + args.dataset + '_resize')(args, img_size) if args.dataset == 'cifar10': train_data = eval('dset.' + dataset[args.dataset])( root=args.data, train=True, download=True, transform=train_transform) elif args.dataset == 'stl10': train_data = eval('dset.' + dataset[args.dataset])( root=args.data, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.gen_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.gen_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=2) logging.info('length of train_queue is {}'.format(len(train_queue))) logging.info('length of valid_queue is {}'.format(len(valid_queue))) max_iter = len(train_queue) * args.epochs scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( gen_optimizer, float(args.epochs), eta_min=args.learning_rate_min) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, max_iter * args.n_critic) architect = Architect_gen(gen, dis, args, 'duality_gap_with_mm', logging) gen.set_gumbel(args.use_gumbel) dis.set_gumbel(args.use_gumbel) for epoch in range(args.start_epoch + 1, args.epochs): scheduler.step() lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) logging.info('epoch %d gen_lr %e', epoch, args.g_lr) logging.info('epoch %d dis_lr %e', epoch, args.d_lr) genotype_gen = gen.genotype() logging.info('gen_genotype = %s', genotype_gen) if 'Discriminator' not in args.dis: genotype_dis = dis.genotype() logging.info('dis_genotype = %s', genotype_dis) print('up_1: ', F.softmax(gen.alphas_up_1, dim=-1)) print('up_2: ', F.softmax(gen.alphas_up_2, dim=-1)) print('up_3: ', F.softmax(gen.alphas_up_3, dim=-1)) # determine whether use gumbel or not if epoch == args.fix_alphas_epochs + 1: gen.set_gumbel(args.use_gumbel) dis.set_gumbel(args.use_gumbel) # grow discriminator and generator if args.grow: dis.cur_stage = grow_ctrl(epoch, args.grow_epoch) gen.cur_stage = grow_ctrl(epoch, args.grow_epoch) if args.restrict_dis_grow and dis.cur_stage > 1: dis.cur_stage = 1 print('debug: dis.cur_stage is {}'.format(dis.cur_stage)) if epoch in args.grow_epoch: train_transform, valid_transform = utils._data_transforms_cifar10_resize( args, 2**(gen.cur_stage + 3)) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.gen_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[:split]), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.gen_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=2) else: gen.cur_stage = 2 dis.cur_stage = 2 # training parameters train_gan_parameter(args, train_queue, gen, dis, gen_optimizer, dis_optimizer, gen_avg_param, logging, writer_dict) # training alphas if epoch > args.fix_alphas_epochs: train_gan_alpha(args, train_queue, valid_queue, gen, dis, architect, gen_optimizer, gen_avg_param, epoch, lr, writer_dict, logging) # evaluate the IS and FID if args.eval and epoch % args.eval_every == 0: inception_score, std, fid_score = validate(args, fixed_z, fid_stat, gen, writer_dict, path_helper) logging.info('epoch {}: IS is {}+-{}, FID is {}'.format( epoch, inception_score, std, fid_score)) if inception_score > IS_best: IS_best = inception_score IS_epoch_best = epoch if fid_score < FID_best: FID_best = fid_score FID_epoch_best = epoch logging.info('best epoch {}: IS is {}'.format( IS_best_epoch, IS_best)) logging.info('best epoch {}: FID is {}'.format( FID_best_epoch, FID_best)) utils.save( gen, os.path.join(path_helper['model'], 'weights_gen_{}.pt'.format('last'))) utils.save( dis, os.path.join(path_helper['model'], 'weights_dis_{}.pt'.format('last'))) genotype_gen = gen.genotype() if 'Discriminator' not in args.dis: genotype_dis = dis.genotype() logging.info('best epoch {}: IS is {}'.format(IS_best_epoch, IS_best)) logging.info('best epoch {}: FID is {}'.format(FID_best_epoch, FID_best)) logging.info('final discovered gen_arch is {}'.format(genotype_gen)) if 'Discriminator' not in args.dis: logging.info('final discovered dis_arch is {}'.format(genotype_dis))
def main(): args = parse_args() args.pretrain = False root_path = 'exps/exp_{}'.format(args.exp) if not os.path.exists(root_path): os.mkdir(root_path) os.mkdir(os.path.join(root_path, "log")) os.mkdir(os.path.join(root_path, "model")) base_lr = args.lr # base learning rate train_dataset, val_dataset = build_dataset(args.dataset, args.data_root, args.train_list) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, num_workers=args.num_workers, pin_memory=True) model = VNet(args.n_channels, args.n_classes).cuda() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=0.0005) #scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.7) model = torch.nn.DataParallel(model) model.train() if args.resume is None: assert os.path.exists(args.load_path) state_dict = model.state_dict() print("Loading weights...") pretrain_state_dict = torch.load(args.load_path, map_location="cpu")['state_dict'] for k in list(pretrain_state_dict.keys()): if k not in state_dict: del pretrain_state_dict[k] model.load_state_dict(pretrain_state_dict) print("Loaded weights") else: print("Resuming from {}".format(args.resume)) checkpoint = torch.load(args.resume, map_location="cpu") optimizer.load_state_dict(checkpoint['optimizer_state_dict']) model.load_state_dict(checkpoint['state_dict']) logger = Logger(root_path) saver = Saver(root_path) for epoch in range(args.start_epoch, args.epochs): train(model, train_loader, optimizer, logger, args, epoch) validate(model, val_loader, optimizer, logger, saver, args, epoch) adjust_learning_rate(args, optimizer, epoch)
def main(): args = cfg.parse_args() random.seed(args.random_seed) torch.manual_seed(args.random_seed) torch.cuda.manual_seed(args.random_seed) # set tf env _init_inception() inception_path = check_or_download_inception(None) create_inception_graph(inception_path) # weight init gen_net = eval('models.' + args.model + '.Generator')(args=args) dis_net = eval('models.' + args.model + '.Discriminator')(args=args) # weight init def weights_init(m): if isinstance(m, nn.Conv2d): if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format( args.init_type)) elif isinstance(m, nn.BatchNorm2d): nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net.apply(weights_init) dis_net.apply(weights_init) gen_net = gen_net.cuda() dis_net = dis_net.cuda() avg_gen_net = deepcopy(gen_net) initial_gen_net_weight = deepcopy(gen_net.state_dict()) initial_dis_net_weight = deepcopy(dis_net.state_dict()) assert id(initial_dis_net_weight) != id(dis_net.state_dict()) assert id(initial_gen_net_weight) != id(gen_net.state_dict()) # set optimizer gen_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr, (args.beta1, args.beta2)) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic) # set up data_loader dataset = datasets.ImageDataset(args) train_loader = dataset.train # fid stat if args.dataset.lower() == 'cifar10': fid_stat = 'fid_stat/fid_stats_cifar10_train.npz' elif args.dataset.lower() == 'stl10': fid_stat = 'fid_stat/fid_stats_stl10_train.npz' else: raise NotImplementedError('no fid stat for %s' % args.dataset.lower()) assert os.path.exists(fid_stat) # epoch number for dis_net args.max_epoch = args.max_epoch * args.n_critic if args.max_iter: args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader)) # initial np.random.seed(args.random_seed) fixed_z = torch.cuda.FloatTensor( np.random.normal(0, 1, (25, args.latent_dim))) start_epoch = 0 best_fid = 1e4 args.path_helper = set_log_dir('logs', args.exp_name + "_{}".format(args.percent)) logger = create_logger(args.path_helper['log_path']) # logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch)) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } print('=> resuming from %s' % args.load_path) assert os.path.exists(args.load_path) checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) gen_net.load_state_dict(checkpoint['gen_state_dict']) torch.manual_seed(args.random_seed) pruning_generate(gen_net, (1 - args.percent), args.pruning_method) torch.manual_seed(args.random_seed) pruning_generate(avg_gen_net, (1 - args.percent), args.pruning_method) see_remain_rate(gen_net) if args.second_seed: dis_net.apply(weights_init) if args.finetune_D: dis_net.load_state_dict(checkpoint['dis_state_dict']) else: dis_net.load_state_dict(initial_dis_net_weight) gen_weight = gen_net.state_dict() gen_orig_weight = rewind_weight(initial_gen_net_weight, gen_weight.keys()) assert id(gen_weight) != id(gen_orig_weight) gen_weight.update(gen_orig_weight) gen_net.load_state_dict(gen_weight) gen_avg_param = copy_params(gen_net) if args.use_kd_D: orig_dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda() orig_dis_net.load(checkpoint['dis_state_dict']) orig_dis_net.eval() # train loop for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)), desc='total progress'): lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None see_remain_rate(gen_net) if not args.use_kd_D: train(args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers) else: train_kd(args, gen_net, dis_net, orig_dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers) if epoch and epoch % args.val_freq == 0 or epoch == int( args.max_epoch) - 1: backup_param = copy_params(gen_net) load_params(gen_net, gen_avg_param) inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict, epoch) logger.info( 'Inception score: %.4f, FID score: %.4f || @ epoch %d.' % (inception_score, fid_score, epoch)) load_params(gen_net, backup_param) if fid_score < best_fid: best_fid = fid_score is_best = True else: is_best = False else: is_best = False avg_gen_net.load_state_dict(gen_net.state_dict()) load_params(avg_gen_net, gen_avg_param) save_checkpoint( { 'epoch': epoch + 1, 'model': args.model, 'gen_state_dict': gen_net.state_dict(), 'dis_state_dict': dis_net.state_dict(), 'avg_gen_state_dict': avg_gen_net.state_dict(), 'gen_optimizer': gen_optimizer.state_dict(), 'dis_optimizer': dis_optimizer.state_dict(), 'best_fid': best_fid, 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path'])
def main(index, args): device = xm.xla_device() gen_net = Generator(args).to(device) dis_net = Discriminator(args).to(device) enc_net = Encoder(args).to(device) def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv2d') != -1: if args.init_type == 'normal': nn.init.normal_(m.weight.data, 0.0, 0.02) elif args.init_type == 'orth': nn.init.orthogonal_(m.weight.data) elif args.init_type == 'xavier_uniform': nn.init.xavier_uniform(m.weight.data, 1.) else: raise NotImplementedError('{} unknown inital type'.format( args.init_type)) elif classname.find('BatchNorm2d') != -1: nn.init.normal_(m.weight.data, 1.0, 0.02) nn.init.constant_(m.bias.data, 0.0) gen_net.apply(weights_init) dis_net.apply(weights_init) enc_net.apply(weights_init) ae_recon_optimizer = torch.optim.Adam( itertools.chain(enc_net.parameters(), gen_net.parameters()), args.ae_recon_lr, (args.beta1, args.beta2)) ae_reg_optimizer = torch.optim.Adam( itertools.chain(enc_net.parameters(), gen_net.parameters()), args.ae_reg_lr, (args.beta1, args.beta2)) dis_optimizer = torch.optim.Adam(dis_net.parameters(), args.d_lr, (args.beta1, args.beta2)) gen_optimizer = torch.optim.Adam(gen_net.parameters(), args.g_lr, (args.beta1, args.beta2)) dataset = datasets.ImageDataset(args) train_loader = dataset.train valid_loader = dataset.valid para_loader = pl.ParallelLoader(train_loader, [device]) fid_stat = str(pathlib.Path( __file__).parent.absolute()) + '/fid_stat/fid_stat_cifar10_test.npz' if not os.path.exists(fid_stat): download_stat_cifar10_test() is_best = True args.num_epochs = np.ceil(args.num_iter / len(train_loader)) gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0, args.num_iter / 2, args.num_iter) dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0, args.num_iter / 2, args.num_iter) ae_recon_scheduler = LinearLrDecay(ae_recon_optimizer, args.ae_recon_lr, 0, args.num_iter / 2, args.num_iter) ae_reg_scheduler = LinearLrDecay(ae_reg_optimizer, args.ae_reg_lr, 0, args.num_iter / 2, args.num_iter) # initial start_epoch = 0 best_fid = 1e4 # set writer if args.load_path: print(f'=> resuming from {args.load_path}') assert os.path.exists(args.load_path) checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint.pth') assert os.path.exists(checkpoint_file) checkpoint = torch.load(checkpoint_file) start_epoch = checkpoint['epoch'] best_fid = checkpoint['best_fid'] gen_net.load_state_dict(checkpoint['gen_state_dict']) enc_net.load_state_dict(checkpoint['enc_state_dict']) dis_net.load_state_dict(checkpoint['dis_state_dict']) gen_optimizer.load_state_dict(checkpoint['gen_optimizer']) dis_optimizer.load_state_dict(checkpoint['dis_optimizer']) ae_recon_optimizer.load_state_dict(checkpoint['ae_recon_optimizer']) ae_reg_optimizer.load_state_dict(checkpoint['ae_reg_optimizer']) args.path_helper = checkpoint['path_helper'] logger = create_logger(args.path_helper['log_path']) logger.info( f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})') else: # create new log dir assert args.exp_name logs_dir = str(pathlib.Path(__file__).parent.parent) + '/logs' args.path_helper = set_log_dir(logs_dir, args.exp_name) logger = create_logger(args.path_helper['log_path']) logger.info(args) writer_dict = { 'writer': SummaryWriter(args.path_helper['log_path']), 'train_global_steps': start_epoch * len(train_loader), 'valid_global_steps': start_epoch // args.val_freq, } # train loop for epoch in tqdm(range(int(start_epoch), int(args.num_epochs)), desc='total progress'): lr_schedulers = (gen_scheduler, dis_scheduler, ae_recon_scheduler, ae_reg_scheduler) train(device, args, gen_net, dis_net, enc_net, gen_optimizer, dis_optimizer, ae_recon_optimizer, ae_reg_optimizer, para_loader, epoch, writer_dict, lr_schedulers) if epoch and epoch % args.val_freq == 0 or epoch == args.num_epochs - 1: fid_score = validate(args, fid_stat, gen_net, writer_dict, valid_loader) logger.info(f'FID score: {fid_score} || @ epoch {epoch}.') if fid_score < best_fid: best_fid = fid_score is_best = True else: is_best = False else: is_best = False save_checkpoint( { 'epoch': epoch + 1, 'gen_state_dict': gen_net.state_dict(), 'dis_state_dict': dis_net.state_dict(), 'enc_state_dict': enc_net.state_dict(), 'gen_optimizer': gen_optimizer.state_dict(), 'dis_optimizer': dis_optimizer.state_dict(), 'ae_recon_optimizer': ae_recon_optimizer.state_dict(), 'ae_reg_optimizer': ae_reg_optimizer.state_dict(), 'best_fid': best_fid, 'path_helper': args.path_helper }, is_best, args.path_helper['ckpt_path'])
def main(): args = parse_args() args.pretrain = False print("Using GPU: {}".format(args.local_rank)) root_path = 'exps/exp_{}'.format(args.exp) if args.local_rank == 0 and not os.path.exists(root_path): os.mkdir(root_path) os.mkdir(os.path.join(root_path, "log")) os.mkdir(os.path.join(root_path, "model")) base_lr = args.lr # base learning rate os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu train_dataset, val_dataset = build_dataset(args.dataset, args.data_root, args.train_list) args.world_size = len(args.gpu.split(",")) if args.world_size > 1: os.environ['MASTER_PORT'] = args.port torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group('nccl') device = torch.device('cuda:{}'.format(args.local_rank)) train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset, num_replicas=len(args.gpu.split(",")), rank=args.local_rank) else: train_sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, num_workers=args.num_workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, num_workers=args.num_workers, pin_memory=True) model = VNet(args.n_channels, args.n_classes).cuda(args.local_rank) optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=0.0005) #scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.7) if args.world_size > 1: model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) model.train() print("Loaded weights") logger = Logger(root_path) saver = Saver(root_path) for epoch in range(args.start_epoch, args.epochs): train(model, train_loader, optimizer, logger, args, epoch) validate(model, val_loader, optimizer, logger, saver, args, epoch) adjust_learning_rate(args, optimizer, epoch)