def __init__(self, seq, max_scan=45):
"""
cs_probs_all = probability of finding a cleavage site at all
positions. Scored by 5 random forests. The position
here is 0 based. shape=(5, num_subseqs)
cleavage_sites = Max of cs_probs_all for each random forest
(length 5)
"""
self.max_scan = functions.validate_scan(seq, max_scan)
self.seq = functions.validate(seq, self.max_scan)
self.s_scores = None
self.c_scores = None
self.y_scores = None
self.cs_probs_all = None
self.cleavage_sites = None
self.preds = None
self.final_cleavage_prob = None
self.final_cleavage = 0
self.final_score_sp = None
self.fungi_scores = functions.np.zeros(5)
self.fungi_preds = functions.np.zeros(5, dtype=bool)
self.final_score_fungi = None
self.toxin_scores = functions.np.zeros(5)
self.toxin_preds = functions.np.zeros(5, dtype=bool)
self.final_score_toxin = None
self.run_fungi = False
self.run_toxin = False
def main():
# 사용자가 종료할 때까지 무한 루프
while True:
equation = input('Please input equation(quit: q): ')
if equation == 'q':
print('Bye')
break
x_value = input('Please input x: ')
# 입력값이 유효한지를 체크
is_validated = validate(equation, x_value)
if is_validated is False:
continue
# 방정식에 x값을 대입하고 우변만 쪼갠다
expression = make_expression(equation, x_value)
# 입력된 계산이 전위/중위/후위 표기인지 판단
# (오류를 발견해 기능을 숨깁니다)
# notation = select_notations(expression)
#
# # 판단된 표기법에 따라 계산 진행
# # (전위, 후위 표기법에 따른 계산은 해결하지 못함)
# if notation is 'prefix':
# answer = 'prefix'
# elif notation is 'infix':
# answer = evaluate_infix(expression)
# elif notation is 'postfix':
# answer = 'postfix'
answer = evaluate_infix(expression)
# 해답 출력 및 1초간 휴식 후 루프 재개
print(f'y: {answer}')
time.sleep(1)
def main():
args = cfg.parse_args()
torch.manual_seed(args.random_seed)
random.seed(args.random_seed)
torch.cuda.manual_seed(args.random_seed)
assert args.exp_name
assert args.load_path.endswith('.pth')
assert os.path.exists(args.load_path)
args.path_helper = set_log_dir('logs_eval', args.exp_name)
logger = create_logger(args.path_helper['log_path'], phase='test')
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import network
gen_net = eval('models.' + args.model + '.Generator')(args=args).cuda()
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
assert os.path.exists(fid_stat)
# initial
np.random.seed(args.random_seed)
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
if args.percent < 0.9:
pruning_generate(gen_net, (1 - args.percent))
see_remain_rate(gen_net)
# set writer
logger.info(f'=> resuming from {args.load_path}')
checkpoint_file = args.load_path
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
if 'avg_gen_state_dict' in checkpoint:
gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])
epoch = checkpoint['epoch']
logger.info(f'=> loaded checkpoint {checkpoint_file} (epoch {epoch})')
else:
gen_net.load_state_dict(checkpoint)
logger.info(f'=> loaded checkpoint {checkpoint_file}')
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'valid_global_steps': 0,
}
inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net,
writer_dict, epoch)
logger.info(f'Inception score: {inception_score}, FID score: {fid_score}.')
writer_dict['writer'].close()
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
assert args.exp_name
assert args.load_path.endswith(".pth")
assert os.path.exists(args.load_path)
args.path_helper = set_log_dir("logs_eval", args.exp_name)
logger = create_logger(args.path_helper["log_path"], phase="test")
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import network
gen_net = eval("models." + args.gen_model + ".Generator")(args=args).cuda()
# fid stat
if args.dataset.lower() == "cifar10":
fid_stat = "fid_stat/fid_stats_cifar10_train.npz"
elif args.dataset.lower() == "stl10":
fid_stat = "fid_stat/stl10_train_unlabeled_fid_stats_48.npz"
else:
raise NotImplementedError(f"no fid stat for {args.dataset.lower()}")
assert os.path.exists(fid_stat)
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
# set writer
logger.info(f"=> resuming from {args.load_path}")
checkpoint_file = args.load_path
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
if "avg_gen_state_dict" in checkpoint:
gen_net.load_state_dict(checkpoint["avg_gen_state_dict"])
epoch = checkpoint["epoch"]
logger.info(f"=> loaded checkpoint {checkpoint_file} (epoch {epoch})")
else:
gen_net.load_state_dict(checkpoint)
logger.info(f"=> loaded checkpoint {checkpoint_file}")
logger.info(args)
writer_dict = {
"writer": SummaryWriter(args.path_helper["log_path"]),
"valid_global_steps": 0,
}
inception_score, fid_score = validate(args,
fixed_z,
fid_stat,
gen_net,
writer_dict,
clean_dir=False)
logger.info(f"Inception score: {inception_score}, FID score: {fid_score}.")
def main(args):
logging.info(args)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
cudnn.benchmark = True
model = get_network_func(args.norm_module)
model.cuda()
print('load checkpoint')
assert os.path.exists(args.weight_path), print(f"Cannot find {args.weight_path}")
checkpoint = torch.load(args.weight_path)
model.load_state_dict(checkpoint['state_dict'])
# dataset process
test_dataset = datasets.CIFAR10(root=args.data, train=False, transform=transforms.Compose([
transforms.ToTensor()
]), download=True)
test_loader = torch.utils.data.DataLoader(
test_dataset,
batch_size=args.batch_size, shuffle=False,
num_workers=2, pin_memory=True)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
# evaluate on test set
# clean branch
print(f"=>test on clean branch")
test_tacc_clean, test_tloss_clean = validate(args, test_loader, model, criterion, 0)
print(f"Test accuracy: {test_tacc_clean}, Test loss: {test_tloss_clean}")
# test_atacc_clean, test_atloss_clean = validate_adv(args, test_loader, model, criterion, 0)
# print(f"Adversarial test accuracy: {test_atacc_clean}, Adversarial test loss: {test_atloss_clean}")
# adv branch
print(f"=>test on adv branch")
test_tacc_adv, test_tloss_adv = validate(args, test_loader, model, criterion, 1)
print(f"Test accuracy: {test_tacc_adv}, Test loss: {test_tloss_adv}")
test_atacc_adv, test_atloss_adv = validate_adv(args, test_loader, model, criterion, 1)
print(f"Adversarial test accuracy: {test_atacc_adv}, Adversarial test loss: {test_atloss_adv}")
def login():
error = None
if request.method == 'POST':
username = request.form['username']
password = request.form['password']
completion = validate(username, password)
if completion == False:
error = 'Invalid Credentials. Please try again.'
else:
messages = json.dumps({"user": username})
session['messages'] = messages
return redirect(url_for('search'))
return render_template('login.html', error=error)
def main(args):
assert args.exp_name
logger, final_output_dir, _ = create_logger(args, args.exp_name, 'test')
args.sample_path = final_output_dir
torch.cuda.manual_seed(args.random_seed)
# set tf env
init_inception()
# fid stat
if args.dataset.lower() == 'cifar10':
args.n_classes = 10
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'imagenet':
# TODO: Support intra-FID
args.n_classes = 143
fid_stat = None
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
if fid_stat:
assert os.path.exists(fid_stat), f"{fid_stat} not found"
# get network
gen_net, _ = get_network_func(args)
gen_net.cuda()
# load checkpoint
checkpoint_file = args.load_path
assert os.path.exists(checkpoint_file), print(
f"checkpoint file {checkpoint_file} not found.")
logger.info("=> loading checkpoint '{}'".format(checkpoint_file))
checkpoint = torch.load(checkpoint_file,
map_location=lambda storage, loc: storage)
gen_net.load_state_dict(checkpoint)
logger.info(f"=> loaded checkpoint '{checkpoint_file}' ")
# evaluation
torch.cuda.empty_cache()
inception_score, fid_score = validate(args, fid_stat, gen_net, None)
logger.info(f'Inception score: {inception_score}, FID score: {fid_score} ')
def login():
error = None
if request.method == 'POST':
username = request.form['username']
password = request.form['password']
completion = validate(username, password)
if completion is False:
error = 'Niepoprawny login lub hasło'
else:
session['username'] = request.form['username']
username = session['username']
if check_grupa(username) == 'admin':
info = "Witaj" + " " + check_grupa(username)+"ie"
flash(info)
if check_grupa(username) == 'nauczyciel':
info = "Witaj" + " " + check_grupa(username) + "u"
flash(info)
if check_grupa(username) == 'rodzic':
info = "Witaj" + " " + check_grupa(username) + "u"
flash(info)
return render_template('base.html', error=error, info=username, grupa=check_grupa(username))
return render_template('login.html', error=error)
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import network
gen_net = eval('models.' + args.model + '.Generator')(args=args).cuda()
dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda()
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
gen_avg_param = copy_params(gen_net)
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model',
'checkpoint.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
avg_gen_net = deepcopy(gen_net)
avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])
gen_avg_param = copy_params(avg_gen_net)
del avg_gen_net
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info(
f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})')
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
train(args, gen_net, dis_net, gen_optimizer, dis_optimizer,
gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat,
gen_net, writer_dict)
logger.info(
f'Inception score: {inception_score}, FID score: {fid_score} || @ epoch {epoch}.'
)
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net = deepcopy(gen_net)
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
del avg_gen_net
def getHistoricalData(country,
indicator,
initDate=None,
endDate=None,
output_type=None):
"""
Return historical information for specific country and indicator.
=================================================================
Parameters:
-----------
country: string or list.
String to get data for one country. List of strings to get data for
several countries. For example, country = ['United States', 'Australia'].
indicator: string or list.
String to get data for one category. List of strings to get data for several calendar events.
For example, category = 'GDP Growth Rate' or
category = ['Exports', 'Imports']
initDate: string with format: YYYY-MM-DD.
For example: '2011-01-01'
endDate: string with format: YYYY-MM-DD.
output_type: string.
'dict'(default) for dictionary format output,
'raw' for list of dictionaries without any parsing.
Notes
-----
Without credentials only sample data will be provided.
Example
-------
getHistoricalData(country = 'United States', indicator = 'Imports', initDate = '2011-01-01', endDate = '2016-01-01')
getHistoricalData(country = ['United States', 'United Kingdom'], indicator = ['Imports','Exports'], initDate = '2011-01-01', endDate = '2016-01-01')
"""
if type(country) is not str or type(indicator) is not str:
linkAPI = paramCheck(country, indicator)
else:
linkAPI = 'http://api.tradingeconomics.com/historical/country/' + urllib.quote(
country) + '/indicator/' + urllib.quote(indicator)
if initDate == None and endDate == None:
minDate = [
(datetime.now() - relativedelta(years=10)).strftime('%Y-%m-%d')
]
linkAPI = fn.finalLink(linkAPI, minDate)
if initDate == None and (endDate is not None):
iDate = (datetime.now() - relativedelta(years=10)).strftime('%Y-%m-%d')
try:
fn.validate(endDate)
except ValueError:
raise DateError(
'Incorrect endDate format, should be YYYY-MM-DD or MM-DD-YYYY.'
)
try:
fn.validatePeriod(iDate, endDate)
except ValueError:
raise DateError('Incorrect time period.')
param = [iDate, endDate]
linkAPI = fn.finalLink(linkAPI, param)
if (initDate is not None) and (endDate is not None):
try:
fn.validate(initDate)
except ValueError:
raise DateError(
'Incorrect initDate format, should be YYYY-MM-DD or MM-DD-YYYY.'
)
try:
fn.validate(endDate)
except ValueError:
raise DateError(
'Incorrect endDate format, should be YYYY-MM-DD or MM-DD-YYYY.'
)
try:
fn.validatePeriod(initDate, endDate)
except ValueError:
raise DateError('Invalid time period.')
param = [initDate, endDate]
linkAPI = fn.finalLink(linkAPI, param)
if (initDate is not None) and endDate == None:
try:
fn.validate(initDate)
except ValueError:
raise DateError(
'Incorrect initDate format, should be YYYY-MM-DD or MM-DD-YYYY.'
)
if initDate > str(date.today()):
raise DateError('Initial date out of range.')
myDate = [initDate]
linkAPI = fn.finalLink(linkAPI, myDate)
try:
linkAPI = linkAPI + '?c=' + glob.apikey
except AttributeError:
raise LoginError('You need to do login before making any request')
try:
webResults = json.load(urllib.urlopen(linkAPI))
except ValueError:
raise CredentialsError('Invalid credentials')
if len(webResults) > 0:
date = [d['DateTime'] for d in webResults]
value = [d[u'Value'] for d in webResults]
results = {'dates': date, 'values': value}
if (type(country) == str and type(indicator) == str):
results = parseData(results)
else:
results = multiParams(webResults)
else:
raise ParametersError('No data available for the provided parameters.')
if output_type == None or output_type == 'dict':
output = results
elif output_type == 'raw':
output = webResults
else:
raise ParametersError(
'output_type options : dict(defoult) for dictionary or raw for unparsed results.'
)
return output
def getCalendarData(country = None, category = None, initDate = None, endDate = None, output_type = None):
"""
Return calendar events.
===========================================================
Parameters:
-----------
country: string or list.
String to get data for one country. List of strings to get data for
several countries. For example, country = ['United States', 'Australia'].
category: string or list.
String to get data for one category. List of strings to get data for several calendar events.
For example, category = 'GDP Growth Rate' or
category = ['Exports', 'Imports']
initDate: string with format: YYYY-MM-DD.
For example: '2011-01-01'
endDate: string with format: YYYY-MM-DD.
output_type: string.
'dict'(default) for dictionary format output, 'df' for data frame,
'raw' for list of dictionaries without any parsing.
Notes
-----
All parameters are optional. When not supplying parameters, data for all countries and indicators will be provided.
Example
-------
getCalendarData(country = 'United States', category = 'Imports', initDate = '2011-01-01', endDate = '2016-01-01')
getCalendarData(country = ['United States', 'India'], category = ['Imports','Exports'], initDate = '2011-01-01', endDate = '2016-01-01')
"""
if country == None and category == None:
linkAPI = 'http://api.tradingeconomics.com/calendar'
elif country == None and category != None:
country_all = 'all'
linkAPI = paramCheck(country_all, category)
elif type(country) is str and type(category) is str:
linkAPI = 'http://api.tradingeconomics.com/calendar/country/' + urllib.quote(country) + '/indicator/' + urllib.quote(category)
else:
linkAPI = paramCheck(country, category)
if initDate == None and endDate == None:
linkAPI = linkAPI
elif endDate > str(datetime.now()):
raise DateError ('End date could not be greater than actual date')
else:
try:
fn.validate(initDate)
except ValueError:
raise DateError ('Incorrect initial date format, should be YYYY-MM-DD ')
try:
fn.validate(endDate)
except ValueError:
raise DateError ('Incorrect end date format, should be YYYY-MM-DD ')
try:
fn.validatePeriod(initDate, endDate)
except ValueError:
raise DateError ('Invalid time period.')
param=[initDate, endDate]
linkAPI = fn.finalLink(linkAPI, param)
try:
linkAPI = linkAPI + '?c=' + glob.apikey
except AttributeError:
raise LoginError('You need to do login before making any request')
try:
webResults = json.load(urllib.urlopen(linkAPI))
except ValueError:
raise CredentialsError ('Invalid credentials')
if len(webResults) > 0:
names = ['date', 'country', 'category', 'event', 'reference', 'unit', 'source', 'actual', 'previous', 'forecast', 'teforecast']
names2 = ['Date', 'Country', 'Category', 'Event', 'Reference', 'Unit', 'Source', 'Actual', 'Previous', 'Forecast', 'TEForecast']
maindf = pd.DataFrame()
for i in range(len(names)):
names[i] = [d[names2[i]] for d in webResults]
maindf = pd.concat([maindf, pd.DataFrame(names[i], columns = [names2[i]])], axis = 1)
else:
raise ParametersError ('No data available for the provided parameters.')
if output_type == None or output_type =='dict':
output = fn.out_type(maindf)
elif output_type == 'df':
output = maindf
elif output_type == 'raw':
output = webResults
else:
raise ParametersError ('output_type options : df for data frame, dict(defoult) for dictionary by country, raw for unparsed results.')
return output
def main():
args = cfg.parse_args()
random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
torch.cuda.manual_seed(args.random_seed)
np.random.seed(args.random_seed)
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
os.environ['PYTHONHASHSEED'] = str(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import network
gen_net = eval('models.'+args.model+'.Generator')(args=args)
dis_net = eval('models.'+args.model+'.Discriminator')(args=args)
initial_gen_net_weight = torch.load(os.path.join(args.init_path, 'initial_gen_net.pth'), map_location="cpu")
initial_dis_net_weight = torch.load(os.path.join(args.init_path, 'initial_dis_net.pth'), map_location="cpu")
gen_net = gen_net.cuda()
dis_net = dis_net.cuda()
gen_net.load_state_dict(initial_gen_net_weight)
dis_net.load_state_dict(initial_dis_net_weight)
# set optimizer
gen_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, gen_net.parameters()),
args.g_lr, (args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, dis_net.parameters()),
args.d_lr, (args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/fid_stats_stl10_train.npz'
else:
raise NotImplementedError('no fid stat for %s' % args.dataset.lower())
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic / len(train_loader))
# initial
fixed_z = torch.cuda.FloatTensor(np.random.normal(0, 1, (25, args.latent_dim)))
gen_avg_param = copy_params(gen_net)
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print('=> resuming from %s' % args.load_path)
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
avg_gen_net = deepcopy(gen_net)
avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])
gen_avg_param = copy_params(avg_gen_net)
del avg_gen_net
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch))
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
# train loop
switch = False
for epoch in range(int(start_epoch), int(args.max_epoch)):
lr_schedulers = (gen_scheduler, dis_scheduler) if args.lr_decay else None
train(args, gen_net, dis_net, gen_optimizer, dis_optimizer, gen_avg_param, train_loader, epoch, writer_dict,
lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(args.max_epoch)-1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat, gen_net, writer_dict, epoch)
logger.info('Inception score: %.4f, FID score: %.4f || @ epoch %d.' % (inception_score, fid_score, epoch))
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net = deepcopy(gen_net)
load_params(avg_gen_net, gen_avg_param)
save_checkpoint({
'epoch': epoch + 1,
'model': args.model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper,
'seed': args.random_seed
}, is_best, args.path_helper['ckpt_path'])
del avg_gen_net
def validate(task_info, data_type):
""" check if the data_type can be correctly calculated in the task
:param task_info: task_info dict, e.g.
{
'input_data_type_id': 121,
'time-range': '1m',
'functions': {
'1': {
'name': 'filter',
'target': 'status',
'conditions': [
{
'target': 'cost', 'operator': 'bt', 'param1': 100, 'param2': 1000
}, {...}
]
},
'2': {
'name': 'average',
'target': 'cost',
'tag': 'avg_cost',
'group_by': ['function', 'status']
}
}
}
:param data_type: data_type dict, e.g.
{'userid': 'string', 'function': 'string', 'status': 'integer', 'cost': 'real'}
:return: the result dict, in which 'result' is a boolean indicate if it is validate, if not, the 'info' would
contain some information of the reason
"""
result = {'succ': True}
if not isinstance(task_info, dict):
result['succ'] = False
result['info'] = 'task desc should be dict'
return result
# check title
if len(task_info['title']) == 0:
result['succ'] = False
result['info'] = 'title is empty.'
return result
# check basic attributes
for key in task_structure:
if key not in task_info:
result['succ'] = False
result['info'] = 'key not found: {key}'.format(key=key)
return result
if not task_structure[key](task_info[key]):
result['succ'] = False
result['info'] = 'type error: {key}'.format(key=key)
return result
# check time range
pattern = re.compile("\A[1-9][0-9]{0,2}[dhms]\Z")
if not pattern.match(task_info['time_range']):
result['succ'] = False
result['info'] = 'time range format not valid'.format(
func_name=task_info['name'])
return result
# check each function
temp_type = data_type
for i in xrange(0, len(task_info['functions'])):
if str(i + 1) not in task_info['functions']:
result['succ'] = False
result['info'] = 'function key error: expected {key}'.format(
key=i + 1)
return result
temp_result = functions.validate(task_info['functions'][str(i + 1)],
temp_type)
if temp_result['succ']:
temp_type = temp_result['result_type']
else:
result['succ'] = False
result['info'] = temp_result['info']
result['result_type'] = temp_type
return result
def main():
args = cfg.parse_args()
random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
torch.cuda.manual_seed(args.random_seed)
np.random.seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import netwo
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net = eval('models.' + args.model + '.Generator')(args=args).cuda()
dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda()
gen_net.apply(weights_init)
dis_net.apply(weights_init)
avg_gen_net = deepcopy(gen_net)
initial_gen_net_weight = torch.load(os.path.join(args.init_path,
'initial_gen_net.pth'),
map_location="cpu")
initial_dis_net_weight = torch.load(os.path.join(args.init_path,
'initial_dis_net.pth'),
map_location="cpu")
assert id(initial_dis_net_weight) != id(dis_net.state_dict())
assert id(initial_gen_net_weight) != id(gen_net.state_dict())
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/fid_stats_stl10_train.npz'
else:
raise NotImplementedError('no fid stat for %s' % args.dataset.lower())
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
start_epoch = 0
best_fid = 1e4
print('=> resuming from %s' % args.load_path)
assert os.path.exists(args.load_path)
checkpoint_file = args.load_path
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
pruning_generate(gen_net, checkpoint['gen_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
total = 0
total_nonzero = 0
for m in dis_net.modules():
if isinstance(m, nn.Conv2d):
total += m.weight_orig.data.numel()
mask = m.weight_orig.data.abs().clone().gt(0).float().cuda()
total_nonzero += torch.sum(mask)
conv_weights = torch.zeros(total)
index = 0
for m in dis_net.modules():
if isinstance(m, nn.Conv2d):
size = m.weight_orig.data.numel()
conv_weights[index:(
index + size)] = m.weight_orig.data.view(-1).abs().clone()
index += size
y, i = torch.sort(conv_weights)
# thre_index = int(total * args.percent)
# only care about the non zero weights
# e.g: total = 100, total_nonzero = 80, percent = 0.2, thre_index = 36, that means keep 64
thre_index = total - total_nonzero
thre = y[int(thre_index)]
pruned = 0
print('Pruning threshold: {}'.format(thre))
zero_flag = False
masks = OrderedDict()
for k, m in enumerate(dis_net.modules()):
if isinstance(m, nn.Conv2d):
weight_copy = m.weight_orig.data.abs().clone()
mask = weight_copy.gt(thre).float()
masks[k] = mask
pruned = pruned + mask.numel() - torch.sum(mask)
m.weight_orig.data.mul_(mask)
if int(torch.sum(mask)) == 0:
zero_flag = True
print(
'layer index: {:d} \t total params: {:d} \t remaining params: {:d}'
.format(k, mask.numel(), int(torch.sum(mask))))
print('Total conv params: {}, Pruned conv params: {}, Pruned ratio: {}'.
format(total, pruned, pruned / total))
pruning_generate(avg_gen_net, checkpoint['gen_state_dict'])
see_remain_rate(gen_net)
if not args.finetune_G:
gen_weight = gen_net.state_dict()
gen_orig_weight = rewind_weight(initial_gen_net_weight,
gen_weight.keys())
gen_weight.update(gen_orig_weight)
gen_net.load_state_dict(gen_weight)
gen_avg_param = copy_params(gen_net)
if args.finetune_D:
dis_net.load_state_dict(checkpoint['dis_state_dict'])
else:
dis_net.load_state_dict(initial_dis_net_weight)
for k, m in enumerate(dis_net.modules()):
if isinstance(m, nn.Conv2d):
m.weight_orig.data.mul_(masks[k])
orig_dis_net = eval('models.' + args.model +
'.Discriminator')(args=args).cuda()
orig_dis_net.load_state_dict(checkpoint['dis_state_dict'])
orig_dis_net.eval()
args.path_helper = set_log_dir('logs',
args.exp_name + "_{}".format(args.percent))
logger = create_logger(args.path_helper['log_path'])
#logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch))
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
see_remain_rate(gen_net)
see_remain_rate_orig(dis_net)
if not args.use_kd_D:
train_with_mask(args, gen_net, dis_net, gen_optimizer,
dis_optimizer, gen_avg_param, train_loader, epoch,
writer_dict, masks, lr_schedulers)
else:
train_with_mask_kd(args, gen_net, dis_net, orig_dis_net,
gen_optimizer, dis_optimizer, gen_avg_param,
train_loader, epoch, writer_dict, masks,
lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat,
gen_net, writer_dict, epoch)
logger.info(
'Inception score: %.4f, FID score: %.4f || @ epoch %d.' %
(inception_score, fid_score, epoch))
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net.load_state_dict(gen_net.state_dict())
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
def main():
functions.clear()
if not functions.exists():
try:
functions.intro()
except KeyboardInterrupt:
functions.clear()
import sys
sys.exit(0)
functions.clear()
try:
password = getpass.getpass("Password: "******"Invalid password")
import sys
sys.exit(1)
while True:
functions.clear()
print("Vault")
print(
"Use Ctrl+C to return back to this menu, or use it here to exit.\n"
)
try:
action = input(
"1: Encrypt a file\n2: Download and encrypt a file from a URL\n3: Decrypt a file\n4: Change password\nSelect an option: "
)
except:
break
action = action.strip().lstrip()
try:
action = int(action)
except:
continue
if action not in [1, 2, 3, 4]:
continue
if action == 1:
functions.clear()
try:
path = input("Please specify a file to encrypt: ")
except KeyboardInterrupt:
continue
if functions.validate(path) and not path.endswith(".png"):
functions.clear()
try:
response = input(
"This file may be a valid image file. Convert to .png (y/n)? "
)
except KeyboardInterrupt:
continue
if response.lower().lstrip().strip() == "y":
temp = path
path = functions.convert(path)
delete = input("Delete original file (y/n)? ")
if delete.lower().lstrip().strip() == "y":
if not windows:
temp = temp.replace("\\", "")
try:
directory = os.path.dirname(os.path.realpath(temp))
os.chdir(directory)
os.remove(temp)
except:
pass
functions.clear()
try:
save = functions.encodepath(path)
if save == None:
continue
except Exception as e:
print(str(e))
try:
input("Invalid file path. Press enter to continue. ")
except KeyboardInterrupt:
continue
continue
else:
try:
input(
"File encrypted.\nEncrypted file path: {}\nPress enter to continue. "
.format(save))
except KeyboardInterrupt:
continue
continue
elif action == 2:
functions.clear()
try:
url = input("Please specify a file URL to download: ")
except KeyboardInterrupt:
continue
name = os.path.split(url)[-1]
base = os.path.dirname(os.path.realpath(__file__))
path = os.path.join(base, name)
functions.clear()
try:
r = requests.get(url, stream=True)
with open(path, 'wb') as out_file:
shutil.copyfileobj(r.raw, out_file)
except:
try:
input("Invalid URL: {}\nPress enter to continue. ".format(
url))
except KeyboardInterrupt:
continue
continue
if functions.validate(path) and not path.endswith(".png"):
functions.clear()
try:
response = input(
"This file may be a valid image file. Convert to .png (y/n)? "
)
except KeyboardInterrupt:
continue
if response.lower().lstrip().strip() == "y":
temp = path
path = functions.convert(path)
delete = input("Delete original file (y/n)? ")
if delete.lower().lstrip().strip() == "y":
if not windows:
temp = temp.replace("\\", "")
try:
directory = os.path.dirname(os.path.realpath(temp))
os.chdir(directory)
os.remove(temp)
except:
pass
functions.clear()
try:
save = functions.encodepath(path)
except:
try:
input("Invalid file path. Press enter to continue. ")
except KeyboardInterrupt:
continue
continue
else:
try:
input(
"File encrypted.\nEncrypted file path: {}\nPress enter to continue. "
.format(save))
except KeyboardInterrupt:
continue
continue
elif action == 3:
functions.clear()
try:
path = input("Please specify a .enc file to decrypt: ")
except KeyboardInterrupt:
continue
functions.clear()
try:
save = functions.decodepath(path)
except Exception as e:
print(str(e))
try:
input("Invalid file path. Press enter to continue. ")
except KeyboardInterrupt:
continue
continue
else:
try:
input(
"File decrypted.\nDecrypted file path: {}\nPress enter to continue. "
.format(save))
except KeyboardInterrupt:
continue
continue
elif action == 4:
try:
functions.clear()
current = input("Current password: "******"Passwords do not match. Press enter to continue. ")
continue
new = input("New password: "******"You will need to decrypt all files with your old password. Your new password will not be able to decrypt already encrypted files.\nRetype your password to confirm: "
)
if confirm != new:
input("Passwords do not match. Press enter to continue. ")
continue
functions.writepw(new)
functions.clear()
input("Password changed to: {}.\nPress enter to continue. ".
format(new))
except KeyboardInterrupt:
continue
functions.clear()
def main():
args = cfg_train.parse_args()
torch.cuda.manual_seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import network
# gen_net = eval('models.' + args.gen_model + '.' + args.gen)(args=args).cuda()
genotype_gen = eval('genotypes.%s' % args.arch_gen)
gen_net = eval('models.' + args.gen_model + '.' + args.gen)(
args, genotype_gen).cuda()
# gen_net = eval('models.' + args.gen_model + '.' + args.gen)(args = args).cuda()
if 'Discriminator' not in args.dis:
genotype_dis = eval('genotypes.%s' % args.arch_dis)
dis_net = eval('models.' + args.dis_model + '.' + args.dis)(
args, genotype_dis).cuda()
else:
dis_net = eval('models.' + args.dis_model + '.' +
args.dis)(args=args).cuda()
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
val_loader = dataset.valid
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, args.g_lr * 0.01,
260 * len(train_loader),
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, args.d_lr * 0.01,
260 * len(train_loader),
args.max_iter * args.n_critic)
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
elif args.dataset.lower() == 'mnist':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
fixed_z_sample = torch.cuda.FloatTensor(
np.random.normal(0, 1, (args.eval_batch_size, args.latent_dim)))
gen_avg_param = copy_params(gen_net)
start_epoch = 0
best_fid = 1e4
best_fid_epoch = 0
is_with_fid = 0
std_with_fid = 0.
best_is = 0
best_is_epoch = 0
fid_with_is = 0
best_dts = 0
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model',
'checkpoint.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
avg_gen_net = deepcopy(gen_net)
avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])
gen_avg_param = copy_params(avg_gen_net)
del avg_gen_net
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info(
f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})')
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
# calculate the FLOPs and param count of G
input = torch.randn(args.gen_batch_size, args.latent_dim).cuda()
flops, params = profile(gen_net, inputs=(input, ))
flops, params = clever_format([flops, params], "%.3f")
logger.info('FLOPs is {}, param count is {}'.format(flops, params))
# train loop
dg_list = []
worst_lr = 1e-5
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
train(args, gen_net, dis_net, gen_optimizer, dis_optimizer,
gen_avg_param, train_loader, epoch, writer_dict, args.consistent,
lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, std, fid_score = validate(args,
fixed_z,
fid_stat,
gen_net,
writer_dict,
args.path_helper,
search=False)
logger.info(
f'Inception score: {inception_score}, FID score: {fid_score}+-{std} || @ epoch {epoch}.'
)
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
best_fid_epoch = epoch
is_with_fid = inception_score
std_with_fid = std
is_best = True
else:
is_best = False
if inception_score > best_is:
best_is = inception_score
best_std = std
fid_with_is = fid_score
best_is_epoch = epoch
else:
is_best = False
# save generated images
if epoch % args.image_every == 0:
gen_noise = torch.cuda.FloatTensor(
np.random.normal(0, 1,
(args.eval_batch_size, args.latent_dim)))
# gen_images = gen_net(fixed_z_sample)
# gen_images = gen_images.reshape(args.eval_batch_size, 32, 32, 3)
# gen_images = gen_images.cpu().detach()
gen_images = gen_net(fixed_z_sample).mul_(127.5).add_(
127.5).clamp_(0.0, 255.0).permute(0, 2, 3,
1).to('cpu',
torch.uint8).numpy()
fig = plt.figure()
grid = ImageGrid(fig, 111, nrows_ncols=(10, 10), axes_pad=0)
for x in range(args.eval_batch_size):
grid[x].imshow(gen_images[x]) # cmap="gray")
grid[x].set_xticks([])
grid[x].set_yticks([])
plt.savefig(
os.path.join(args.path_helper['sample_path'],
"epoch_{}.png".format(epoch)))
plt.close()
avg_gen_net = deepcopy(gen_net)
# avg_gen_net = eval('models.'+args.gen_model+'.' + args.gen)(args, genotype_gen).cuda()
# avg_gen_net = eval('models.' + args.gen_model + '.' + args.gen)(args=args).cuda()
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
'epoch': epoch + 1,
'gen_model': args.gen_model,
'dis_model': args.dis_model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
del avg_gen_net
logger.info(
'best_is is {}+-{}@{} epoch, fid is {}, best_fid is {}@{}, is is {}+-{}'
.format(best_is, best_std, best_is_epoch, fid_with_is, best_fid,
best_fid_epoch, is_with_fid, std_with_fid))
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
torch.cuda.manual_seed_all(args.random_seed)
np.random.seed(args.random_seed)
random.seed(args.random_seed)
torch.backends.cudnn.deterministic = True
# import network
# args.gen_model is TransGAN_8_8_1 for example
gen_net = eval('models.'+args.gen_model+'.Generator')(args=args).cuda()
dis_net = eval('models.'+args.dis_model+'.Discriminator')(args=args).cuda()
gen_net.set_arch(args.arch, cur_stage=2)
print("The shit!")
# weight init: Xavier Uniform
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
gpu_ids = [i for i in range(int(torch.cuda.device_count()))]
gen_net = torch.nn.DataParallel(gen_net.to("cuda:0"), device_ids=gpu_ids)
dis_net = torch.nn.DataParallel(dis_net.to("cuda:0"), device_ids=gpu_ids)
# print(gen_net.module.cur_stage)
if args.optimizer == "adam":
gen_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, gen_net.parameters()),
args.g_lr, (args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, dis_net.parameters()),
args.d_lr, (args.beta1, args.beta2))
elif args.optimizer == "adamw":
gen_optimizer = AdamW(filter(lambda p: p.requires_grad, gen_net.parameters()),
args.g_lr, weight_decay=args.wd)
dis_optimizer = AdamW(filter(lambda p: p.requires_grad, dis_net.parameters()),
args.g_lr, weight_decay=args.wd)
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0, args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0, args.max_iter * args.n_critic)
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
elif args.fid_stat is not None:
fid_stat = args.fid_stat
else:
raise NotImplementedError # (f"no fid stat for %s"%args.dataset.lower()")
assert os.path.exists(fid_stat)
dataset = datasets.ImageDataset(args, cur_img_size=8)
train_loader = dataset.train
writer=SummaryWriter()
writer_dict = {'writer':writer}
writer_dict["train_global_steps"]=0
writer_dict["valid_global_steps"]=0
best = 1e4
for epoch in range(args.max_epoch):
train(args, gen_net = gen_net, dis_net = dis_net, gen_optimizer = gen_optimizer, dis_optimizer = dis_optimizer, gen_avg_param = None, train_loader = train_loader,
epoch = epoch, writer_dict = writer_dict, fixed_z = None, schedulers=[gen_scheduler, dis_scheduler])
checkpoint = {'epoch':epoch, 'best_fid':best}
checkpoint['gen_state_dict'] = gen_net.state_dict()
checkpoint['dis_state_dict'] = dis_net.state_dict()
score = validate(args, None, fid_stat, epoch, gen_net, writer_dict, clean_dir=True)
# print these scores, is it really the latest
print(f'FID score: {score} - best ID score: {best} || @ epoch {epoch}.')
if epoch == 0 or epoch > 30:
if score < best:
save_checkpoint(checkpoint, is_best=(score<best), output_dir=args.output_dir)
print("Saved Latest Model!")
best = score
checkpoint = {'epoch':epoch, 'best_fid':best}
checkpoint['gen_state_dict'] = gen_net.state_dict()
checkpoint['dis_state_dict'] = dis_net.state_dict()
score = validate(args, None, fid_stat, epoch, gen_net, writer_dict, clean_dir=True)
save_checkpoint(checkpoint, is_best=(score<best), output_dir=args.output_dir)
def validate(task_info, data_type):
""" check if the data_type can be correctly calculated in the task
:param task_info: task_info dict, e.g.
{
'input_data_type_id': 121,
'time-range': '1m',
'functions': {
'1': {
'name': 'filter',
'target': 'status',
'conditions': [
{
'target': 'cost', 'operator': 'bt', 'param1': 100, 'param2': 1000
}, {...}
]
},
'2': {
'name': 'average',
'target': 'cost',
'tag': 'avg_cost',
'group_by': ['function', 'status']
}
}
}
:param data_type: data_type dict, e.g.
{'userid': 'string', 'function': 'string', 'status': 'integer', 'cost': 'real'}
:return: the result dict, in which 'result' is a boolean indicate if it is validate, if not, the 'info' would
contain some information of the reason
"""
result = {
'succ': True
}
if not isinstance(task_info, dict):
result['succ'] = False
result['info'] = 'task desc should be dict'
return result
# check title
if len(task_info['title']) == 0:
result['succ'] = False
result['info'] = 'title is empty.'
return result
# check basic attributes
for key in task_structure:
if key not in task_info:
result['succ'] = False
result['info'] = 'key not found: {key}'.format(key=key)
return result
if not task_structure[key](task_info[key]):
result['succ'] = False
result['info'] = 'type error: {key}'.format(key=key)
return result
# check time range
pattern = re.compile("\A[1-9][0-9]{0,2}[dhms]\Z")
if not pattern.match(task_info['time_range']):
result['succ'] = False
result['info'] = 'time range format not valid'.format(
func_name=task_info['name'])
return result
# check each function
temp_type = data_type
for i in xrange(0, len(task_info['functions'])):
if str(i + 1) not in task_info['functions']:
result['succ'] = False
result['info'] = 'function key error: expected {key}'.format(key=i + 1)
return result
temp_result = functions.validate(task_info['functions'][str(i + 1)], temp_type)
if temp_result['succ']:
temp_type = temp_result['result_type']
else:
result['succ'] = False
result['info'] = temp_result['info']
result['result_type'] = temp_type
return result
def main():
args = cfg.parse_args()
random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
torch.cuda.manual_seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net = Generator(bottom_width=args.bottom_width,
gf_dim=args.gf_dim,
latent_dim=args.latent_dim).cuda()
dis_net = eval('models.' + args.model + '.Discriminator')(args=args).cuda()
gen_net.apply(weights_init)
dis_net.apply(weights_init)
initial_gen_net_weight = torch.load(os.path.join(args.init_path,
'initial_gen_net.pth'),
map_location="cpu")
initial_dis_net_weight = torch.load(os.path.join(args.init_path,
'initial_dis_net.pth'),
map_location="cpu")
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
exp_str = args.dir
args.load_path = os.path.join('output', exp_str, 'pth',
'epoch{}.pth'.format(args.load_epoch))
# state dict:
assert os.path.exists(args.load_path)
checkpoint = torch.load(args.load_path)
print('=> loaded checkpoint %s' % args.load_path)
state_dict = checkpoint['generator']
gen_net = load_subnet(args, state_dict, initial_gen_net_weight).cuda()
avg_gen_net = deepcopy(gen_net)
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
else:
raise NotImplementedError('no fid stat for %s' % args.dataset.lower())
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
np.random.seed(args.random_seed)
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
start_epoch = 0
best_fid = 1e4
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
#logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch))
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
gen_avg_param = copy_params(gen_net)
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
train(args, gen_net, dis_net, gen_optimizer, dis_optimizer,
gen_avg_param, train_loader, epoch, writer_dict, lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat,
gen_net, writer_dict)
logger.info(
'Inception score: %.4f, FID score: %.4f || @ epoch %d.' %
(inception_score, fid_score, epoch))
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net.load_state_dict(gen_net.state_dict())
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
torch.cuda.manual_seed_all(args.random_seed)
np.random.seed(args.random_seed)
random.seed(args.random_seed)
torch.backends.cudnn.deterministic = True
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# epoch number for dis_net
dataset = datasets.ImageDataset(args, cur_img_size=8)
train_loader = dataset.train
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter / len(train_loader))
else:
args.max_iter = args.max_epoch * len(train_loader)
args.max_epoch = args.max_epoch * args.n_critic
# import network
gen_net = eval('models.' + args.gen_model + '.Generator')(args=args).cuda()
dis_net = eval('models.' + args.dis_model +
'.Discriminator')(args=args).cuda()
gen_net.set_arch(args.arch, cur_stage=2)
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform_(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
gpu_ids = [i for i in range(int(torch.cuda.device_count()))]
gen_net = torch.nn.DataParallel(gen_net.to("cuda:0"), device_ids=gpu_ids)
dis_net = torch.nn.DataParallel(dis_net.to("cuda:0"), device_ids=gpu_ids)
gen_net.module.cur_stage = 0
dis_net.module.cur_stage = 0
gen_net.module.alpha = 1.
dis_net.module.alpha = 1.
# set optimizer
if args.optimizer == "adam":
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
elif args.optimizer == "adamw":
gen_optimizer = AdamW(filter(lambda p: p.requires_grad,
gen_net.parameters()),
args.g_lr,
weight_decay=args.wd)
dis_optimizer = AdamW(filter(lambda p: p.requires_grad,
dis_net.parameters()),
args.g_lr,
weight_decay=args.wd)
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
elif args.fid_stat is not None:
fid_stat = args.fid_stat
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
assert os.path.exists(fid_stat)
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (64, args.latent_dim)))
gen_avg_param = copy_params(gen_net)
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path)
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
# avg_gen_net = deepcopy(gen_net)
# avg_gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])
gen_avg_param = checkpoint['gen_avg_param']
# del avg_gen_net
cur_stage = cur_stages(start_epoch, args)
gen_net.module.cur_stage = cur_stage
dis_net.module.cur_stage = cur_stage
gen_net.module.alpha = 1.
dis_net.module.alpha = 1.
args.path_helper = checkpoint['path_helper']
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir('logs', args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
def return_states():
states = {}
states['epoch'] = epoch
states['best_fid'] = best_fid_score
states['gen_state_dict'] = gen_net.state_dict()
states['dis_state_dict'] = dis_net.state_dict()
states['gen_optimizer'] = gen_optimizer.state_dict()
states['dis_optimizer'] = dis_optimizer.state_dict()
states['gen_avg_param'] = gen_avg_param
states['path_helper'] = args.path_helper
return states
# train loop
for epoch in range(start_epoch + 1, args.max_epoch):
train(
args,
gen_net,
dis_net,
gen_optimizer,
dis_optimizer,
gen_avg_param,
train_loader,
epoch,
writer_dict,
fixed_z,
)
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
fid_score = validate(
args,
fixed_z,
fid_stat,
epoch,
gen_net,
writer_dict,
)
logger.info(f'FID score: {fid_score} || @ epoch {epoch}.')
load_params(gen_net, backup_param)
is_best = False
if epoch == 1 or fid_score < best_fid_score:
best_fid_score = fid_score
is_best = True
if is_best or epoch % 1 == 0:
states = return_states()
save_checkpoint(states,
is_best,
args.path_helper['ckpt_path'],
filename=f'checkpoint_epoch_{epoch}.pth')
def main(args):
logging.info(args)
writer = SummaryWriter(args.save_dir)
best_prec1, best_ata, cln_best_prec1 = 0, 0, 0
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
cudnn.benchmark = True
setup_seed(args.seed)
model = get_network_func(args.norm_module)
model.cuda()
start_epoch = 0
if args.resume:
print('resume from checkpoint')
checkpoint = torch.load(os.path.join(args.save_dir, 'model.pt'))
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
# dataset process
train_datasets = datasets.CIFAR10(root=args.data,
train=True,
transform=transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(32, 4),
transforms.ToTensor()
]),
download=True)
test_dataset = datasets.CIFAR10(root=args.data,
train=False,
transform=transforms.Compose(
[transforms.ToTensor()]),
download=True)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=2,
pin_memory=True)
valid_size = 0.1
indices = list(range(len(train_datasets)))
split = int(np.floor(valid_size * len(train_datasets)))
np.random.seed(args.seed)
np.random.shuffle(indices)
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
train_loader = torch.utils.data.DataLoader(train_datasets,
batch_size=args.batch_size,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(train_datasets,
batch_size=args.batch_size,
sampler=valid_sampler)
decreasing_lr = list(map(int, args.decreasing_lr.split(',')))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=decreasing_lr,
gamma=0.1)
print('starting adv training')
all_result = {}
train_acc = []
ta0 = []
ata0 = []
ta1 = []
ata1 = []
test_ta0 = []
test_ata0 = []
test_ta1 = []
test_ata1 = []
if os.path.exists(args.save_dir) is not True:
os.mkdir(args.save_dir)
for epoch in tqdm(range(args.epochs)):
if epoch < start_epoch:
scheduler.step()
continue
print(optimizer.state_dict()['param_groups'][0]['lr'])
acc, train_loss = train(args, train_loader, model, criterion,
optimizer, epoch)
writer.add_scalar('acc/train_acc', acc, epoch)
writer.add_scalar('loss/train_loss', train_loss, epoch)
# evaluate on validation set
tacc_clean, tloss_clean = validate(args, val_loader, model, criterion,
0)
writer.add_scalar('acc/tacc_clean', tacc_clean, epoch)
writer.add_scalar('loss/tloss_clean', tloss_clean, epoch)
atacc_clean, atloss_clean = validate_adv(args, val_loader, model,
criterion, 0)
writer.add_scalar('acc/atacc_clean', atacc_clean, epoch)
writer.add_scalar('loss/atloss_clean', atloss_clean, epoch)
tacc_adv, tloss_adv = validate(args, val_loader, model, criterion, 1)
writer.add_scalar('acc/tacc_adv', tacc_adv, epoch)
writer.add_scalar('loss/tloss_adv', tloss_adv, epoch)
atacc_adv, atloss_adv = validate_adv(args, val_loader, model,
criterion, 1)
writer.add_scalar('acc/atacc_adv', atacc_adv, epoch)
writer.add_scalar('loss/atloss_adv', atloss_adv, epoch)
# evaluate on test set
# clean branch
test_tacc_clean, test_tloss_clean = validate(args, test_loader, model,
criterion, 0)
writer.add_scalar('acc/test_tacc_clean', test_tacc_clean, epoch)
writer.add_scalar('loss/test_tloss_clean', test_tloss_clean, epoch)
test_atacc_clean, test_atloss_clean = validate_adv(
args, test_loader, model, criterion, 0)
writer.add_scalar('acc/test_atacc_clean', test_atacc_clean, epoch)
writer.add_scalar('loss/test_atloss_clean', test_atloss_clean, epoch)
# adv branch
test_tacc_adv, test_tloss_adv = validate(args, test_loader, model,
criterion, 1)
writer.add_scalar('acc/test_tacc_adv', test_tacc_adv, epoch)
writer.add_scalar('loss/test_tloss_adv', test_tloss_adv, epoch)
test_atacc_adv, test_atloss_adv = validate_adv(args, test_loader,
model, criterion, 1)
writer.add_scalar('acc/test_atacc_adv', test_atacc_adv, epoch)
writer.add_scalar('loss/test_atloss_adv', test_atloss_adv, epoch)
scheduler.step()
train_acc.append(acc)
ta0.append(tacc_clean)
ata0.append(atacc_clean)
ta1.append(tacc_adv)
ata1.append(atacc_adv)
test_ta0.append(test_tacc_clean)
test_ata0.append(test_atacc_clean)
test_ta1.append(test_tacc_adv)
test_ata1.append(test_atacc_adv)
# remember best prec@1 and save checkpoint
is_best = tacc_adv > best_prec1
best_prec1 = max(tacc_adv, best_prec1)
ata_is_best = atacc_adv > best_ata
best_ata = max(atacc_adv, best_ata)
cln_is_best = tacc_clean > cln_best_prec1
cln_best_prec1 = max(tacc_clean, cln_best_prec1)
if is_best:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'best_model.pt'))
if cln_is_best:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'clean_best_model.pt'))
if ata_is_best:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'ata_best_model.pt'))
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, 'model.pt'))
if epoch and epoch % 10 == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'ata_best_prec1': best_ata,
},
is_best,
filename=os.path.join(args.save_dir, f'model_epoch{epoch}.pt'))
all_result['train'] = train_acc
all_result['test_ta0'] = test_ta0
all_result['test_ata0'] = test_ata0
all_result['test_ta1'] = test_ta1
all_result['test_ata1'] = test_ata1
all_result['ta0'] = ta0
all_result['ata0'] = ata0
all_result['ta1'] = ta1
all_result['ata1'] = ata1
pickle.dump(all_result,
open(os.path.join(args.save_dir, 'result_c.pkl'), 'wb'))
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# import network
gen_net = eval("models_search." + args.gen_model +
".Generator")(args=args).cuda()
dis_net = eval("models_search." + args.dis_model +
".Discriminator")(args=args).cuda()
gen_net.set_arch(args.arch, cur_stage=2)
dis_net.cur_stage = 2
# weight init
def weights_init(m):
classname = m.__class__.__name__
if classname.find("Conv2d") != -1:
if args.init_type == "normal":
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == "orth":
nn.init.orthogonal_(m.weight.data)
elif args.init_type == "xavier_uniform":
nn.init.xavier_uniform(m.weight.data, 1.0)
else:
raise NotImplementedError("{} unknown inital type".format(
args.init_type))
elif classname.find("BatchNorm2d") != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()),
args.g_lr,
(args.beta1, args.beta2),
)
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()),
args.d_lr,
(args.beta1, args.beta2),
)
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == "cifar10":
fid_stat = "fid_stat/fid_stats_cifar10_train.npz"
elif args.dataset.lower() == "stl10":
fid_stat = "fid_stat/stl10_train_unlabeled_fid_stats_48.npz"
else:
raise NotImplementedError(f"no fid stat for {args.dataset.lower()}")
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
gen_avg_param = copy_params(gen_net)
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print(f"=> resuming from {args.load_path}")
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, "Model",
"checkpoint.pth")
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint["epoch"]
best_fid = checkpoint["best_fid"]
gen_net.load_state_dict(checkpoint["gen_state_dict"])
dis_net.load_state_dict(checkpoint["dis_state_dict"])
gen_optimizer.load_state_dict(checkpoint["gen_optimizer"])
dis_optimizer.load_state_dict(checkpoint["dis_optimizer"])
avg_gen_net = deepcopy(gen_net)
avg_gen_net.load_state_dict(checkpoint["avg_gen_state_dict"])
gen_avg_param = copy_params(avg_gen_net)
del avg_gen_net
args.path_helper = checkpoint["path_helper"]
logger = create_logger(args.path_helper["log_path"])
logger.info(
f"=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})")
else:
# create new log dir
assert args.exp_name
args.path_helper = set_log_dir("logs", args.exp_name)
logger = create_logger(args.path_helper["log_path"])
logger.info(args)
writer_dict = {
"writer": SummaryWriter(args.path_helper["log_path"]),
"train_global_steps": start_epoch * len(train_loader),
"valid_global_steps": start_epoch // args.val_freq,
}
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc="total progress"):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
train(
args,
gen_net,
dis_net,
gen_optimizer,
dis_optimizer,
gen_avg_param,
train_loader,
epoch,
writer_dict,
lr_schedulers,
)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat,
gen_net, writer_dict)
logger.info(
f"Inception score: {inception_score}, FID score: {fid_score} || @ epoch {epoch}."
)
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net = deepcopy(gen_net)
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
"epoch": epoch + 1,
"gen_model": args.gen_model,
"dis_model": args.dis_model,
"gen_state_dict": gen_net.state_dict(),
"dis_state_dict": dis_net.state_dict(),
"avg_gen_state_dict": avg_gen_net.state_dict(),
"gen_optimizer": gen_optimizer.state_dict(),
"dis_optimizer": dis_optimizer.state_dict(),
"best_fid": best_fid,
"path_helper": args.path_helper,
},
is_best,
args.path_helper["ckpt_path"],
)
del avg_gen_net
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# Create tensorboard logger
writer_dict = {
'writer': SummaryWriter(path_helper['log']),
'inner_steps': 0,
'val_steps': 0,
'valid_global_steps': 0
}
# set tf env
if args.eval:
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# fid_stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
elif args.dataset.lower() == 'mnist':
fid_stat = 'fid_stat/stl10_train_unlabeled_fid_stats_48.npz'
else:
raise NotImplementedError(f'no fid stat for {args.dataset.lower()}')
assert os.path.exists(fid_stat)
# initial
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
FID_best = 1e+4
IS_best = 0.
FID_best_epoch = 0
IS_best_epoch = 0
# build gen and dis
gen = eval('model_search_gan.' + args.gen)(args)
gen = gen.cuda()
dis = eval('model_search_gan.' + args.dis)(args)
dis = dis.cuda()
logging.info("generator param size = %fMB",
utils.count_parameters_in_MB(gen))
logging.info("discriminator param size = %fMB",
utils.count_parameters_in_MB(dis))
if args.parallel:
gen = nn.DataParallel(gen)
dis = nn.DataParallel(dis)
# resume training
if args.load_path != '':
gen.load_state_dict(
torch.load(
os.path.join(args.load_path, 'model',
'weights_gen_' + 'last' + '.pt')))
dis.load_state_dict(
torch.load(
os.path.join(args.load_path, 'model',
'weights_dis_' + 'last' + '.pt')))
# set optimizer for parameters W of gen and dis
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis.parameters()), args.d_lr,
(args.beta1, args.beta2))
# set moving average parameters for generator
gen_avg_param = copy_params(gen)
img_size = 8 if args.grow else args.img_size
train_transform, valid_transform = eval('utils.' + '_data_transforms_' +
args.dataset + '_resize')(args,
img_size)
if args.dataset == 'cifar10':
train_data = eval('dset.' + dataset[args.dataset])(
root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'stl10':
train_data = eval('dset.' + dataset[args.dataset])(
root=args.data, download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.gen_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.gen_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
logging.info('length of train_queue is {}'.format(len(train_queue)))
logging.info('length of valid_queue is {}'.format(len(valid_queue)))
max_iter = len(train_queue) * args.epochs
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
gen_optimizer, float(args.epochs), eta_min=args.learning_rate_min)
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
max_iter * args.n_critic)
architect = Architect_gen(gen, dis, args, 'duality_gap_with_mm', logging)
gen.set_gumbel(args.use_gumbel)
dis.set_gumbel(args.use_gumbel)
for epoch in range(args.start_epoch + 1, args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
logging.info('epoch %d gen_lr %e', epoch, args.g_lr)
logging.info('epoch %d dis_lr %e', epoch, args.d_lr)
genotype_gen = gen.genotype()
logging.info('gen_genotype = %s', genotype_gen)
if 'Discriminator' not in args.dis:
genotype_dis = dis.genotype()
logging.info('dis_genotype = %s', genotype_dis)
print('up_1: ', F.softmax(gen.alphas_up_1, dim=-1))
print('up_2: ', F.softmax(gen.alphas_up_2, dim=-1))
print('up_3: ', F.softmax(gen.alphas_up_3, dim=-1))
# determine whether use gumbel or not
if epoch == args.fix_alphas_epochs + 1:
gen.set_gumbel(args.use_gumbel)
dis.set_gumbel(args.use_gumbel)
# grow discriminator and generator
if args.grow:
dis.cur_stage = grow_ctrl(epoch, args.grow_epoch)
gen.cur_stage = grow_ctrl(epoch, args.grow_epoch)
if args.restrict_dis_grow and dis.cur_stage > 1:
dis.cur_stage = 1
print('debug: dis.cur_stage is {}'.format(dis.cur_stage))
if epoch in args.grow_epoch:
train_transform, valid_transform = utils._data_transforms_cifar10_resize(
args, 2**(gen.cur_stage + 3))
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.gen_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[:split]),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.gen_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
else:
gen.cur_stage = 2
dis.cur_stage = 2
# training parameters
train_gan_parameter(args, train_queue, gen, dis, gen_optimizer,
dis_optimizer, gen_avg_param, logging, writer_dict)
# training alphas
if epoch > args.fix_alphas_epochs:
train_gan_alpha(args, train_queue, valid_queue, gen, dis,
architect, gen_optimizer, gen_avg_param, epoch, lr,
writer_dict, logging)
# evaluate the IS and FID
if args.eval and epoch % args.eval_every == 0:
inception_score, std, fid_score = validate(args, fixed_z, fid_stat,
gen, writer_dict,
path_helper)
logging.info('epoch {}: IS is {}+-{}, FID is {}'.format(
epoch, inception_score, std, fid_score))
if inception_score > IS_best:
IS_best = inception_score
IS_epoch_best = epoch
if fid_score < FID_best:
FID_best = fid_score
FID_epoch_best = epoch
logging.info('best epoch {}: IS is {}'.format(
IS_best_epoch, IS_best))
logging.info('best epoch {}: FID is {}'.format(
FID_best_epoch, FID_best))
utils.save(
gen,
os.path.join(path_helper['model'],
'weights_gen_{}.pt'.format('last')))
utils.save(
dis,
os.path.join(path_helper['model'],
'weights_dis_{}.pt'.format('last')))
genotype_gen = gen.genotype()
if 'Discriminator' not in args.dis:
genotype_dis = dis.genotype()
logging.info('best epoch {}: IS is {}'.format(IS_best_epoch, IS_best))
logging.info('best epoch {}: FID is {}'.format(FID_best_epoch, FID_best))
logging.info('final discovered gen_arch is {}'.format(genotype_gen))
if 'Discriminator' not in args.dis:
logging.info('final discovered dis_arch is {}'.format(genotype_dis))
def main():
args = parse_args()
args.pretrain = False
root_path = 'exps/exp_{}'.format(args.exp)
if not os.path.exists(root_path):
os.mkdir(root_path)
os.mkdir(os.path.join(root_path, "log"))
os.mkdir(os.path.join(root_path, "model"))
base_lr = args.lr # base learning rate
train_dataset, val_dataset = build_dataset(args.dataset, args.data_root,
args.train_list)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
num_workers=args.num_workers,
pin_memory=True)
model = VNet(args.n_channels, args.n_classes).cuda()
optimizer = torch.optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=0.0005)
#scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.7)
model = torch.nn.DataParallel(model)
model.train()
if args.resume is None:
assert os.path.exists(args.load_path)
state_dict = model.state_dict()
print("Loading weights...")
pretrain_state_dict = torch.load(args.load_path,
map_location="cpu")['state_dict']
for k in list(pretrain_state_dict.keys()):
if k not in state_dict:
del pretrain_state_dict[k]
model.load_state_dict(pretrain_state_dict)
print("Loaded weights")
else:
print("Resuming from {}".format(args.resume))
checkpoint = torch.load(args.resume, map_location="cpu")
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
model.load_state_dict(checkpoint['state_dict'])
logger = Logger(root_path)
saver = Saver(root_path)
for epoch in range(args.start_epoch, args.epochs):
train(model, train_loader, optimizer, logger, args, epoch)
validate(model, val_loader, optimizer, logger, saver, args, epoch)
adjust_learning_rate(args, optimizer, epoch)
def main():
args = cfg.parse_args()
random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
torch.cuda.manual_seed(args.random_seed)
# set tf env
_init_inception()
inception_path = check_or_download_inception(None)
create_inception_graph(inception_path)
# weight init
gen_net = eval('models.' + args.model + '.Generator')(args=args)
dis_net = eval('models.' + args.model + '.Discriminator')(args=args)
# weight init
def weights_init(m):
if isinstance(m, nn.Conv2d):
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif isinstance(m, nn.BatchNorm2d):
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
gen_net = gen_net.cuda()
dis_net = dis_net.cuda()
avg_gen_net = deepcopy(gen_net)
initial_gen_net_weight = deepcopy(gen_net.state_dict())
initial_dis_net_weight = deepcopy(dis_net.state_dict())
assert id(initial_dis_net_weight) != id(dis_net.state_dict())
assert id(initial_gen_net_weight) != id(gen_net.state_dict())
# set optimizer
gen_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, gen_net.parameters()), args.g_lr,
(args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, dis_net.parameters()), args.d_lr,
(args.beta1, args.beta2))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0.0, 0,
args.max_iter * args.n_critic)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0.0, 0,
args.max_iter * args.n_critic)
# set up data_loader
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
# fid stat
if args.dataset.lower() == 'cifar10':
fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
elif args.dataset.lower() == 'stl10':
fid_stat = 'fid_stat/fid_stats_stl10_train.npz'
else:
raise NotImplementedError('no fid stat for %s' % args.dataset.lower())
assert os.path.exists(fid_stat)
# epoch number for dis_net
args.max_epoch = args.max_epoch * args.n_critic
if args.max_iter:
args.max_epoch = np.ceil(args.max_iter * args.n_critic /
len(train_loader))
# initial
np.random.seed(args.random_seed)
fixed_z = torch.cuda.FloatTensor(
np.random.normal(0, 1, (25, args.latent_dim)))
start_epoch = 0
best_fid = 1e4
args.path_helper = set_log_dir('logs',
args.exp_name + "_{}".format(args.percent))
logger = create_logger(args.path_helper['log_path'])
# logger.info('=> loaded checkpoint %s (epoch %d)' % (checkpoint_file, start_epoch))
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
print('=> resuming from %s' % args.load_path)
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model', 'checkpoint.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
gen_net.load_state_dict(checkpoint['gen_state_dict'])
torch.manual_seed(args.random_seed)
pruning_generate(gen_net, (1 - args.percent), args.pruning_method)
torch.manual_seed(args.random_seed)
pruning_generate(avg_gen_net, (1 - args.percent), args.pruning_method)
see_remain_rate(gen_net)
if args.second_seed:
dis_net.apply(weights_init)
if args.finetune_D:
dis_net.load_state_dict(checkpoint['dis_state_dict'])
else:
dis_net.load_state_dict(initial_dis_net_weight)
gen_weight = gen_net.state_dict()
gen_orig_weight = rewind_weight(initial_gen_net_weight, gen_weight.keys())
assert id(gen_weight) != id(gen_orig_weight)
gen_weight.update(gen_orig_weight)
gen_net.load_state_dict(gen_weight)
gen_avg_param = copy_params(gen_net)
if args.use_kd_D:
orig_dis_net = eval('models.' + args.model +
'.Discriminator')(args=args).cuda()
orig_dis_net.load(checkpoint['dis_state_dict'])
orig_dis_net.eval()
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.max_epoch)),
desc='total progress'):
lr_schedulers = (gen_scheduler,
dis_scheduler) if args.lr_decay else None
see_remain_rate(gen_net)
if not args.use_kd_D:
train(args, gen_net, dis_net, gen_optimizer, dis_optimizer,
gen_avg_param, train_loader, epoch, writer_dict,
lr_schedulers)
else:
train_kd(args, gen_net, dis_net, orig_dis_net, gen_optimizer,
dis_optimizer, gen_avg_param, train_loader, epoch,
writer_dict, lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == int(
args.max_epoch) - 1:
backup_param = copy_params(gen_net)
load_params(gen_net, gen_avg_param)
inception_score, fid_score = validate(args, fixed_z, fid_stat,
gen_net, writer_dict, epoch)
logger.info(
'Inception score: %.4f, FID score: %.4f || @ epoch %d.' %
(inception_score, fid_score, epoch))
load_params(gen_net, backup_param)
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
avg_gen_net.load_state_dict(gen_net.state_dict())
load_params(avg_gen_net, gen_avg_param)
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'avg_gen_state_dict': avg_gen_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
def main(index, args):
device = xm.xla_device()
gen_net = Generator(args).to(device)
dis_net = Discriminator(args).to(device)
enc_net = Encoder(args).to(device)
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
if args.init_type == 'normal':
nn.init.normal_(m.weight.data, 0.0, 0.02)
elif args.init_type == 'orth':
nn.init.orthogonal_(m.weight.data)
elif args.init_type == 'xavier_uniform':
nn.init.xavier_uniform(m.weight.data, 1.)
else:
raise NotImplementedError('{} unknown inital type'.format(
args.init_type))
elif classname.find('BatchNorm2d') != -1:
nn.init.normal_(m.weight.data, 1.0, 0.02)
nn.init.constant_(m.bias.data, 0.0)
gen_net.apply(weights_init)
dis_net.apply(weights_init)
enc_net.apply(weights_init)
ae_recon_optimizer = torch.optim.Adam(
itertools.chain(enc_net.parameters(), gen_net.parameters()),
args.ae_recon_lr, (args.beta1, args.beta2))
ae_reg_optimizer = torch.optim.Adam(
itertools.chain(enc_net.parameters(), gen_net.parameters()),
args.ae_reg_lr, (args.beta1, args.beta2))
dis_optimizer = torch.optim.Adam(dis_net.parameters(), args.d_lr,
(args.beta1, args.beta2))
gen_optimizer = torch.optim.Adam(gen_net.parameters(), args.g_lr,
(args.beta1, args.beta2))
dataset = datasets.ImageDataset(args)
train_loader = dataset.train
valid_loader = dataset.valid
para_loader = pl.ParallelLoader(train_loader, [device])
fid_stat = str(pathlib.Path(
__file__).parent.absolute()) + '/fid_stat/fid_stat_cifar10_test.npz'
if not os.path.exists(fid_stat):
download_stat_cifar10_test()
is_best = True
args.num_epochs = np.ceil(args.num_iter / len(train_loader))
gen_scheduler = LinearLrDecay(gen_optimizer, args.g_lr, 0,
args.num_iter / 2, args.num_iter)
dis_scheduler = LinearLrDecay(dis_optimizer, args.d_lr, 0,
args.num_iter / 2, args.num_iter)
ae_recon_scheduler = LinearLrDecay(ae_recon_optimizer, args.ae_recon_lr, 0,
args.num_iter / 2, args.num_iter)
ae_reg_scheduler = LinearLrDecay(ae_reg_optimizer, args.ae_reg_lr, 0,
args.num_iter / 2, args.num_iter)
# initial
start_epoch = 0
best_fid = 1e4
# set writer
if args.load_path:
print(f'=> resuming from {args.load_path}')
assert os.path.exists(args.load_path)
checkpoint_file = os.path.join(args.load_path, 'Model',
'checkpoint.pth')
assert os.path.exists(checkpoint_file)
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint['epoch']
best_fid = checkpoint['best_fid']
gen_net.load_state_dict(checkpoint['gen_state_dict'])
enc_net.load_state_dict(checkpoint['enc_state_dict'])
dis_net.load_state_dict(checkpoint['dis_state_dict'])
gen_optimizer.load_state_dict(checkpoint['gen_optimizer'])
dis_optimizer.load_state_dict(checkpoint['dis_optimizer'])
ae_recon_optimizer.load_state_dict(checkpoint['ae_recon_optimizer'])
ae_reg_optimizer.load_state_dict(checkpoint['ae_reg_optimizer'])
args.path_helper = checkpoint['path_helper']
logger = create_logger(args.path_helper['log_path'])
logger.info(
f'=> loaded checkpoint {checkpoint_file} (epoch {start_epoch})')
else:
# create new log dir
assert args.exp_name
logs_dir = str(pathlib.Path(__file__).parent.parent) + '/logs'
args.path_helper = set_log_dir(logs_dir, args.exp_name)
logger = create_logger(args.path_helper['log_path'])
logger.info(args)
writer_dict = {
'writer': SummaryWriter(args.path_helper['log_path']),
'train_global_steps': start_epoch * len(train_loader),
'valid_global_steps': start_epoch // args.val_freq,
}
# train loop
for epoch in tqdm(range(int(start_epoch), int(args.num_epochs)),
desc='total progress'):
lr_schedulers = (gen_scheduler, dis_scheduler, ae_recon_scheduler,
ae_reg_scheduler)
train(device, args, gen_net, dis_net, enc_net, gen_optimizer,
dis_optimizer, ae_recon_optimizer, ae_reg_optimizer, para_loader,
epoch, writer_dict, lr_schedulers)
if epoch and epoch % args.val_freq == 0 or epoch == args.num_epochs - 1:
fid_score = validate(args, fid_stat, gen_net, writer_dict,
valid_loader)
logger.info(f'FID score: {fid_score} || @ epoch {epoch}.')
if fid_score < best_fid:
best_fid = fid_score
is_best = True
else:
is_best = False
else:
is_best = False
save_checkpoint(
{
'epoch': epoch + 1,
'gen_state_dict': gen_net.state_dict(),
'dis_state_dict': dis_net.state_dict(),
'enc_state_dict': enc_net.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'dis_optimizer': dis_optimizer.state_dict(),
'ae_recon_optimizer': ae_recon_optimizer.state_dict(),
'ae_reg_optimizer': ae_reg_optimizer.state_dict(),
'best_fid': best_fid,
'path_helper': args.path_helper
}, is_best, args.path_helper['ckpt_path'])
def main():
args = parse_args()
args.pretrain = False
print("Using GPU: {}".format(args.local_rank))
root_path = 'exps/exp_{}'.format(args.exp)
if args.local_rank == 0 and not os.path.exists(root_path):
os.mkdir(root_path)
os.mkdir(os.path.join(root_path, "log"))
os.mkdir(os.path.join(root_path, "model"))
base_lr = args.lr # base learning rate
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
train_dataset, val_dataset = build_dataset(args.dataset, args.data_root,
args.train_list)
args.world_size = len(args.gpu.split(","))
if args.world_size > 1:
os.environ['MASTER_PORT'] = args.port
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group('nccl')
device = torch.device('cuda:{}'.format(args.local_rank))
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset,
num_replicas=len(args.gpu.split(",")),
rank=args.local_rank)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
sampler=train_sampler,
num_workers=args.num_workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
num_workers=args.num_workers,
pin_memory=True)
model = VNet(args.n_channels, args.n_classes).cuda(args.local_rank)
optimizer = torch.optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=0.0005)
#scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.7)
if args.world_size > 1:
model = DDP(model,
device_ids=[args.local_rank],
output_device=args.local_rank,
find_unused_parameters=True)
model.train()
print("Loaded weights")
logger = Logger(root_path)
saver = Saver(root_path)
for epoch in range(args.start_epoch, args.epochs):
train(model, train_loader, optimizer, logger, args, epoch)
validate(model, val_loader, optimizer, logger, saver, args, epoch)
adjust_learning_rate(args, optimizer, epoch)
