def shallow(inputs, name, units, verbose=1):
with tf.variable_scope(name):
hidden = tf.layers.dense(inputs,
units,
activation=tf.nn.relu,
name='hidden')
outputs = tf.layers.dense(hidden, 1, name='outputs')
if verbose: summary(name)
return outputs
def activeUnsL(model, node, label, features, adj_lists, num_features, num_hidden, num_cls, filetime,labels, xi=1e-6, eps=2.5, num_iters=10):
#obtain the adj matrix and find the best perturbation direction then add perturbation to the attention matrix
encSpc = model(node, actE = True)
dec_ae = DEC_AE(50, 100, num_hidden)
dec = DEC(num_cls, 50, dec_ae)
kmeans = KMeans(n_clusters=dec.cluster_number, n_init=20)
features = []
# form initial cluster centres
dec.pretrain(encSpc.data)
features = dec.ae.encoder(encSpc).detach()
predicted = kmeans.fit_predict(features)
predicted_previous = torch.tensor(np.copy(predicted), dtype=torch.long)
_, accuracy, _, _ = cluster_accuracy( label, predicted)
print("ACCU", accuracy)
cluster_centers = torch.tensor(kmeans.cluster_centers_, dtype=torch.float)
#print(features)
dec.assignment.cluster_centers = torch.nn.Parameter(cluster_centers)
loss_function = nn.KLDivLoss(size_average=False)
delta_label = None
optimizer = torch.optim.SGD(dec.parameters(), lr = 0.01, momentum=0.9)
for epoch in range(250):
dec.train()
output = dec(encSpc)
target = target_distribution(output).detach()
loss = loss_function(output.log(), target) / output.shape[0]
optimizer.zero_grad()
loss.backward(retain_graph=True)
optimizer.step(closure=None)
features = dec.ae.encoder(encSpc).detach()
#predicted = dec(test)
predicted = output.argmax(dim = 1)
delta_label = float((predicted != predicted_previous ).float().sum().item()) / predicted_previous.shape[0]
predicted_previous = predicted
_, accuracy, _, _ = cluster_accuracy(np.array(predicted), np.array(label))
if epoch % 50 == 49:
count_matrix = np.zeros((num_cls, num_cls), dtype=np.int64)
for i in range(len(predicted)):
count_matrix[np.array(predicted)[i], np.array(label)[i]] += 1
for i in range(num_cls):
print(count_matrix[i])
summary(node, labels, np.array(predicted), num_cls, filetime, outlog = False, output=None)
print(loss)
print("ACCU", accuracy)
def rand_crop(image,size):
real32 = tf.map_fn(lambda image: tf.random_crop(image,size = [32, 32, 3]), real)
fake32 = tf.map_fn(lambda image: tf.random_crop(image,size = [32, 32, 3]), fake)
r_logit32 = discriminator32(real32, reuse=False)
f_logit32 = discriminator32(fake32)
# losses
def gradient_penalty(real, fake, f):
def interpolate(a, b):
shape = tf.concat((tf.shape(a)[0:1], tf.tile([1], [a.shape.ndims - 1])), axis=0)
alpha = tf.random_uniform(shape=shape, minval=0., maxval=1.)
inter = a + alpha * (b - a)
inter.set_shape(a.get_shape().as_list())
return inter
x = interpolate(real, fake)
pred = f(x)
gradients = tf.gradients(pred, x)[0]
slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=list(range(1, x.shape.ndims))))
gp = tf.reduce_mean((slopes - 1.)**2)
return gp
wd = tf.reduce_mean(r_logit) - tf.reduce_mean(f_logit)
gp = gradient_penalty(real, fake, discriminator)
d_loss = -wd + gp * 10.0
g_loss = -tf.reduce_mean(f_logit)
wd32 = tf.reduce_mean(r_logit32) - tf.reduce_mean(f_logit32)
gp32 = gradient_penalty(real32, fake32, discriminator32)
d_loss32 = -wd32 + gp32 * 10.0
g_loss += -tf.reduce_mean(f_logit32)
# otpims
d_var = utils.trainable_variables('discriminator')
d32_var = utils.trainable_variables('discriminator32')
g_var = utils.trainable_variables('generator')
d_step = tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5).minimize(d_loss, var_list=d_var)
d32_step = tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5).minimize(d_loss, var_list=d32_var)
g_step = tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5).minimize(g_loss, var_list=g_var)
# summaries
d_summary = utils.summary({wd: 'wd', gp: 'gp'})
d32_summary = utils.summary({wd32: 'wd32', gp32: 'gp32'})
g_summary = utils.summary({g_loss: 'g_loss'})
# sample
f_sample = generator(z, training=False)
def load_model(state, default_model_names, customized_models_names, use_cuda):
if state['arch'] in default_model_names:
if state['pretrained']:
print("=> using pre-trained model '{}'".format(state['arch']))
model = models.__dict__[state['arch']](pretrained=True)
# resnet和densenet的最后一层名字不同
if 'resnet' in state['arch']:
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, state['num_classes'])
elif 'densenet' in state['arch']:
num_ftrs = model.classifier.in_features
model.classifier = nn.Linear(num_ftrs, state['num_classes'])
else:
print("=> creating model '{}'".format(state['arch']))
model = models.__dict__[state['arch']](
num_classes=state['num_classes'])
elif state['arch'].startswith('resnext') or state['arch'].startswith(
'se_resnext'):
print("=> creating model '{}'".format(state['arch']))
model = customized_models.__dict__[state['arch']](
baseWidth=state['base_width'],
cardinality=state['cardinality'],
num_class=state['num_classes'])
else:
raise 'model {} is not supported! Please choose model form {}'.format(
state['arch'], default_model_names + customized_models_names)
if use_cuda:
if state['arch'].startswith('alexnet') or state['arch'].startswith(
'vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
# 输出网络信息
fout = open(os.path.join(state['checkpoint'], 'out.txt'), 'w')
summary(model, (3, state['image_size'], state['image_size']),
print_fn=lambda x: fout.write(x + '\n'))
num_params = 'Total params: %.2fM' % (sum(p.numel()
for p in model.parameters()) /
1000000.0)
print(num_params)
fout.write(num_params + '\n')
fout.flush()
fout.close()
return model
def cnn(inputs, name, units, verbose=1):
with tf.variable_scope(name):
inputs = tf.reshape(inputs, (-1, 28, 28, 1), name='inputs')
conv1 = tf.layers.conv2d(inputs,
units, (3, 3),
padding='same',
activation=tf.nn.relu,
name='conv1')
conv2 = tf.layers.conv2d(conv1,
units, (3, 3),
padding='same',
activation=tf.nn.relu,
name='conv2')
flatten = tf.reshape(conv2, (-1, np.prod(conv2.shape.as_list()[1:])),
name='flatten')
outputs = tf.layers.dense(flatten, 10, name='outputs')
if verbose: summary(name)
return outputs
def main(args, **model_kwargs):
device = torch.device(args.device)
args.device = device
if args.dataset == 'abilene_tm':
args.nNodes = 12
args.day_size = 288
elif args.dataset == 'geant_tm':
args.nNodes = 22
args.day_size = 96
elif args.dataset == 'brain_tm':
args.nNodes = 9
elif 'sinet' in args.dataset:
args.nNodes = 73
args.day_size = 288
else:
raise ValueError('Dataset not found!')
test_loader = utils.get_dataloader(args)
args.test_size, args.nSeries = test_loader.dataset.gt_data_set.shape
in_dim = 1
args.in_dim = in_dim
model = models.get_model(args)
logger = utils.Logger(args)
engine = utils.Trainer.from_args(model, test_loader.dataset.scaler, args)
utils.print_args(args)
if not args.test:
test_met_df, x_gt, y_gt, y_real, yhat = engine.test(
test_loader, engine.model, args.out_seq_len)
test_met_df.round(6).to_csv(
os.path.join(logger.log_dir, 'test_metrics.csv'))
print('Prediction Accuracy:')
print(utils.summary(logger.log_dir))
np.save(os.path.join(logger.log_dir, 'x_gt'), x_gt)
np.save(os.path.join(logger.log_dir, 'y_gt'), y_gt)
np.save(os.path.join(logger.log_dir, 'y_real'), y_real)
np.save(os.path.join(logger.log_dir, 'yhat'), yhat)
else:
x_gt = np.load(os.path.join(logger.log_dir, 'x_gt.npy'))
y_gt = np.load(os.path.join(logger.log_dir, 'y_gt.npy'))
y_real = np.load(os.path.join(logger.log_dir, 'y_real.npy'))
yhat = np.load(os.path.join(logger.log_dir, 'yhat.npy'))
if args.plot:
logger.plot(x_gt, y_real, yhat)
# run TE
if args.run_te:
run_te(x_gt, y_gt, yhat, args)
def main(args):
root = args.path
if not check(root):
raise ValueError("Warning! Root directory isn't valid")
max_dep = args.depth
max_file = args.file
assert max_dep > 0 and max_file > 0
print("{Visual Tree}")
dir_count = 0
file_count = 0
for path, _, files in sorted(filtered_walk(root, depth=max_dep)):
level = offset(root, path)
base = os.path.basename(path)
if level == 1:
print(path)
else:
print(INDENT * level + SYMBOL + base)
dir_count += 1
# display files
level += 1
local_count = 0
display = True
for file in sorted(files):
file_count += 1
local_count += 1
if display:
print(INDENT * level + SYMBOL + file)
if local_count == max_file + 1:
display = False
print(INDENT * level + SYMBOL + "...")
summary(root, dir_count, file_count)
def __init__(self, args, manager, config, pretrain=False, inference=False):
self.disc_type = args.gan_type
if self.disc_type == 'vanilla':
self.irl = DISC_Vanilla(config, args.gamma).to(device=DEVICE)
elif self.disc_type == 'wgan':
self.irl = DISC_MTSA(config, args.gamma).to(device=DEVICE)
else:
raise ValueError(" No such type: {}".format(self.disc_type))
logging.info(summary(self.irl, show_weights=False))
self.net_type = args.irl_net
self.train_direc = args.train_direc
self.mt_factor = args.mt_factor
self.action_lambda = 0
self.bce_loss = nn.BCEWithLogitsLoss()
self.multi_entropy_loss = nn.MultiLabelSoftMarginLoss()
self.loss_BCE = nn.BCELoss()
self.step = 0
self.anneal = args.anneal
self.irl_params = self.irl.parameters()
self.irl_optim = optim.Adam(self.irl_params, lr=args.lr_irl)
self.weight_cliping_limit = args.clip
self.save_dir = args.save_dir
self.save_per_epoch = args.save_per_epoch
self.optim_batchsz = args.batchsz
self.irl.eval()
db = DBQuery(args.data_dir)
if pretrain:
self.print_per_batch = args.print_per_batch
self.data_train = manager.create_dataset_irl(
'train', args.batchsz, config, db)
self.data_valid = manager.create_dataset_irl(
'valid', args.batchsz, config, db)
self.data_test = manager.create_dataset_irl(
'test', args.batchsz, config, db)
self.irl_iter = iter(self.data_train)
self.irl_iter_valid = iter(self.data_valid)
self.irl_iter_test = iter(self.data_test)
elif not inference:
self.data_train = manager.create_dataset_irl(
'train', args.batchsz, config, db)
self.data_valid = manager.create_dataset_irl(
'valid', args.batchsz, config, db)
self.irl_iter = iter(self.data_train)
self.irl_iter_valid = iter(self.data_valid)
def trade(request):
OC_trades = match.OC_trades
NE_trades = match.NE_trades
# trade exceptions
unmatched_instruments = set([t[0] for t in OC_trades] + [t[0] for t in NE_trades])
trade_exceptions = [
[
i,
filter(lambda x: x[0] == i, OC_trades),
filter(lambda x: x[0] == i, NE_trades),
summary(i, OC_trades),
summary(i, NE_trades),
]
for i in unmatched_instruments
]
trade_exceptions.sort(cmp_instruments)
# matched legs
matched_legs = [
[
i,
filter(lambda x: x[0] == i, match.OC_matched_legs),
filter(lambda x: x[0] == i, match.NE_matched_legs),
summary(i, match.OC_matched_legs),
summary(i, match.NE_matched_legs),
]
for i in set([t[0] for t in match.OC_matched_legs])
]
matched_legs.sort(cmp_instruments)
# matched split quantity/price single legs
matched_split_qty = [
[
i,
filter(lambda x: x[0] == i, match.OC_matched_split_qty),
filter(lambda x: x[0] == i, match.NE_matched_split_qty),
summary(i, match.OC_matched_split_qty),
summary(i, match.NE_matched_split_qty),
]
for i in set([t[0] for t in match.OC_matched_split_qty])
]
matched_split_qty.sort(cmp_instruments)
# exact matches
exact_matches = match.exact_matches
template_name = "trade.html"
return direct_to_template(request, template_name, locals())
def build(batch_size):
gp_lambda = 10
""" graphs """
with tf.device('/gpu:0'):
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0, beta2 = 0.99)
''' graph definition '''
''' declare all of variables to be used with dummy '''
alpha = tf.placeholder(tf.float32, shape=[1])
real = tf.placeholder(tf.float32, shape=[batch_size, target_size, target_size, 3])
real = (1.-alpha)*down_up(real) + alpha * real
z1 = tf.placeholder(tf.float32, shape=[batch_size, z_dim])
z2 = tf.placeholder(tf.float32, shape=[batch_size, z_dim])
real_dummy = tf.placeholder(tf.float32, shape=[2, final_size, final_size, 3])
z1_dummy = tf.placeholder(tf.float32, shape=[2, z_dim])
z2_dummy = tf.placeholder(tf.float32, shape=[2, z_dim])
fake1 = generator(z1, target_size, alpha)
fake2 = generator(z2, target_size, alpha)
fake1_dummy = generator(z1_dummy, final_size, alpha)
fake2_dummy = generator(z2_dummy, final_size, alpha)
r_logit = discriminator(real, alpha)
f1_logit = discriminator(fake1, alpha)
f2_logit = discriminator(fake2, alpha)
r_logit_dummy = discriminator(real_dummy, alpha)
f1_logit_dummy = discriminator(fake1_dummy, alpha)
f2_logit_dummy = discriminator(fake2_dummy, alpha)
# Cramer GAN losses
def L2(logit):
ret = tf.sqrt( tf.reduce_sum( tf.square( logit ), axis = 1 ) )
return ret
def critic(logit, logitd):
return tf.reduce_mean(L2( logit - logitd ) - L2( logit ))
def gradient_penalty(bsz, real, fake, logitd):
def interpolate(a, b):
rg = tf.random_uniform(shape=[bsz,1,1,1], minval=0., maxval=1.)
return rg * a + (1.-rg)*b
x = interpolate(real, fake)
pred = discriminator(x, alpha)
pred = L2( pred - logitd ) - L2( pred )
grad = tf.reshape(tf.gradients(pred, x)[0], shape=[bsz, -1])
# when P_fake ~= P_real, for almost all x in the distribution has 1 L2-norm
return tf.reduce_mean((L2(grad)-1)**2.0)
g_loss = tf.reduce_mean(L2(r_logit - f1_logit) + L2( r_logit - f2_logit ) - L2( f1_logit - f2_logit ))
g_surr = critic(r_logit, f2_logit) - critic( f1_logit, f2_logit )
gp = gp_lambda * gradient_penalty(batch_size, real, fake1, f2_logit)
d_loss = -g_surr + gp
g_loss_dummy = tf.reduce_mean(L2(r_logit_dummy - f1_logit_dummy) + L2( r_logit_dummy - f2_logit_dummy ) - L2( f1_logit_dummy - f2_logit_dummy ))
g_surr_dummy = critic(r_logit_dummy, f2_logit_dummy) - critic( f1_logit_dummy, f2_logit_dummy )
gp_dummy = gp_lambda * gradient_penalty(2, real_dummy, fake1_dummy, f2_logit_dummy)
d_loss_dummy = -g_surr_dummy + gp_dummy
g_vars = []
d_vars = []
for vari in tf.trainable_variables():
if vari.name.startswith('x/g'):
g_vars.append(vari)
if vari.name.startswith('x/d'):
d_vars.append(vari)
# otpims
g_step = optimizer.minimize(g_loss, var_list=g_vars)
d_step = optimizer.minimize(d_loss, var_list=d_vars)
g_step_dummy = optimizer.minimize(g_loss_dummy, var_list=g_vars)
d_step_dummy = optimizer.minimize(d_loss_dummy, var_list=d_vars)
# summaries
g_summary = utils.summary({g_loss: 'g_loss'})
d_summary = utils.summary({d_loss: 'd_loss', gp : 'grad_penal'})#, wd: 'wd', gp: 'gp', rlm:'rlm'})
# sample
f_sample = generator(z1, target_size, alpha)
return {\
'dummy' : { 'd' : d_step_dummy, 'g' : g_step_dummy,\
'input' : { 'real' : real_dummy, 'z1' : z1_dummy, 'z2' : z2_dummy, 'alpha' : alpha } },\
'product' : { 'd' : d_step, 'g' : g_step,\
'input' : { 'real' : real, 'z1' : z1, 'z2' : z2, 'alpha' : alpha } },\
'sample' : f_sample, 'summaries' : {'d' : d_summary, 'g': g_summary } }
def train(ctx):
if isinstance(ctx, mx.Context):
ctx = [ctx]
if opt.resume_params == '':
net.initialize(mx.init.MSRAPrelu(), ctx=ctx)
if opt.summary:
# net.summary(mx.nd.zeros((1, 3, opt.input_size, opt.input_size), ctx=ctx[0]))
summary(net, mx.nd.zeros((1, 3, opt.input_size, opt.input_size), ctx=ctx[0]))
sys.exit()
if opt.no_wd:
for k, v in net.collect_params('.*beta|.*gamma|.*bias').items():
v.wd_mult = 0.0
trainer = gluon.Trainer(net.collect_params(), optimizer, optimizer_params)
if opt.resume_states != '':
trainer.load_states(opt.resume_states)
if opt.label_smoothing or opt.mixup:
sparse_label_loss = False
else:
sparse_label_loss = True
if distillation:
L = gcv.loss.DistillationSoftmaxCrossEntropyLoss(temperature=opt.temperature,
hard_weight=opt.hard_weight,
sparse_label=sparse_label_loss)
else:
L = gluon.loss.SoftmaxCrossEntropyLoss(sparse_label=sparse_label_loss)
best_val_score = 1
for epoch in range(opt.resume_epoch, opt.num_epochs):
tic = time.time()
if opt.use_rec:
train_data.reset()
train_metric.reset()
btic = time.time()
for i, batch in enumerate(train_data):
data, label = batch_fn(batch, ctx)
if opt.mixup:
lam = np.random.beta(opt.mixup_alpha, opt.mixup_alpha)
if epoch >= opt.num_epochs - opt.mixup_off_epoch:
lam = 1
data = [lam*X + (1-lam)*X[::-1] for X in data]
if opt.label_smoothing:
eta = 0.1
else:
eta = 0.0
label = mixup_transform(label, classes, lam, eta)
elif opt.label_smoothing:
hard_label = label
label = smooth(label, classes)
if distillation:
teacher_prob = [nd.softmax(teacher(X.astype(opt.dtype, copy=False)) / opt.temperature) \
for X in data]
with ag.record():
outputs = [net(X.astype(opt.dtype, copy=False)) for X in data]
if distillation:
loss = [L(yhat.astype('float32', copy=False),
y.astype('float32', copy=False),
p.astype('float32', copy=False)) for yhat, y, p in zip(outputs, label, teacher_prob)]
else:
loss = [L(yhat, y.astype(opt.dtype, copy=False)) for yhat, y in zip(outputs, label)]
for l in loss:
l.backward()
trainer.step(batch_size)
if opt.mixup:
output_softmax = [nd.SoftmaxActivation(out.astype('float32', copy=False)) \
for out in outputs]
train_metric.update(label, output_softmax)
else:
if opt.label_smoothing:
train_metric.update(hard_label, outputs)
else:
train_metric.update(label, outputs)
if opt.log_interval and not (i+1)%opt.log_interval:
train_metric_name, train_metric_score = train_metric.get()
logger.info('Epoch[%d] Batch [%d]\tSpeed: %f samples/sec\t%s=%f\tlr=%f'%(
epoch, i, batch_size*opt.log_interval/(time.time()-btic),
train_metric_name, train_metric_score, trainer.learning_rate))
btic = time.time()
train_metric_name, train_metric_score = train_metric.get()
throughput = int(batch_size * i /(time.time() - tic))
err_top1_val, err_top5_val = test(ctx, val_data)
logger.info('[Epoch %d] training: %s=%f'%(epoch, train_metric_name, train_metric_score))
logger.info('[Epoch %d] speed: %d samples/sec\ttime cost: %f'%(epoch, throughput, time.time()-tic))
logger.info('[Epoch %d] validation: err-top1=%f err-top5=%f'%(epoch, err_top1_val, err_top5_val))
if err_top1_val < best_val_score:
best_val_score = err_top1_val
net.save_parameters('%s/%.4f-imagenet-%s-%d-best.params'%(save_dir, best_val_score, model_name, epoch))
trainer.save_states('%s/%.4f-imagenet-%s-%d-best.states'%(save_dir, best_val_score, model_name, epoch))
if save_frequency and save_dir and (epoch + 1) % save_frequency == 0:
net.save_parameters('%s/imagenet-%s-%d.params'%(save_dir, model_name, epoch))
trainer.save_states('%s/imagenet-%s-%d.states'%(save_dir, model_name, epoch))
if save_frequency and save_dir:
net.save_parameters('%s/imagenet-%s-%d.params'%(save_dir, model_name, opt.num_epochs-1))
trainer.save_states('%s/imagenet-%s-%d.states'%(save_dir, model_name, opt.num_epochs-1))
def main(args, **model_kwargs):
device = torch.device(args.device)
args.device = device
if args.dataset == 'abilene_tm':
args.nNodes = 12
args.day_size = 288
elif args.dataset == 'geant_tm':
args.nNodes = 22
args.day_size = 96
elif args.dataset == 'brain_tm':
args.nNodes = 9
args.day_size = 1440
elif 'sinet' in args.dataset:
args.nNodes = 73
args.day_size = 288
else:
raise ValueError('Dataset not found!')
train_loader, val_loader, test_loader, graphs, top_k_index = utils.get_dataloader(
args)
args.train_size, args.nSeries = train_loader.dataset.X.shape
args.val_size = val_loader.dataset.X.shape[0]
args.test_size = test_loader.dataset.X.shape[0]
in_dim = 1
if args.tod:
in_dim += 1
if args.ma:
in_dim += 1
if args.mx:
in_dim += 1
args.in_dim = in_dim
model = models.get_model(args)
logger = utils.Logger(args)
engine = utils.Trainer.from_args(model, train_loader.dataset.scaler, \
train_loader.dataset.scaler_top_k, args)
utils.print_args(args)
if not args.test:
iterator = trange(args.epochs)
try:
if os.path.isfile(logger.best_model_save_path):
print('Model checkpoint exist!')
print('Load model checkpoint? (y/n)')
_in = input()
if _in == 'y' or _in == 'yes':
print('Loading model...')
engine.model.load_state_dict(
torch.load(logger.best_model_save_path))
else:
print('Training new model')
for epoch in iterator:
train_loss, train_rse, train_mae, train_mse, train_mape, train_rmse = [], [], [], [], [], []
for iter, batch in enumerate(train_loader):
# x = batch['x'] # [b, seq_x, n, f]
# y = batch['y'] # [b, seq_y, n]
x = batch['x_top_k']
y = batch['y_top_k']
if y.max() == 0: continue
loss, rse, mae, mse, mape, rmse = engine.train(x, y)
train_loss.append(loss)
train_rse.append(rse)
train_mae.append(mae)
train_mse.append(mse)
train_mape.append(mape)
train_rmse.append(rmse)
engine.scheduler.step()
with torch.no_grad():
val_loss, val_rse, val_mae, val_mse, val_mape, val_rmse = engine.eval(
val_loader)
m = dict(train_loss=np.mean(train_loss),
train_rse=np.mean(train_rse),
train_mae=np.mean(train_mae),
train_mse=np.mean(train_mse),
train_mape=np.mean(train_mape),
train_rmse=np.mean(train_rmse),
val_loss=np.mean(val_loss),
val_rse=np.mean(val_rse),
val_mae=np.mean(val_mae),
val_mse=np.mean(val_mse),
val_mape=np.mean(val_mape),
val_rmse=np.mean(val_rmse))
description = logger.summary(m, engine.model)
if logger.stop:
break
description = 'Epoch: {} '.format(epoch) + description
iterator.set_description(description)
except KeyboardInterrupt:
pass
# Metrics on test data
engine.model.load_state_dict(torch.load(logger.best_model_save_path))
with torch.no_grad():
test_met_df, x_gt, y_gt, y_real, yhat = engine.test(
test_loader, engine.model, args.out_seq_len)
test_met_df.round(6).to_csv(
os.path.join(logger.log_dir, 'test_metrics.csv'))
print('Prediction Accuracy:')
print(utils.summary(logger.log_dir))
if args.plot:
logger.plot(x_gt, y_real, yhat)
x_gt = x_gt.cpu().data.numpy() # [timestep, seq_x, seq_y]
y_gt = y_gt.cpu().data.numpy()
yhat = yhat.cpu().data.numpy()
# run TE
if args.run_te:
psi = get_psi(args)
G = get_G(args)
R = get_R(args)
A = np.dot(R * G, psi)
y_cs = np.zeros(y_gt.shape)
# for i in range(y_gt.shape[0]):
# temp = np.linalg.inv(np.dot(A, A.T))
# S = np.dot(np.dot(A.T, temp), yhat[i].T)
# y_cs[i] = np.dot(psi, S).T
for i in range(y_gt.shape[0]):
m = A.shape[1]
S = cvx.Variable(m)
objective = cvx.Minimize(cvx.norm(S, p=0))
constraint = [yhat[i].T == A * S]
prob = cvx.Problem(objective, constraint)
prob.solve()
y_cs[i] = S.value.reshape(1, m)
run_te(x_gt, y_gt, y_cs, args)
plt.vlines(x=df_temp1['LOW_SPEC'][0], ymin=0, ymax=max(counts), color='red',label='Lower specification')
if str(df_temp1['HIGH_SPEC'][0]) != 'nan':
plt.vlines(x=df_temp1['HIGH_SPEC'][0], ymin=0, ymax=max(counts), color='red',label='Higher specification')
plt.grid()
plt.legend()
plt.savefig('../infoMatcode/' + folder + '/' + str(anal) + '/brut/' + 'distribution.jpg')
# Necessary statistics and data cleaning:
nb_points = int(len(df_temp1['FINAL']))
nb_batchs = int(len(np.unique(df_temp1['BATCH'])))
low_spec = df_temp1['LOW_SPEC'][0]
high_spec = df_temp1['HIGH_SPEC'][0]
min_val = df_temp1['FINAL'].min()
max_val = df_temp1['FINAL'].max()
Low_tuck,Upp_tuck,mean,std = utils.summary(df_temp1['FINAL'].tolist())
df_out_sup = df_temp1[df_temp1.FINAL > (mean + 5*std)]
df_out_inf = df_temp1[df_temp1.FINAL < (mean - 5*std)]
df_out = pd.concat([df_out_sup,df_out_inf])
nb_outliers = float(len(df_out))
rate = round(float((nb_outliers/nb_points))*100,3)
df_concat = pd.concat([df_temp1,df_out])
df_temp1 = df_concat.drop_duplicates(keep = False)
new_mean = round(np.mean(df_temp1['FINAL'].tolist()),3)
nb_new_data = float(len(df_temp1))
# Write statistics in a file:
file1 = open('../infoMatcode/' + folder + '/' + str(anal) + '/' + 'statistics.txt',"w")
L = ["Number of material tested : "+str(nb_points)+"\n",
"Number of batch tested : "+str(nb_batchs)+"\n\n",
"Minimum value of tests : "+str(min_val)+"\n",
gp = tf.reduce_mean((slopes - 1.)**2)
return gp
wd = tf.reduce_mean(r_logit) - tf.reduce_mean(f_logit)
gp = gradient_penalty(real, fake, discriminator)
d_loss = -wd + gp * 10.0
g_loss = -tf.reduce_mean(f_logit)
# otpims
d_var = utils.trainable_variables('discriminator')
g_var = utils.trainable_variables('generator')
d_step = tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5).minimize(d_loss, var_list=d_var)
g_step = tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5).minimize(g_loss, var_list=g_var)
# summaries
d_summary = utils.summary({wd: 'wd', gp: 'gp', d_loss: 'd_loss'})
g_summary = utils.summary({g_loss: 'g_loss'})
# sample
f_sample = generator(z, training=False)
""" train """
''' init '''
# session
sess = utils.session()
# iteration counter
it_cnt, update_cnt = utils.counter()
# saver
saver = tf.train.Saver(max_to_keep=5)
# summary writer
def classify(train, examples):
cv_res = {
"PP": 0,
"PN": 0,
"NP": 0,
"NN": 0,
"contradictory": 0,
}
plus = train["plus"]
minus = train["minus"]
l = len(examples)
i = 0
for elem in examples:
i += 1
print "%i/%i" % (i, l)
result = check_hypothesis(plus, minus, elem)
cv_res[result] += 1
return cv_res
if __name__ == "__main__":
index = int(sys.argv[1])
train = utils.load_train(index)
test = utils.load_test(index)
res = classify(train, test)
print res
print utils.summary(res)
def main(args, **model_kwargs):
device = torch.device(args.device)
args.device = device
if 'abilene' in args.dataset:
args.nNodes = 12
args.day_size = 288
elif 'geant' in args.dataset:
args.nNodes = 22
args.day_size = 96
elif 'brain' in args.dataset:
args.nNodes = 9
args.day_size = 1440
elif 'sinet' in args.dataset:
args.nNodes = 74
args.day_size = 288
else:
raise ValueError('Dataset not found!')
train_loader, val_loader, test_loader, top_k_index = utils.get_dataloader(
args)
args.train_size, args.nSeries = train_loader.dataset.X_scaled_top_k.shape
args.val_size = val_loader.dataset.X_scaled_top_k.shape[0]
args.test_size = test_loader.dataset.X_scaled_top_k.shape[0]
in_dim = 1
if args.tod:
in_dim += 1
if args.ma:
in_dim += 1
if args.mx:
in_dim += 1
args.in_dim = in_dim
aptinit, supports = utils.make_graph_inputs(args, device)
model = models.GWNet.from_args(args, supports, aptinit, **model_kwargs)
model.to(device)
logger = utils.Logger(args)
engine = utils.Trainer.from_args(model, train_loader.dataset.scaler,
train_loader.dataset.scaler_top_k, args)
utils.print_args(args)
if not args.test:
iterator = trange(args.epochs)
try:
if os.path.isfile(logger.best_model_save_path):
print('Model checkpoint exist!')
print('Load model checkpoint? (y/n)')
_in = input()
if _in == 'y' or _in == 'yes':
print('Loading model...')
engine.model.load_state_dict(
torch.load(logger.best_model_save_path))
else:
print('Training new model')
for epoch in iterator:
train_loss, train_rse, train_mae, train_mse, train_mape, train_rmse = [], [], [], [], [], []
for iter, batch in enumerate(train_loader):
# x = batch['x'] # [b, seq_x, n, f]
# y = batch['y'] # [b, seq_y, n]
# sys.exit()
x = batch['x_top_k']
y = batch['y_top_k']
if y.max() == 0: continue
loss, rse, mae, mse, mape, rmse = engine.train(x, y)
train_loss.append(loss)
train_rse.append(rse)
train_mae.append(mae)
train_mse.append(mse)
train_mape.append(mape)
train_rmse.append(rmse)
engine.scheduler.step()
with torch.no_grad():
val_loss, val_rse, val_mae, val_mse, val_mape, val_rmse = engine.eval(
val_loader)
m = dict(train_loss=np.mean(train_loss),
train_rse=np.mean(train_rse),
train_mae=np.mean(train_mae),
train_mse=np.mean(train_mse),
train_mape=np.mean(train_mape),
train_rmse=np.mean(train_rmse),
val_loss=np.mean(val_loss),
val_rse=np.mean(val_rse),
val_mae=np.mean(val_mae),
val_mse=np.mean(val_mse),
val_mape=np.mean(val_mape),
val_rmse=np.mean(val_rmse))
description = logger.summary(m, engine.model)
if logger.stop:
break
description = 'Epoch: {} '.format(epoch) + description
iterator.set_description(description)
except KeyboardInterrupt:
pass
# Metrics on test data
engine.model.load_state_dict(torch.load(logger.best_model_save_path))
with torch.no_grad():
test_met_df, x_gt, y_gt, y_real, yhat = engine.test(
test_loader, engine.model, args.out_seq_len)
test_met_df.round(6).to_csv(
os.path.join(logger.log_dir, 'test_metrics.csv'))
print('Prediction Accuracy:')
print(utils.summary(logger.log_dir))
if args.plot:
logger.plot(x_gt, y_real, yhat)
x_gt = x_gt.cpu().data.numpy() # [timestep, seq_x, seq_y]
y_gt = y_gt.cpu().data.numpy()
yhat = yhat.cpu().data.numpy()
ygt_shape = y_gt.shape
if args.cs:
print('|--- Traffic reconstruction using CS')
y_cs = np.zeros(shape=(ygt_shape[0], 1, ygt_shape[-1]))
# obtain psi, G, R
path_psi_phi = os.path.join(logger.log_dir,
'{}_psi_phi.pkl'.format(args.dataset))
if not os.path.isfile(path_psi_phi):
psi = get_psi(args)
phi = get_phi(args, top_k_index)
obj = {'psi': psi, 'phi': phi}
with open(path_psi_phi, 'wb') as fp:
pickle.dump(obj, fp, protocol=pickle.HIGHEST_PROTOCOL)
else:
with open(path_psi_phi, 'rb') as fp:
obj = pickle.load(fp)
psi = obj['psi']
phi = obj['phi']
np.save('psi.npy', psi.matrix)
# traffic reconstruction using compressive sensing
A = np.dot(phi, psi.matrix)
for i in range(y_cs.shape[0]):
sparse = Solver_l0(A,
max_iter=100,
sparsity=int(args.random_rate / 100 *
y_cs.shape[-1])).fit(yhat[i].T)
y_cs[i] = np.dot(psi.matrix, sparse).T
# y_cs[:, :, top_k_index] = yhat
else:
print('|--- No traffic reconstruction')
y_cs = np.ones(shape=(ygt_shape[0], 1, ygt_shape[-1]))
y_cs[:, :, top_k_index] = yhat
x_gt = torch.from_numpy(x_gt).to(args.device)
y_gt = torch.from_numpy(y_gt).to(args.device)
y_cs = torch.from_numpy(y_cs).to(args.device)
y_cs[y_cs < 0] = 0
test_met = []
for i in range(y_cs.shape[1]):
pred = y_cs[:, i, :]
pred = torch.clamp(pred, min=0., max=10e10)
real = y_real[:, i, :]
test_met.append([x.item() for x in calc_metrics(pred, real)])
test_met_df = pd.DataFrame(test_met,
columns=['rse', 'mae', 'mse', 'mape',
'rmse']).rename_axis('t')
test_met_df.round(6).to_csv(
os.path.join(logger.log_dir, 'test_metrics.csv'))
print('Prediction Accuracy:')
print(utils.summary(logger.log_dir))
if args.run_te:
x_gt = x_gt.cpu().data.numpy() # [timestep, seq_x, seq_y]
y_gt = y_gt.cpu().data.numpy()
y_cs = y_cs.cpu().data.numpy()
run_te(x_gt, y_gt, y_cs, args)
def train():
epoch = 200
batch_size = 1
lr = 0.0002
crop_size = 128
load_size = 128
tar_db_a = "cameron_images.tgz"
tar_db_b = "teresa_images.tgz"
db_a_i = importer_tar.Importer(tar_db_a)
db_b_i = importer_tar.Importer(tar_db_b)
image_a_names = db_a_i.get_sorted_image_name()
image_b_names = db_b_i.get_sorted_image_name()
train_a_size = int(len(image_a_names) * 0.8)
train_b_size = int(len(image_b_names) * 0.8)
image_a_train_names = image_a_names[0:train_a_size]
image_b_train_names = image_b_names[0:train_b_size]
image_a_test_names = image_a_names[train_a_size:]
image_b_test_names = image_b_names[train_b_size:]
print("A train size:{},test size:{}".format(len(image_a_train_names),
len(image_a_test_names)))
print("B train size:{},test size:{}".format(len(image_b_train_names),
len(image_b_test_names)))
""" graph """
# models
generator_a2b = partial(models.generator, scope='a2b')
generator_b2a = partial(models.generator, scope='b2a')
discriminator_a = partial(models.discriminator, scope='a')
discriminator_b = partial(models.discriminator, scope='b')
# operations
a_real_in = tf.placeholder(tf.float32,
shape=[None, load_size, load_size, 3],
name="a_real")
b_real_in = tf.placeholder(tf.float32,
shape=[None, load_size, load_size, 3],
name="b_real")
a_real = utils.preprocess_image(a_real_in, crop_size=crop_size)
b_real = utils.preprocess_image(b_real_in, crop_size=crop_size)
a2b = generator_a2b(a_real)
b2a = generator_b2a(b_real)
b2a2b = generator_a2b(b2a)
a2b2a = generator_b2a(a2b)
a_logit = discriminator_a(a_real)
b2a_logit = discriminator_a(b2a)
b_logit = discriminator_b(b_real)
a2b_logit = discriminator_b(a2b)
# losses
g_loss_a2b = -tf.reduce_mean(a2b_logit)
g_loss_b2a = -tf.reduce_mean(b2a_logit)
cyc_loss_a = tf.losses.absolute_difference(a_real, a2b2a)
cyc_loss_b = tf.losses.absolute_difference(b_real, b2a2b)
g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0
wd_a = tf.reduce_mean(a_logit) - tf.reduce_mean(b2a_logit)
wd_b = tf.reduce_mean(b_logit) - tf.reduce_mean(a2b_logit)
gp_a = gradient_penalty(a_real, b2a, discriminator_a)
gp_b = gradient_penalty(b_real, a2b, discriminator_b)
d_loss_a = -wd_a + 10.0 * gp_a
d_loss_b = -wd_b + 10.0 * gp_b
# summaries
utils.summary({
g_loss_a2b: 'g_loss_a2b',
g_loss_b2a: 'g_loss_b2a',
cyc_loss_a: 'cyc_loss_a',
cyc_loss_b: 'cyc_loss_b'
})
utils.summary({d_loss_a: 'd_loss_a'})
utils.summary({d_loss_b: 'd_loss_b'})
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
merged = tf.summary.merge_all()
im1_op = tf.summary.image("real_a", a_real_in)
im2_op = tf.summary.image("a2b", a2b)
im3_op = tf.summary.image("b2a2b", b2a2b)
im4_op = tf.summary.image("real_b", b_real_in)
im5_op = tf.summary.image("b2a", b2a)
im6_op = tf.summary.image("b2a2b", b2a2b)
# optim
t_var = tf.trainable_variables()
d_a_var = [var for var in t_var if 'a_discriminator' in var.name]
d_b_var = [var for var in t_var if 'b_discriminator' in var.name]
g_var = [
var for var in t_var
if 'a2b_generator' in var.name or 'b2a_generator' in var.name
]
d_a_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_a,
var_list=d_a_var)
d_b_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_b,
var_list=d_b_var)
g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss,
var_list=g_var)
""" train """
''' init '''
# session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# counter
it_cnt, update_cnt = utils.counter()
''' summary '''
summary_writer = tf.summary.FileWriter('./outputs/summaries/', sess.graph)
''' saver '''
saver = tf.train.Saver(max_to_keep=5)
''' restore '''
ckpt_dir = './outputs/checkpoints/'
utils.mkdir(ckpt_dir)
try:
utils.load_checkpoint(ckpt_dir, sess)
except:
sess.run(tf.global_variables_initializer())
'''train'''
try:
batch_epoch = min(train_a_size, train_b_size) // batch_size
max_it = epoch * batch_epoch
for it in range(sess.run(it_cnt), max_it):
sess.run(update_cnt)
epoch = it // batch_epoch
it_epoch = it % batch_epoch + 1
# read data
a_real_np = cv2.resize(
db_a_i.get_image(image_a_train_names[it_epoch % batch_epoch]),
(load_size, load_size))
b_real_np = cv2.resize(
db_b_i.get_image(image_b_train_names[it_epoch % batch_epoch]),
(load_size, load_size))
# train G
sess.run(g_train_op,
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
# train discriminator
sess.run([d_a_train_op, d_b_train_op],
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
# display
if it % 100 == 0:
# make summary
summary = sess.run(merged,
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
summary_writer.add_summary(summary, it)
print("Epoch: (%3d) (%5d/%5d)" %
(epoch, it_epoch, batch_epoch))
# save
if (it + 1) % 1000 == 0:
save_path = saver.save(
sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch,
it_epoch))
print('###Model saved in file: {}'.format(save_path))
# sample
if (it + 1) % 1000 == 0:
a_test_index = int(
np.random.uniform(high=len(image_a_test_names)))
b_test_index = int(
np.random.uniform(high=len(image_b_test_names)))
a_real_np = cv2.resize(
db_a_i.get_image(image_a_test_names[a_test_index]),
(load_size, load_size))
b_real_np = cv2.resize(
db_b_i.get_image(image_b_test_names[b_test_index]),
(load_size, load_size))
[a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt
] = sess.run([a_real, a2b, a2b2a, b_real, b2a, b2a2b],
feed_dict={
a_real_in: [a_real_np],
b_real_in: [b_real_np]
})
sample_opt = np.concatenate(
(a_opt, a2b_opt, a2b2a_opt, b_opt, b2a_opt, b2a2b_opt),
axis=0)
[im1_sum,im2_sum,im3_sum,im4_sum,im5_sum,im6_sum] = \
sess.run([im1_op,im2_op,im3_op,im4_op,im5_op,im6_op],
feed_dict={a_real_in: [a_real_np], b_real_in: [b_real_np]})
summary_writer.add_summary(im1_sum, it)
summary_writer.add_summary(im2_sum, it)
summary_writer.add_summary(im3_sum, it)
summary_writer.add_summary(im4_sum, it)
summary_writer.add_summary(im5_sum, it)
summary_writer.add_summary(im6_sum, it)
save_dir = './outputs/sample_images_while_training/'
utils.mkdir(save_dir)
im.imwrite(
im.immerge(sample_opt, 2, 3),
'{}/epoch_{}_it_{}.jpg'.format(save_dir, epoch, it_epoch))
except:
raise
finally:
save_path = saver.save(
sess, '{}/epoch_{}_{}.ckpt'.format(ckpt_dir, epoch, it_epoch))
print('###Model saved in file: {}'.format(save_path))
sess.close()
def main_worker(gpu, args):
"""
模型训练、测试、转JIT、蒸馏文件制作
:param gpu: 运行的gpu id
:param args: 运行超参
"""
args.gpu = gpu
utils.generate_logger(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{gpu}.log")
logging.info(f'args: {args}')
# 可复现性
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
logging.warning('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
if args.cuda:
logging.info(f"Use GPU: {args.gpu} ~")
if args.distributed:
args.rank = args.rank * args.gpus + gpu
dist.init_process_group(backend='nccl', init_method=args.init_method,
world_size=args.world_size, rank=args.rank)
else:
logging.info(f"Use CPU ~")
# 创建/加载模型，使用预训练模型时，需要自己先下载好放到 pretrained 文件夹下，以网络名词命名
logging.info(f"=> creating model '{args.arch}'")
model = my_models.get_model(args.arch, args.pretrained, num_classes=args.num_classes)
# 重加载之前训练好的模型
if args.resume:
if os.path.isfile(args.resume):
logging.info(f"=> loading checkpoint '{args.resume}'")
checkpoint = torch.load(args.resume, map_location=torch.device('cpu'))
acc = model.load_state_dict(checkpoint['state_dict'], strict=True)
logging.info(f'missing keys of models: {acc.missing_keys}')
del checkpoint
else:
raise Exception(f"No checkpoint found at '{args.resume}' to be resumed")
# 模型信息
image_height, image_width = args.image_size
logging.info(f'Model {args.arch} input size: ({image_height}, {image_width})')
utils.summary(size=(image_height, image_width), channel=3, model=model)
# 模型转换：转为 torch.jit.script
if args.jit:
if not args.resume:
raise Exception('Option --resume must specified!')
applications.convert_to_jit(model, args=args)
return
if args.criterion == 'softmax':
criterion = criterions.HybridCELoss(args=args) # 混合策略多分类
elif args.criterion == 'bce':
criterion = criterions.HybridBCELoss(args=args) # 混合策略多标签二分类
else:
raise NotImplementedError(f'Not loss function {args.criterion}')
if args.cuda:
if args.distributed and args.sync_bn:
model = apex.parallel.convert_syncbn_model(model)
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
criterion = criterion.cuda(args.gpu)
if args.knowledge in ('train', 'test', 'val'):
torch.set_flush_denormal(True)
distill_loader = dataloader.load(args, name=args.knowledge)
applications.distill(distill_loader, model, criterion, args, is_confuse_matrix=True)
return
if args.make_curriculum in ('train', 'test', 'val'):
torch.set_flush_denormal(True)
curriculum_loader = dataloader.load(args, name=args.make_curriculum)
applications.make_curriculum(curriculum_loader, model, criterion, args, is_confuse_matrix=True)
return
if args.visual_data in ('train', 'test', 'val'):
torch.set_flush_denormal(True)
test_loader = dataloader.load(args, name=args.visual_data)
applications.Visualize.visualize(test_loader, model, args)
return
# 优化器
opt_set = {
'sgd': partial(torch.optim.SGD, momentum=args.momentum),
'adam': torch.optim.Adam, 'adamw': AdamW,
'radam': RAdam, 'ranger': Ranger, 'lookaheadadam': LookaheadAdam,
'ralamb': Ralamb, 'rangerlars': RangerLars,
'novograd': Novograd,
}
optimizer = opt_set[args.opt](model.parameters(), lr=args.lr) # weight decay转移到train那里了
# 随机均值平均优化器
# from optim.swa import SWA
# optimizer = SWA(optimizer, swa_start=10, swa_freq=5, swa_lr=0.05)
# 混合精度训练
if args.cuda:
model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
if args.distributed:
model = apex.parallel.DistributedDataParallel(model)
else:
model = torch.nn.DataParallel(model)
if args.train:
train_loader = dataloader.load(args, 'train')
val_loader = dataloader.load(args, 'val')
scheduler = LambdaLR(optimizer,
lambda epoch: adjust_learning_rate(epoch, args=args))
applications.train(train_loader, val_loader, model, criterion, optimizer, scheduler, args)
args.evaluate = True
if args.evaluate:
torch.set_flush_denormal(True)
test_loader = dataloader.load(args, name='test')
acc, loss, paths_targets_preds_probs = applications.test(test_loader, model,
criterion, args, is_confuse_matrix=True)
logging.info(f'Evaluation: * Acc@1 {acc:.3f} and loss {loss:.3f}.')
logging.info(f'Evaluation Result:\n')
for path, target, pred, prob in paths_targets_preds_probs:
logging.info(path + ' ' + str(target) + ' ' + str(pred) + ' ' + ','.join([f'{num:.2f}' for num in prob]))
logging.info('Evaluation Over~')
g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0
d_loss_a_real = tf.losses.mean_squared_error(a_logit, tf.ones_like(a_logit))
d_loss_b2a_sample = tf.losses.mean_squared_error(
b2a_sample_logit, tf.zeros_like(b2a_sample_logit))
d_loss_a = d_loss_a_real + d_loss_b2a_sample
d_loss_b_real = tf.losses.mean_squared_error(b_logit, tf.ones_like(b_logit))
d_loss_a2b_sample = tf.losses.mean_squared_error(
a2b_sample_logit, tf.zeros_like(a2b_sample_logit))
d_loss_b = d_loss_b_real + d_loss_a2b_sample
# summaries
g_summary = utils.summary({
g_loss_a2b: 'g_loss_a2b',
g_loss_b2a: 'g_loss_b2a',
cyc_loss_a: 'cyc_loss_a',
cyc_loss_b: 'cyc_loss_b'
})
d_summary_a = utils.summary({d_loss_a: 'd_loss_a'})
d_summary_b = utils.summary({d_loss_b: 'd_loss_b'})
# optim
t_var = tf.trainable_variables()
d_a_var = [var for var in t_var if 'a_discriminator' in var.name]
d_b_var = [var for var in t_var if 'b_discriminator' in var.name]
g_var = [
var for var in t_var
if 'a2b_generator' in var.name or 'b2a_generator' in var.name
]
d_a_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_a,
d_loss = -wd
g_loss = -tf.reduce_mean(f_logit)
# otpims
d_var = utils.trainable_variables('discriminator')
g_var = utils.trainable_variables('generator')
d_step_ = tf.train.RMSPropOptimizer(learning_rate=lr).minimize(
d_loss, var_list=d_var)
with tf.control_dependencies([d_step_]):
d_step = tf.group(*(tf.assign(var, tf.clip_by_value(var, -clip, clip))
for var in d_var))
g_step = tf.train.RMSPropOptimizer(learning_rate=lr).minimize(
g_loss, var_list=g_var)
# summaries
d_summary = utils.summary({wd: 'wd'})
g_summary = utils.summary({g_loss: 'g_loss'})
# sample
f_sample = generator(z, training=False)
""" train """
''' init '''
# session
sess = utils.session()
# iteration counter
it_cnt, update_cnt = utils.counter()
# saver
saver = tf.train.Saver(max_to_keep=5)
# summary writer
summary_writer = tf.summary.FileWriter('./summaries/celeba_wgan', sess.graph)
''' initialization '''
num_workers=train_args.num_workers,
# sampler=train_sampler,
pin_memory=False,
)
valid_loader = DataLoader(
valid_set,
batch_size=train_args.batch_size,
shuffle=False,
num_workers=train_args.num_workers,
# sampler=valid_sampler,
pin_memory=False,
)
# model summary
data, _, _ = train_set[0]
summary(
model,
tuple(data.shape),
batch_size=train_args.batch_size,
device='cuda',
model_name=train_args.model_name.upper(),
)
# training
train(model, train_loader, valid_loader, optimizer, criterion, train_args)
# config
model_config(train_args,
save=False) # print model configuration after training
postprocessor(train_args.save_path)
def __init__(self, config):
super(Trainer, self).__init__(config)
# Datamanager
self.datamanager, params_data = build_datamanager(
config['type'], config['data'])
# model
self.model, params_model = build_model(
config,
num_classes=len(self.datamanager.datasource.get_attribute()),
device=self.device)
# losses
pos_ratio = torch.tensor(
self.datamanager.datasource.get_weight('train'))
self.criterion, params_loss = build_losses(
config,
pos_ratio=pos_ratio,
num_attribute=len(self.datamanager.datasource.get_attribute()))
# optimizer
self.optimizer, params_optimizers = build_optimizers(
config, self.model)
# learing rate scheduler
self.lr_scheduler, params_lr_scheduler = build_lr_scheduler(
config, self.optimizer)
# callbacks for freeze backbone
if config['freeze']['enable']:
self.freeze = FreezeLayers(self.model, config['freeze']['layers'],
config['freeze']['epochs'])
else:
self.freeze = None
# list of metrics
self.lst_metrics = ['mA', 'accuracy', 'f1_score']
# track metric
self.train_metrics = MetricTracker('loss', *self.lst_metrics)
self.valid_metrics = MetricTracker('loss', *self.lst_metrics)
# step log loss and accuracy
self.log_step = (len(self.datamanager.get_dataloader('train')) // 5,
len(self.datamanager.get_dataloader('val')) // 5)
self.log_step = (self.log_step[0] if self.log_step[0] > 0 else 1,
self.log_step[1] if self.log_step[1] > 0 else 1)
# best accuracy and loss
self.best_loss = None
self.best_metrics = dict()
for x in self.lst_metrics:
self.best_metrics[x] = None
# print config
self._print_config(
params_data=params_data,
params_model=params_model,
params_loss=params_loss,
params_optimizers=params_optimizers,
params_lr_scheduler=params_lr_scheduler,
freeze_layers=False if self.freeze == None else True,
clip_grad_norm_=self.config['clip_grad_norm_']['enable'])
# send model to device
self.model.to(self.device)
self.criterion.to(self.device)
# summary model
summary(model=self.model,
input_data=torch.zeros((self.datamanager.get_batch_size(), 3,
self.datamanager.get_image_size()[0],
self.datamanager.get_image_size()[1])),
batch_dim=None,
device='cuda' if self.use_gpu else 'cpu',
print_func=self.logger.info,
print_step=False)
# resume model from last checkpoint
if config['resume'] != '':
self._resume_checkpoint(config['resume'], config['only_model'])
from Person import Person
import utils
persons = utils.loadPersons('data/persons.json')
names = list(map(lambda person : person.name, persons))
bill = utils.loadBill('data/bill.json', names)
print('Bill')
for b in bill:
b.show()
utils.calculatePayMent(persons, bill)
utils.optimize(persons)
print('')
print('Payments')
for person in persons:
person.show()
print('')
utils.check(persons)
utils.summary(bill, persons)
def __init__(self, args):
self.args = args
filehandler = logging.FileHandler(args.logging_file)
streamhandler = logging.StreamHandler()
self.logger = logging.getLogger('')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(filehandler)
self.logger.addHandler(streamhandler)
self.logger.info(args)
# image transform
input_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([.485, .456, .406], [.229, .224, .225]), # Default mean and std
])
################################# dataset and dataloader #################################
if platform.system() == "Darwin":
data_root = os.path.join('~', 'Nutstore Files', 'Dataset') # Mac
elif platform.system() == "Linux":
data_root = os.path.join('~', 'datasets') # Laplace or HPC
if args.colab:
data_root = '/content/datasets' # Colab
else:
raise ValueError('Notice Dataset Path')
data_kwargs = {'base_size': args.base_size, 'transform': input_transform,
'crop_size': args.crop_size, 'root': data_root,
'base_dir': args.dataset}
trainset = IceContrast(split=args.train_split, mode='train', **data_kwargs)
valset = IceContrast(split=args.val_split, mode='testval', **data_kwargs)
self.train_data = gluon.data.DataLoader(trainset, args.batch_size, shuffle=True,
last_batch='rollover', num_workers=args.workers)
self.eval_data = gluon.data.DataLoader(valset, args.test_batch_size,
last_batch='rollover', num_workers=args.workers)
layers = [args.blocks] * 3
channels = [x * args.channel_times for x in [8, 16, 32, 64]]
if args.model == 'ResNetFPN':
model = ASKCResNetFPN(layers=layers, channels=channels, fuse_mode=args.fuse_mode,
tiny=args.tiny, classes=trainset.NUM_CLASS)
elif args.model == 'ResUNet':
model = ASKCResUNet(layers=layers, channels=channels, fuse_mode=args.fuse_mode,
tiny=args.tiny, classes=trainset.NUM_CLASS)
print("layers: ", layers)
print("channels: ", channels)
print("fuse_mode: ", args.fuse_mode)
print("tiny: ", args.tiny)
print("classes: ", trainset.NUM_CLASS)
if args.host == 'xxx':
self.host_name = socket.gethostname() # automatic
else:
self.host_name = args.host # Puma needs to be specified
self.save_prefix = '_'.join([args.model, args.fuse_mode, args.dataset, self.host_name,
'GPU', args.gpus])
model.cast(args.dtype)
# self.logger.info(model)
# resume checkpoint if needed
if args.resume is not None:
if os.path.isfile(args.resume):
model.load_parameters(args.resume, ctx=args.ctx)
else:
raise RuntimeError("=> no checkpoint found at '{}'".format(args.resume))
else:
model.initialize(init=init.MSRAPrelu(), ctx=args.ctx, force_reinit=True)
print("Model Initializing")
print("args.ctx: ", args.ctx)
self.net = model
if args.summary:
summary(self.net, mx.nd.zeros((1, 3, args.crop_size, args.crop_size), ctx=args.ctx[0]))
sys.exit()
# create criterion
self.criterion = SoftIoULoss()
# optimizer and lr scheduling
self.lr_scheduler = LRSequential([
LRScheduler('linear', base_lr=0, target_lr=args.lr,
nepochs=args.warmup_epochs, iters_per_epoch=len(self.train_data)),
LRScheduler(mode='poly', base_lr=args.lr,
nepochs=args.epochs-args.warmup_epochs,
iters_per_epoch=len(self.train_data),
power=0.9)
])
kv = mx.kv.create(args.kvstore)
if args.optimizer == 'sgd':
optimizer_params = {'lr_scheduler': self.lr_scheduler,
'wd': args.weight_decay,
'momentum': args.momentum,
'learning_rate': args.lr}
elif args.optimizer == 'adam':
optimizer_params = {'lr_scheduler': self.lr_scheduler,
'wd': args.weight_decay,
'learning_rate': args.lr}
elif args.optimizer == 'adagrad':
optimizer_params = {
'wd': args.weight_decay,
'learning_rate': args.lr
}
else:
raise ValueError('Unsupported optimizer {} used'.format(args.optimizer))
if args.dtype == 'float16':
optimizer_params['multi_precision'] = True
if args.no_wd:
for k, v in self.net.collect_params('.*beta|.*gamma|.*bias').items():
v.wd_mult = 0.0
self.optimizer = gluon.Trainer(self.net.collect_params(), args.optimizer,
optimizer_params, kvstore=kv)
################################# evaluation metrics #################################
self.iou_metric = SigmoidMetric(1)
self.nIoU_metric = SamplewiseSigmoidMetric(1, score_thresh=self.args.score_thresh)
self.best_iou = 0
self.best_nIoU = 0
self.is_best = False
# discriminative loss
d_loss_ab_match = tf.losses.mean_squared_error(ab_match_logit,
tf.ones_like(ab_match_logit))
d_loss_ab_unmatch = tf.losses.mean_squared_error(
ab_unmatch_logit, tf.zeros_like(ab_unmatch_logit))
d_loss_aa2b_pair = tf.losses.mean_squared_error(
a2b_sample_logit, tf.zeros_like(a2b_sample_logit))
d_loss_b = d_loss_ab_match + d_loss_ab_unmatch + d_loss_aa2b_pair
# d_loss_b *= 0.1
# summaries
g_summary = utils.summary({g_loss_a2b: 'g_loss_a2b'})
d_summary_b = utils.summary({d_loss_b: 'd_loss_b'})
eval_summary = utils.summary({sl_loss: 'eval_loss'})
# optim
t_var = tf.trainable_variables()
d_b_var = [var for var in t_var if 'b_discriminator' in var.name]
g_var = [
var for var in t_var
if 'a2b_generator' in var.name or 'b2a_generator' in var.name
]
d_b_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(d_loss_b,
var_list=d_b_var)
g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss,
var_list=g_var)
# losses
d_r_loss = tf.losses.sigmoid_cross_entropy(tf.ones_like(r_logit), r_logit)
d_f_loss = tf.losses.sigmoid_cross_entropy(tf.zeros_like(f_logit), f_logit)
d_loss = (d_r_loss + d_f_loss) / 2.0
g_loss = tf.losses.sigmoid_cross_entropy(tf.ones_like(f_logit), f_logit)
# otpims
d_var = utils.trainable_variables('discriminator')
g_var = utils.trainable_variables('generator')
d_step = tf.train.AdamOptimizer(learning_rate=lr,
beta1=0.3).minimize(d_loss, var_list=d_var)
g_step = tf.train.AdamOptimizer(learning_rate=lr,
beta1=0.3).minimize(g_loss, var_list=g_var)
# summaries
d_summary = utils.summary({d_loss: 'd_loss'})
g_summary = utils.summary({g_loss: 'g_loss'})
# sample
f_sample = generator(z, training=False)
""" train """
''' init '''
# session
sess = utils.session()
# iteration counter
it_cnt, update_cnt = utils.counter()
# saver
saver = tf.train.Saver(max_to_keep=5)
# summary writer
summary_writer = tf.summary.FileWriter('./summaries/cartoon_dcgan', sess.graph)
''' initialization '''
d_loss = d_r_loss + d_f_loss + 10.0 * gp
g_loss = tf.losses.sigmoid_cross_entropy(tf.ones_like(f_logit), f_logit)
# otpims
d_var = utils.trainable_variables('discriminator')
g_var = utils.trainable_variables('generator')
d_step = tf.train.AdamOptimizer(learning_rate=lr,
beta1=0.5).minimize(d_loss, var_list=d_var)
g_step = tf.train.AdamOptimizer(learning_rate=lr,
beta1=0.5).minimize(g_loss, var_list=g_var)
# summaries
d_summary = utils.summary({
d_r_loss: 'd_r_loss',
d_f_loss: 'd_f_loss',
gp: 'gp'
})
g_summary = utils.summary({g_loss: 'g_loss'})
# sample
f_sample = generator(z, training=False)
""" train """
''' init '''
# session
sess = utils.session()
# iteration counter
it_cnt, update_cnt = utils.counter()
# saver
saver = tf.train.Saver(max_to_keep=5)
# summary writer
'-------------------------------------------')
df_dataset[col] = df_dataset[col].astype(str)
vect_val = df_dataset[col].drop(np.where(df_dataset[col] == 'nan')[0])
vect_val = vect_val.astype(float)
# Necessary informations:
kde = False
bins = 30
nb_points = float(len(vect_val))
min_val = vect_val.min()
max_val = vect_val.max()
nb_miss = float(len(df_dataset[col].tolist()) - nb_points)
rate = round(100 * nb_miss / len(df_dataset[col].tolist()), 2)
if len(vect_val.tolist()) != 0:
Low_tuck, Upp_tuck, mean, std = utils.summary(vect_val.tolist())
else:
Low_tuck = 'nan'
Upp_tuck = 'nan'
mean = 'nan'
std = 'nan'
print('|nb_points : ', nb_points)
print('|rate : ', rate)
# Write data in a file:
file1 = open('../infoDataset/details/' + col + '/statistics.txt', "w")
L = [
"Number of datas : " + str(nb_points) + "\n",
"Min : " + str(min_val) + "\n",
"Max : " + str(max_val) + "\n",
def train(logdir, model_name, iterations, checkpoint_interval, batch_size,
temperature, hidden_size, n_layers, rnn_cell, learning_rate,
learning_rate_decay_steps, learning_rate_decay_rate):
device = torch.device(
'cuda:0') if torch.cuda.is_available() else torch.device('cpu')
os.makedirs(logdir, exist_ok=True)
dataset = dataloader.JazzDataset()
loader = DataLoader(dataset, batch_size, shuffle=True)
model_class = getattr(rnn, model_name)
model = model_class(hidden_size, rnn_cell, n_layers)
optimizer = torch.optim.Adam(model.parameters(), learning_rate)
criterion = nn.NLLLoss()
scheduler = StepLR(optimizer,
step_size=learning_rate_decay_steps,
gamma=learning_rate_decay_rate)
model = model.to(device)
summary(model)
loop = tqdm(range(0, iterations + 1), desc='Training', unit='Steps')
for i, batch in zip(loop, cycle(loader)):
scheduler.step()
optimizer.zero_grad()
batch = batch.to(device) # shape of (batch_size, n_steps)
c_0, h_0 = model.init_hidden(batch.shape[0])
c_0 = c_0.to(device)
h_0 = h_0.to(device)
# TODO: Fill in below
init_hidden = None
hidden = init_hidden
loss = 0.0
for step in range(batch.shape[1] - 1): # n_steps - 1
# TODO: Fill in below
# Forward model.
# x=semgent of batch, corresponds to step,
# hidden=state of hidden nodes of last/or initial step
pred, hidden = model(x=None, hidden=None)
# TODO: Fill in below
# Hint: use criterion. See torch.nn.NLLLoss() function
loss += None
loss.backward()
optimizer.step()
# print loss
loop.set_postfix_str("loss: {:.3f}".format(loss))
# save model
if i % checkpoint_interval == 0:
torch.save(
{
'model_state_dict':
model.state_dict(),
'optimizer_state_dict':
optimizer.state_dict(),
'model_name':
model_name,
'hparams':
dict(hidden_size=hidden_size,
n_layers=n_layers,
rnn_cell=rnn_cell)
}, os.path.join(logdir, 'model-{:d}.pt'.format(i)))
def train():
epoch = 200
batch_size = 1
lr = 0.0002
crop_size = 4
tar_db_a = "cameron_images.tgz"
tar_db_b = "teresa_images.tgz"
db_a_i = importer_tar.Importer(tar_db_a)
db_b_i = importer_tar.Importer(tar_db_b)
image_a_names = db_a_i.get_sorted_image_name()
image_b_names = db_b_i.get_sorted_image_name()
train_a_size = int(len(image_a_names) * 0.8)
train_b_size = int(len(image_b_names) * 0.8)
image_a_train_names = image_a_names[0:train_a_size]
image_b_train_names = image_b_names[0:train_b_size]
image_a_test_names = image_a_names[train_a_size:]
image_b_test_names = image_b_names[train_b_size:]
print("A train size:{},test size:{}".format(len(image_a_train_names),
len(image_a_test_names)))
print("B train size:{},test size:{}".format(len(image_b_train_names),
len(image_b_test_names)))
""" graph """
# models
generator_a2b = partial(models.generator, scope='a2b')
generator_b2a = partial(models.generator, scope='b2a')
discriminator_a = partial(models.discriminator, scope='a')
discriminator_b = partial(models.discriminator, scope='b')
#print("Discriminator shape:{}".format())
# operations
a_real = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3])
b_real = tf.placeholder(tf.float32, shape=[None, crop_size, crop_size, 3])
a2b_sample = tf.placeholder(tf.float32,
shape=[None, crop_size, crop_size, 3])
b2a_sample = tf.placeholder(tf.float32,
shape=[None, crop_size, crop_size, 3])
a2b = generator_a2b(a_real)
b2a = generator_b2a(b_real)
b2a2b = generator_a2b(b2a)
a2b2a = generator_b2a(a2b)
a_logit = discriminator_a(a_real)
b2a_logit = discriminator_a(b2a)
b2a_sample_logit = discriminator_a(b2a_sample)
b_logit = discriminator_b(b_real)
a2b_logit = discriminator_b(a2b)
a2b_sample_logit = discriminator_b(a2b_sample)
# losses
#g_loss_a2b = tf.losses.mean_squared_error(a2b_logit, tf.ones_like(a2b_logit))
#g_loss_b2a = tf.losses.mean_squared_error(b2a_logit, tf.ones_like(b2a_logit))
g_loss_a2b = -tf.reduce_mean(a2b_logit)
g_loss_b2a = -tf.reduce_mean(b22_logit)
cyc_loss_a = tf.losses.absolute_difference(a_real, a2b2a)
cyc_loss_b = tf.losses.absolute_difference(b_real, b2a2b)
g_loss = g_loss_a2b + g_loss_b2a + cyc_loss_a * 10.0 + cyc_loss_b * 10.0
# d_loss_a_real = tf.losses.mean_squared_error(a_logit, tf.ones_like(a_logit))
# d_loss_b2a_sample = tf.losses.mean_squared_error(b2a_sample_logit, tf.zeros_like(b2a_sample_logit))
# d_loss_a = d_loss_a_real + d_loss_b2a_sample
#
wd_a = tf.reduce_mean(a_logit) - tf.reduce_mean(b2a_logit)
wd_b = tf.reduce_mean(b_logit) - tf.reduce_mean(a2b_logit)
gp_a = gradient_penalty(a_real, b2a, discriminator_a)
gp_b = gradient_penalty(b_real, a2b, discriminator_b)
d_loss_a = -wd_a + 10.0 * gp_a
d_loss_b = -wd_b + 10.0 * gp_b
# d_loss_b_real = tf.losses.mean_squared_error(b_logit, tf.ones_like(b_logit))
# d_loss_a2b_sample = tf.losses.mean_squared_error(a2b_sample_logit, tf.zeros_like(a2b_sample_logit))
# d_loss_b = d_loss_b_real + d_loss_a2b_sample
# summaries
g_summary = utils.summary({
g_loss_a2b: 'g_loss_a2b',
g_loss_b2a: 'g_loss_b2a',
cyc_loss_a: 'cyc_loss_a',
cyc_loss_b: 'cyc_loss_b'
})
d_summary_a = utils.summary({d_loss_a: 'd_loss_a'})
d_summary_b = utils.summary({d_loss_b: 'd_loss_b'})
# optim
t_var = tf.trainable_variables()
d_a_var = [var for var in t_var if 'a_discriminator' in var.name]
d_b_var = [var for var in t_var if 'b_discriminator' in var.name]
g_var = [
var for var in t_var
if 'a2b_generator' in var.name or 'b2a_generator' in var.name
]
d_a_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_a,
var_list=d_a_var)
d_b_train_op = tf.train.AdamOptimizer(lr,
beta1=0.5).minimize(d_loss_b,
var_list=d_b_var)
g_train_op = tf.train.AdamOptimizer(lr, beta1=0.5).minimize(g_loss,
var_list=g_var)
""" train """
''' init '''
# session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# counter
it_cnt, update_cnt = utils.counter()
''' summary '''
summary_writer = tf.summary.FileWriter('./outputs/summaries/', sess.graph)
''' saver '''
saver = tf.train.Saver(max_to_keep=5)
''' restore '''
ckpt_dir = './outputs/checkpoints/'
utils.mkdir(ckpt_dir)
try:
utils.load_checkpoint(ckpt_dir, sess)
except:
sess.run(tf.global_variables_initializer())
'''train'''
try:
batch_epoch = min(train_a_size, train_b_size) // batch_size
max_it = epoch * batch_epoch
for it in range(sess.run(it_cnt), max_it):
sess.run(update_cnt)
epoch = it // batch_epoch
it_epoch = it % batch_epoch + 1
# prepare data
a_real_ipt = db_a_i.get_image(image_a_train_names[it_epoch])
a_real_ipt = cv2.resize(a_real_ipt, (crop_size, crop_size),
interpolation=cv2.INTER_CUBIC)
b_real_ipt = db_b_i.get_image(image_b_train_names[it_epoch])
b_real_ipt = cv2.resize(b_real_ipt, (crop_size, crop_size),
interpolation=cv2.INTER_CUBIC)
a_real_ipt = [a_real_ipt]
b_real_ipt = [b_real_ipt]
a2b_opt, b2a_opt = sess.run([a2b, b2a],
feed_dict={
a_real: a_real_ipt,
b_real: b_real_ipt
})
a2b_sample_ipt = a2b_opt
b2a_sample_ipt = b2a_opt
# train G
g_summary_opt, _ = sess.run([g_summary, g_train_op],
feed_dict={
a_real: a_real_ipt,
b_real: b_real_ipt
})
summary_writer.add_summary(g_summary_opt, it)
# train D_b
d_summary_b_opt, _ = sess.run([d_summary_b, d_b_train_op],
feed_dict={
b_real: b_real_ipt,
a2b_sample: a2b_sample_ipt
})
summary_writer.add_summary(d_summary_b_opt, it)
# train D_a
d_summary_a_opt, _ = sess.run([d_summary_a, d_a_train_op],
feed_dict={
a_real: a_real_ipt,
b2a_sample: b2a_sample_ipt
})
summary_writer.add_summary(d_summary_a_opt, it)
# display
if it % 1 == 0:
print("Epoch: (%3d) (%5d/%5d)" %
(epoch, it_epoch, batch_epoch))
# save
if (it + 1) % 1000 == 0:
save_path = saver.save(
sess, '%s/Epoch_(%d)_(%dof%d).ckpt' %
(ckpt_dir, epoch, it_epoch, batch_epoch))
print('Model saved in file: % s' % save_path)
# sample
# if (it + 1) % 100 == 0:
# a_real_ipt = a_test_pool.batch()
# b_real_ipt = b_test_pool.batch()
# [a2b_opt, a2b2a_opt, b2a_opt, b2a2b_opt] = sess.run([a2b, a2b2a, b2a, b2a2b], feed_dict={a_real: a_real_ipt, b_real: b_real_ipt})
# sample_opt = np.concatenate((a_real_ipt, a2b_opt, a2b2a_opt, b_real_ipt, b2a_opt, b2a2b_opt), axis=0)
#
# save_dir = './outputs/sample_images_while_training/' + dataset
# utils.mkdir(save_dir)
# im.imwrite(im.immerge(sample_opt, 2, 3), '%s/Epoch_(%d)_(%dof%d).jpg' % (save_dir, epoch, it_epoch, batch_epoch))
except:
save_path = saver.save(
sess, '%s/Epoch_(%d)_(%dof%d).ckpt' %
(ckpt_dir, epoch, it_epoch, batch_epoch))
print('Model saved in file: % s' % save_path)
sess.close()
raise
def train(logdir, model_name, iterations, checkpoint_interval, batch_size,
temperature, hidden_size, n_layers, rnn_cell, learning_rate,
learning_rate_decay_steps, learning_rate_decay_rate):
device = torch.device(
'cuda:0') if torch.cuda.is_available() else torch.device('cpu')
os.makedirs(logdir, exist_ok=True)
dataset = dataloader.JazzDataset()
loader = DataLoader(dataset, batch_size, shuffle=True)
model_class = getattr(rnn, model_name)
model = model_class(hidden_size, rnn_cell, n_layers)
optimizer = torch.optim.Adam(model.parameters(), learning_rate)
criterion = nn.NLLLoss()
scheduler = StepLR(optimizer,
step_size=learning_rate_decay_steps,
gamma=learning_rate_decay_rate)
model = model.to(device)
summary(model)
loop = tqdm(range(0, iterations + 1), desc='Training', unit='Steps')
for i, batch in zip(loop, cycle(loader)):
scheduler.step()
optimizer.zero_grad()
batch = batch.to(device) # shape of (batch_size, sequence_length)
c_0, h_0 = model.init_hidden(batch.shape[0])
c_0 = c_0.to(device)
h_0 = h_0.to(device)
# TODO: Fill in below
init_hidden = (c_0, h_0)
hidden = init_hidden
loss = 0.0
for step in range(batch.shape[1] - 1): # sequence_length - 1
# TODO: Fill in below
# run a step of training model.
# x = semgent of batch, corresponds to current step. shape of (batch_size, 1)
x = [batch[i][step] for i in range(batch.shape[0])]
x = torch.tensor(x, dtype=torch.long)
x = x.to(device)
pred, hidden = model(x=x, hidden=hidden)
# TODO: Fill in below
# calcuate loss between prediction and the values of next step.
# Hint: use criterion. See torch.nn.NLLLoss() function
x1 = [batch[i][step + 1] for i in range(batch.shape[0])]
x1 = torch.tensor(x1, dtype=torch.long)
x1 = x1.to(device)
loss += criterion(pred, x1)
loss.backward()
optimizer.step()
# print loss
loop.set_postfix_str("loss: {:.3f}".format(loss))
# save model
if i % checkpoint_interval == 0:
torch.save(
{
'model_state_dict':
model.state_dict(),
'optimizer_state_dict':
optimizer.state_dict(),
'model_name':
model_name,
'hparams':
dict(hidden_size=hidden_size,
n_layers=n_layers,
rnn_cell=rnn_cell)
}, os.path.join(logdir, 'model-{:d}.pt'.format(i)))
from utils import read_pdf, clean_stem_corpus, summary, save_kesimpulan, tfidf_bobot
from nltk.tokenize import sent_tokenize
import time
print('Starting.....')
since = time.time()
PATH = '39775-697-80327-1-10-20180521.pdf'
content = read_pdf(PATH, semua_halaman=False, halaman=1)
corpus = sent_tokenize(content)
print('ukuran corpus', len(corpus))
print('\ncleaning and stemming sentences....')
kalimat_stem = clean_stem_corpus(corpus)
words_weights = tfidf_bobot(kalimat_stem)
print('\nSummarizing.....')
kesimpulan = summary(corpus, words_weights, n_kalimat_summary=5)
save_kesimpulan(kesimpulan, 'kesimpulan.doc')
print(f"\nKesimpulan:\n{kesimpulan}")
print("Waktu: {:.3f} minutes".format((time.time() - since) / 60))
def train(epochs, ctx):
if isinstance(ctx, mx.Context):
ctx = [ctx]
net.initialize(mx.init.Xavier(), ctx=ctx)
if opt.summary:
summary(net, mx.nd.zeros((1, 3, 32, 32), ctx=ctx[0]))
sys.exit()
if opt.dataset == 'cifar10':
train_data = gluon.data.DataLoader(
gluon.data.vision.CIFAR10(train=True).transform_first(transform_train),
batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers)
val_data = gluon.data.DataLoader(
gluon.data.vision.CIFAR10(train=False).transform_first(transform_test),
batch_size=batch_size, shuffle=False, num_workers=num_workers)
elif opt.dataset == 'cifar100':
train_data = gluon.data.DataLoader(
gluon.data.vision.CIFAR100(train=True).transform_first(transform_train),
batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers)
val_data = gluon.data.DataLoader(
gluon.data.vision.CIFAR100(train=False).transform_first(transform_test),
batch_size=batch_size, shuffle=False, num_workers=num_workers)
else:
raise ValueError('Unknown Dataset')
if opt.no_wd and opt.cosine:
for k, v in net.collect_params('.*beta|.*gamma|.*bias').items():
v.wd_mult = 0.0
trainer = gluon.Trainer(net.collect_params(), optimizer, optimizer_params)
if opt.label_smoothing or opt.mixup:
sparse_label_loss = False
else:
sparse_label_loss = True
metric = mx.metric.Accuracy()
train_metric = mx.metric.RMSE()
loss_fn = gluon.loss.SoftmaxCrossEntropyLoss(sparse_label=sparse_label_loss)
train_history = TrainingHistory(['training-error', 'validation-error'])
iteration = 0
lr_decay_count = 0
best_val_score = 0
for epoch in range(epochs):
tic = time.time()
train_metric.reset()
metric.reset()
train_loss = 0
num_batch = len(train_data)
if not opt.cosine:
if epoch == lr_decay_epoch[lr_decay_count]:
trainer.set_learning_rate(trainer.learning_rate * lr_decay)
lr_decay_count += 1
for i, batch in enumerate(train_data):
data_1 = gluon.utils.split_and_load(batch[0], ctx_list=ctx, batch_axis=0)
label_1 = gluon.utils.split_and_load(batch[1], ctx_list=ctx, batch_axis=0)
if opt.mixup:
lam = np.random.beta(opt.mixup_alpha, opt.mixup_alpha)
if (epoch >= epochs - opt.mixup_off_epoch) or not opt.mixup:
lam = 1
data = [lam * X + (1 - lam) * X[::-1] for X in data_1]
if opt.label_smoothing:
eta = 0.1
else:
eta = 0.0
label = mixup_transform(label_1, classes, lam, eta)
elif opt.label_smoothing:
hard_label = label_1
label = smooth(label_1, classes)
with ag.record():
output = [net(X) for X in data]
loss = [loss_fn(yhat, y) for yhat, y in zip(output, label)]
for l in loss:
l.backward()
trainer.step(batch_size)
train_loss += sum([l.sum().asscalar() for l in loss])
if opt.mixup:
output_softmax = [nd.SoftmaxActivation(out) for out in output]
train_metric.update(label, output_softmax)
else:
if opt.label_smoothing:
train_metric.update(hard_label, output)
else:
train_metric.update(label, output)
name, acc = train_metric.get()
iteration += 1
train_loss /= batch_size * num_batch
name, acc = train_metric.get()
name, val_acc = test(ctx, val_data)
train_history.update([acc, 1 - val_acc])
train_history.plot(save_path='%s/%s_history.png' % (plot_name, model_name))
if val_acc > best_val_score:
best_val_score = val_acc
net.save_parameters('%s/%.4f-%s-best.params' %
(save_dir, best_val_score, model_name))
name, val_acc = test(ctx, val_data)
logging.info('[Epoch %d] train=%f val=%f loss=%f lr: %f time: %f' %
(epoch, acc, val_acc, train_loss, trainer.learning_rate,
time.time() - tic))
host_name = socket.gethostname()
with open(opt.dataset + '_' + host_name + '_GPU_' + opt.gpus + '_best_Acc.log', 'a') as f:
f.write('best Acc: {:.4f}\n'.format(best_val_score))
print("best_val_score: ", best_val_score)
