def load_images(
path,
preprocessor=None,
limit=None,
):
images = []
images_id = next(os.walk(path))[2]
size = limit if limit != None else len(images_id)
print(f"Loading {size} images")
prog = ProgressBar(100, size)
for id in range(size):
name = images_id[id]
filename = path + "/" + name
image = load_img(filename, target_size=(224, 224))
image = img_to_array(image)
image = image.reshape(
(1, image.shape[0], image.shape[1], image.shape[2]))
if preprocessor != None:
image = preprocessor.preprocess_input(image)
image_id = name.split('.')[0]
images.append([image_id, image])
prog.update(id)
print("Loading complete")
return images
def create_examples(self, lines, example_type, cached_file, save_cache):
'''
Creates examples for data
'''
label_list = self.get_labels()
if cached_file and cached_file.exists():
logger.info("Loading examples from cached file %s", cached_file)
examples = torch.load(cached_file)
else:
pbar = ProgressBar(n_total=len(lines), desc='create examples')
examples = []
for i, line in enumerate(lines):
#if i>20:break # for quik debug
guid = '%s-%d' % (example_type, i)
label = line['tags']
text_a = line['info']
text_b = None
match = line["cira_match"]
if self.test_mode == 4 and sum(match) < 4:
continue
else:
examples.append(
InputExample(guid=guid,
text_a=text_a,
text_b=text_b,
label=label,
match=match))
pbar(step=i)
if save_cache:
logger.info("Saving examples into cached file %s", cached_file)
torch.save(examples, cached_file)
return examples
def plot_traj(trajs,
fig_size=(6, 6),
color="mediumpurple",
size=5,
title='',
is_plot_line=False,
od_only=False,
offset=None):
"""plot the traj
"""
if offset is None:
offset = [0, 0]
p = ProgressBar(len(trajs), '绘制轨迹图')
plt.figure(figsize=fig_size)
for i in range(len(trajs)):
p.update(i)
traj = np.array(trajs[i])
if od_only:
traj = [traj[0], traj[-1]]
x = [x[0] + np.random.uniform(-offset[0], offset[0]) for x in traj]
y = [y[1] + np.random.uniform(-offset[1], offset[1]) for y in traj]
if od_only:
if is_plot_line:
plt.plot(x[0], y[0], c=color)
plt.plot(x[1], y[1], c="yellowgreen")
plt.scatter(x[0], y[0], c=color, s=size)
plt.scatter(x[1], y[1], c="yellowgreen", s=size)
else:
if is_plot_line:
plt.plot(x, y, c=color)
plt.scatter(x, y, c=color, s=size)
plt.title(title)
plt.show()
def make_pdb(self, bar_msg=''):
"""
Returns a pdb-like formatted string. bar_msg is a string with message to show at ProgressBar initialization.
bar_msg = '' disables the bar.
:param bar_msg: str
:return: str
"""
models = self.models()
if bar_msg:
bar = ProgressBar(len(models), bar_msg)
else:
bar = None
if len(models) == 1:
s = self.__repr__()
else:
s = ''
for m in models:
s += 'MODEL%9i\n' % m[0].model
s += m.__repr__()
s += '\nENDMDL\n'
if bar:
bar.update()
if bar:
bar.done(False)
return s
def train(self, ts, cm, batchsz=1):
self.model.train()
start_time = time.time()
steps = int(math.floor(len(ts) / float(batchsz)))
shuffle = np.random.permutation(np.arange(steps))
pg = ProgressBar(steps)
cm.reset()
total_loss = 0
for i in range(steps):
self.optimizer.zero_grad()
si = shuffle[i]
x, y = self._batch(ts, si, batchsz)
pred = self.model(x)
loss = self.crit(pred, y)
total_loss += loss.data[0]
loss.backward()
self._add_to_cm(cm, y, pred)
self.optimizer.step()
pg.update()
pg.done()
duration = time.time() - start_time
total_corr = cm.get_correct()
total = cm.get_total()
print('Train (Loss %.4f) (Acc %d/%d = %.4f) (%.3f sec)' %
(float(total_loss) / total, total_corr, total,
float(total_corr) / total, duration))
print(cm)
def remove_noise(dataset, features, verbose=False):
sample_0 = dataset['input'][0]
_, _, F = sample_0.shape
print 'Removing noise: ',
pbar = ProgressBar(len(dataset['input']) * (F - 1))
for f in range(F):
if (features[f] == 'time'): continue # no need to filter time
for sample in dataset['input']:
y = sample[:, 0, f]
# compute FT of the feature f
w = scipy.fftpack.rfft(y)
# compute mean frequency
mean = np.mean(np.abs(w))
# set the threshold to double the mean
thr = 2 * mean
# remove high frequency components
cutoff_idx = np.abs(w) < thr
w[cutoff_idx] = 0
# return to time domain by doing inverseFFT
y = scipy.fftpack.irfft(w)
sample[:, 0, f] = y
# update progress bar
pbar.next()
# return
return None
def train(train_loader, model, criterion, optimizer, epoch):
losses = AverageMeter()
len_train = len(train_loader)
pb = ProgressBar(len_train-1)
print("Training")
# Switch to train mode
model.train()
criterion.cuda()
for i, ((img,ref), score) in enumerate(train_loader):
img, ref, score = img.cuda(), ref.cuda(), score.squeeze().cuda()
# Compute output
output = model(img, ref)
loss = criterion(output, score)
# Measure accuracy and record loss
losses.update(loss.data, img.shape[0])
# Compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
pb.show(i, '[{0:5d}/{1:5d}]\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
.format(i, len_train, loss=losses))
def train(self, ts, batchsz):
self.model.train()
start_time = time.time()
steps = int(math.floor(len(ts) / float(batchsz)))
shuffle = np.random.permutation(np.arange(steps))
total_loss = total = 0
pg = ProgressBar(steps)
for i in range(steps):
self.optimizer.zero_grad()
si = shuffle[i]
ts_i = data.batch(ts, si, batchsz, long_tensor_alloc, tensor_shape,
tensor_max)
src, dst, tgt = self._wrap(ts_i)
pred = self.model((src, dst))
loss = self.crit(pred, tgt)
total_loss += loss.data[0]
loss.backward()
torch.nn.utils.clip_grad_norm(self.model.parameters(), self.clip)
total += self._total(tgt)
self.optimizer.step()
pg.update()
pg.done()
duration = time.time() - start_time
avg_loss = float(total_loss) / total
print('Train (Loss %.4f) (Perplexity %.4f) (%.3f sec)' %
(avg_loss, np.exp(avg_loss), duration))
def predict(self, steps):
print("Forecasting...")
progress_bar = ProgressBar(len(self.models.items()))
self.fcst_ds = pd.date_range(
start=self.train_ds.min(),
freq="D",
periods=len(self.train_ds)+steps)[-365:]
for item, model in self.models.items():
pred = model.predict(
exogenous=fourier(
steps,
seasonality=self.seasonality,
n_terms=self.n_fourier_terms),
n_periods=steps,
return_conf_int=True,
alpha=(1.0 - self.confidence_interval))
self.fcst[item] = pd.DataFrame(
{"yhat":pred[0],
"yhat_lower":pred[1][:,0],
"yhat_upper":pred[1][:,1]},
index=self.fcst_ds)
if self.use_boxcox:
self.fcst[item] = inv_boxcox(
self.fcst[item],
self.lmbda_boxcox[item])
progress_bar.update()
progress_bar.finish()
return pd.concat(self.fcst, axis=1)
def get_most_common(a1, a2):
temp_dict1 = {}
temp_dict2 = {}
pb = ProgressBar(worksum=len(a1), auto_display=False)
pb.startjob()
num = 0
for s1, s2 in zip(a1, a2):
num += 1
pb.complete(1)
if args.max_words != -1 and (len(s1) > args.max_words
or len(s2) > args.max_words):
continue
for w1 in s1:
temp_dict1.setdefault(w1, 0)
temp_dict1[w1] += 1
for w2 in s2:
temp_dict2.setdefault(w2, 0)
temp_dict2[w2] += 1
if num % 32 == 0:
pb.display_progress_bar()
sorted1 = sorted(temp_dict1.items(), key=lambda i: i[1], reverse=True)
sorted2 = sorted(temp_dict2.items(), key=lambda i: i[1], reverse=True)
#print(sorted1[:100])
#print(sorted2[:100])
return [i[0] for i in sorted1[:args.vac_dict_ch]
], [i[0] for i in sorted2[:args.vac_dict_en]]
def train(self, data):
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(self.logdir + "/train", self.sess.graph)
N = int(math.ceil(len(data) / self.batch_size))
cost = 0
x = np.ndarray([self.batch_size, self.win_size], dtype=np.float32)
target = np.zeros([self.batch_size, self.nwords]) # one-hot-encoded
if self.show:
from utils import ProgressBar
bar = ProgressBar('Train', max=N)
m = self.win_size;
clean_data = np.concatenate((np.zeros(self.win_size, dtype=np.int32), data)) #padding head
clean_data = np.concatenate((clean_data, np.zeros(self.batch_size, dtype=np.int32))) #padding tail
for idx in xrange(N): #interations
if self.show: bar.next()
target.fill(0)
for b in xrange(self.batch_size):
target[b][clean_data[m]] = 1 #one-batch, one example
x[b] = clean_data[m-self.win_size : m] # we need padding here!
m += 1
summary, _, loss = self.sess.run([merged, self.optim, self.loss], feed_dict={
self.input: x,
self.targets: target})
cost += np.sum(loss)
train_writer.add_summary(summary, idx)
train_writer.close()
if self.show: bar.finish()
return cost / N
def train(net, lr, trainloader, epoch):
"""
Train SSD
@args
net: (nn.Module) network
lr: (float) learning rate
trainloader: (DataLoader) dataloader
epoch: (int) training epoch
"""
net.train()
optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=1e-4)
criterion = MultiBoxLoss(num_classes=config[args.dataset]['num_classes']+1)
progress_bar = ProgressBar(total=len(trainloader))
train_loss = 0
torch.set_printoptions(threshold=10000)
for batch_idx, (images, loc_targets, conf_targets) in enumerate(trainloader):
images = Variable(images.cuda())
loc_targets = Variable(loc_targets.cuda())
conf_targets = Variable(conf_targets.cuda())
optimizer.zero_grad()
loc_preds, conf_preds = net(images)
loc_loss, conf_loss, loss = criterion(loc_preds, loc_targets, conf_preds, conf_targets)
loss.backward()
optimizer.step()
writer.add_scalar('train/loss_loc', loc_loss, batch_idx + epoch * len(trainloader))
writer.add_scalar('train/loss_conf', conf_loss, batch_idx + epoch * len(trainloader))
writer.add_scalar('train/loss_total', loss, batch_idx + epoch * len(trainloader))
train_loss += loss.item()
progress_bar.move(leftmsg="training epoch " + str(epoch), rightmsg="loss: %.6f" % (train_loss/(batch_idx+1)))
def evaluate(epoch, model, val_loader, criterion, log_path):
model.eval()
val_progressor = ProgressBar(log_path,
mode="Val ",
epoch=epoch,
total_epoch=config.epochs,
model_name=config.model_name,
total=len(val_loader))
losses = AverageMeter()
top1 = AverageMeter()
with torch.no_grad():
for index, (data, label) in enumerate(val_loader):
val_progressor.current = index
data = Variable(data).cuda()
label = Variable(torch.from_numpy(np.asarray(label))).cuda()
output = model(data)
loss = criterion(output, label)
p_top1, p_top2 = accuracy(output, label, topk=(1, 2))
losses.update(loss.item(), data.size(0))
top1.update(p_top1[0], data.size(0))
val_progressor.current_loss = losses.avg
val_progressor.current_top1 = top1.avg
val_progressor()
#print('epoch %d validate iteration %d: loss: %.3f' % (epoch + 1, index + 1, it_loss.data))
#correct += (output == label).sum()
val_progressor.done()
return losses.avg, top1.avg
def rmsd_matrix(self, msg=''):
"""
Calculates rmsd matrix with no fitting for all pairs od models in trajectory.
:return: np.array
"""
def rmsd(m1, m2, ml):
return np.sqrt(np.sum((m1 - m2)**2) / ml)
model_length = len(self.template)
models = self.coordinates.reshape(-1, model_length, 3)
dim = len(models)
result = np.zeros((dim, dim))
if msg:
bar = ProgressBar((dim * dim - dim) / 2, msg=msg)
else:
bar = None
for i in range(dim):
for j in range(i + 1, dim):
if bar:
bar.update()
result[i, j] = result[j, i] = rmsd(models[i], models[j],
model_length)
if bar:
bar.done(True)
return result
def calc_test_loss(test_x, test_y, display=True):
accs = []
worksum = int(len(test_x) / batch_size)
loss_list = []
predict_list = []
target_list = []
source_list = []
pb = ProgressBar(worksum=worksum,
info="validating...",
auto_display=display)
pb.startjob()
#test_set = Dataset(test_x,test_y)
for j in range(0, len(test_x), batch_size):
batch_x, batch_y = test_x[j:j + batch_size], test_y[
j:j + batch_size] #test_set.next_batch(batch_size)
if len(batch_x) < batch_size:
continue
bx = [len(m) + 1 for m in batch_x]
by = [len(m) + 1 for m in batch_y]
lx = [max(bx)] * batch_size
ly = [max(by)] * batch_size
batch_x = preprocessing.sequence.pad_sequences(
batch_x, max(bx), padding='post', value=en2ind_oov['<eos>'])
batch_y = preprocessing.sequence.pad_sequences(
batch_y, max(by), padding='post', value=ch2ind_oov['<eos>'])
tmp_loss, tran = session.run(
[train_loss, translations],
feed_dict={
x:
batch_x,
y:
batch_y,
y_in:
np.concatenate((np.ones(
(batch_y.shape[0], 1), dtype=np.int) * ch2ind['<go>'],
batch_y[:, :-1]),
axis=1),
x_len:
lx,
y_len:
ly,
y_real_len:
by,
x_real_len:
bx,
y_max_len:
max(by)
})
loss_list.append(tmp_loss)
tmp_acc = cal_acc(tran[:, :, 0], batch_y)
accs.append(tmp_acc)
predict_list += [i for i in tran[:, :, 0]]
target_list += [i for i in batch_y]
source_list += [i for i in batch_x]
pb.complete(1)
return np.average(loss_list), np.average(accs), get_bleu_score(
predict_list, target_list), predict_list, target_list, source_list
def fit(self, train_df, regressors=None):
print("Fitting...")
progress_bar = ProgressBar(len(train_df.columns))
for item in train_df.columns:
self.models[item] = Prophet(
yearly_seasonality=self.yearly_seasonality,
weekly_seasonality=self.weekly_seasonality,
daily_seasonality=self.daily_seasonality,
**self.prophet_config)
target = train_df[item].dropna()
if self.use_boxcox:
idx = target.index
target, self.lmbda_boxcox[item] = boxcox(target)
target = pd.Series(target, index=idx)
target.index.name = "ds"
target.name = "y"
if self.country_holidays is not None:
self.models[item].add_country_holidays(country_name=self.country_holidays)
if regressors is not None:
target = pd.merge(target, regressors, left_index=True, right_index=True, how="left")
for reg in regressors.columns:
self.models[item].add_regressor(reg)
target = target.reset_index()
self.models[item].fit(target)
progress_bar.update()
progress_bar.finish()
return self.models
def test(test_data_loader, model):
srocc = SROCC()
plcc = PLCC()
rmse = RMSE()
len_test = len(test_data_loader)
pb = ProgressBar(len_test, show_step=True)
print("Testing")
model.eval()
with torch.no_grad():
for i, ((img, ref), score) in enumerate(test_data_loader):
img, ref = img.cuda(), ref.cuda()
output = model(img, ref).cpu().data.numpy()
score = score.data.numpy()
srocc.update(score, output)
plcc.update(score, output)
rmse.update(score, output)
pb.show(
i, "Test: [{0:5d}/{1:5d}]\t"
"Score: {2:.4f}\t"
"Label: {3:.4f}".format(i + 1, len_test, float(output),
float(score)))
print("\n\nSROCC: {0:.4f}\n"
"PLCC: {1:.4f}\n"
"RMSE: {2:.4f}".format(srocc.compute(), plcc.compute(),
rmse.compute()))
def train_batches(model: Model,
train: Split,
loss,
batch_size: int,
optimizer,
display: bool = True):
losses = []
batches = Batches(train, batch_size)
pb = ProgressBar(batches.batch_N, display=display)
train.shuffle()
pb.reset()
for i in range(batches.batch_N):
model.zero_grad()
optimizer.zero_grad()
y_true = model.get_y(batches, i)
y_true = torch.autograd.Variable(torch.from_numpy(y_true)).cuda().type(
torch.cuda.FloatTensor)
model_output = model.predict(batches, i)
l = loss(model_output, y_true)
l.backward()
optimizer.step()
losses.append(l.data.cpu().numpy()[0])
pb.tick()
return losses
def validate(val_loader, model, criterion, show_step=False):
losses = AverageMeter()
srocc = SROCC()
len_val = len(val_loader)
pb = ProgressBar(len_val-1, show_step=show_step)
print("Validation")
# Switch to evaluate mode
model.eval()
with torch.no_grad():
for i, ((img,ref), score) in enumerate(val_loader):
img, ref, score = img.cuda(), ref.cuda(), score.squeeze().cuda()
# Compute output
output = model(img, ref)
loss = criterion(output, score)
losses.update(loss.data, img.shape[0])
output = output.cpu().data
score = score.cpu().data
srocc.update(score.numpy(), output.numpy())
pb.show(i, '[{0:5d}/{1:5d}]\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Output {out:.4f}\t'
'Target {tar:.4f}\t'
.format(i, len_val, loss=losses,
out=output, tar=score))
return float(1.0-srocc.compute()) # losses.avg
def _load(self):
log('[{time}] loading from {path}'.format(time=get_time(),
path=self._source_path))
for i, label_tag in enumerate(self._label_tags):
path = os.path.join(self._source_path, label_tag)
files = sample(
os.listdir(path)[self._start:self._end], self._max_num
) if self._max_num > 0 else os.listdir(path)[self._start:self._end]
print('start: {}, end: {}'.format(self._start, self._end))
print(len(files))
pbar = ProgressBar(len(files))
for j, filename in enumerate(files):
filepath = os.path.join(path, filename)
try:
with open(filepath, 'r') as f:
log_sequence = json.load(f)
feature = self._sequence2feature(log_sequence)
self._data_ids.append(
filepath.split('/')[-1].split('.')[0].split('_')
[0])
self._feature_data.append(feature)
self._label_data.append(i)
except:
log('[{time}] Failed to load file {filepath}'.format(
time=get_time(), filepath=filepath))
print('[{time}] Failed to load file {filepath}'.format(
time=get_time(), filepath=filepath))
pbar.updateBar(j)
def train(self, ts):
self.model.train()
start_time = time.time()
steps = int(len(ts))
shuffle = np.random.permutation(np.arange(steps))
total_loss = total_corr = total = 0
pg = ProgressBar(steps)
for i in range(steps):
self.optimizer.zero_grad()
si = shuffle[i]
src, dst, tgt = self._wrap(ts[si])
pred = self.model((src, dst))
loss = self.crit(pred, tgt)
total_loss += loss.data[0]
loss.backward()
total_corr += self._right(pred, tgt)
total += self._total(tgt)
self.optimizer.step()
pg.update()
pg.done()
duration = time.time() - start_time
avg_loss = float(total_loss) / total
print(
'Train (Loss %.4f) (Perplexity %.4f) (Acc %d/%d = %.4f) (%.3f sec)'
% (avg_loss, np.exp(avg_loss), total_corr, total,
float(total_corr) / total, duration))
def _preprocess(self):
self.lang = Lang()
for text in self._texts:
self.lang.index_text(text)
for text in self._texts:
indexes = indexes_from_text(self.lang, text)
indexes.append(EOT_token)
padded_indexes = pad_indexes(indexes, self._max_text_length,
PAD_token)
self._indexed_texts.append(padded_indexes)
self._indexed_texts = np.stack(self._indexed_texts, axis=0)
bar = ProgressBar(len(self._audio_files) - 1, unit='')
for (audio_files_read, audio_file) in enumerate(self._audio_files):
# (n_mels, T), (1+n_fft/2, T)
mel, mag = compute_spectrograms(audio_file)
padded_mel = pad_time_dim(mel, self._max_audio_length, 0)
padded_mag = pad_time_dim(mag, self._max_audio_length, 0)
self._mels.append(padded_mel.transpose())
self._mags.append(padded_mag.transpose())
bar.update(audio_files_read)
self._mels = np.stack(self._mels, axis=0)
self._mags = np.stack(self._mags, axis=0)
def generate(self):
progress_bar = ProgressBar(self.settings.nb_rrd_files)
self.add_header(self.settings)
for domain in self.settings.domains:
for host in self.settings.domains[domain].hosts:
row = self.add_row("{0} / {1}".format(domain, host))
for plugin in self.settings.domains[domain].hosts[
host].plugins:
_plugin = self.settings.domains[domain].hosts[
host].plugins[plugin]
panel = row.add_panel(
_plugin.settings["graph_title"] or plugin, plugin)
for field in _plugin.fields:
query = panel.add_query(field)
if "label" in _plugin.fields[field].settings:
query.alias = _plugin.fields[field].settings[
"label"]
progress_bar.update()
panel.width = 12 // self.settings.grafana['graph_per_row']
panel.process_graph_settings(_plugin.settings)
panel.process_graph_thresholds(_plugin.fields)
panel.process_graph_types(_plugin.fields)
def train(self):
avg_G_loss = 0
avg_D_loss = 0
avg_Q_loss = 0
iterations = int(self.mnist.train.num_examples / self.batch_size)
if self.show_progress:
bar = ProgressBar('Train', max=iterations)
for i in range(iterations):
if self.show_progress:
bar.next()
batch_xs, _ = self.mnist.train.next_batch(self.batch_size)
feed_dict = {self.X: batch_xs, \
self.z: self.z_sampler(self.batch_size), \
self.c_i: self.c_cat_sampler(self.batch_size), \
self.c_j: self.c_cont_sampler(self.batch_size), \
self.training: True}
for _ in range(self.d_update):
_, D_loss = self.sess.run([self.D_optim, self.D_loss], feed_dict=feed_dict)
_, G_loss = self.sess.run([self.G_optim, self.G_loss], feed_dict=feed_dict)
_, Q_loss = self.sess.run([self.Q_optim, self.Q_loss], feed_dict=feed_dict)
avg_G_loss += G_loss / iterations
avg_D_loss += D_loss / iterations
avg_Q_loss += Q_loss / iterations
if self.show_progress:
bar.finish()
return avg_G_loss, avg_D_loss, avg_Q_loss
def parallel_test(model_cls,
model_kwargs,
checkpoint,
dataset,
data_func,
gpus,
worker_per_gpu=1):
ctx = multiprocessing.get_context('spawn')
idx_queue = ctx.Queue()
result_queue = ctx.Queue()
num_workers = len(gpus) * worker_per_gpu
workers = [
ctx.Process(target=worker_func,
args=(model_cls, model_kwargs, checkpoint, dataset,
data_func, gpus[i % len(gpus)], idx_queue,
result_queue)) for i in range(num_workers)
]
for w in workers:
w.daemon = True
w.start()
for i in range(len(dataset)):
idx_queue.put(i)
results = {}
prog_bar = ProgressBar(task_num=len(dataset))
for _ in range(len(dataset)):
img_id, res = result_queue.get()
results[img_id] = format_ret(res)
prog_bar.update()
print('\n')
for worker in workers:
worker.terminate()
return results
def calc_test_loss(test_set = Dataset(test_x,test_y),display=True):
accs = []
worksum = int(len(test_x) / batch_size)
loss_list = []
predict_list = []
target_list = []
source_list = []
pb = ProgressBar(worksum=worksum,info="validating...",auto_display=display)
pb.startjob()
#test_set = Dataset(test_x,test_y)
for j in range(worksum):
batch_x,batch_y = test_set.next_batch(batch_size)
lx = [seq_max_len] * batch_size
ly = [seq_max_len] * batch_size
bx = [np.sum(m > 0) for m in batch_x]
by = [np.sum(m > 0) for m in batch_y]
tmp_loss,tran = session.run([train_loss,translations],feed_dict={x:batch_x,y:batch_y,
y_in:
np.concatenate((
np.ones((batch_y.shape[0],1),dtype=np.int) * ch2ind['<go>'],batch_y[:,:-1]) ,axis=1)
,x_len:lx,y_len:ly,
y_real_len:by,
x_real_len:bx})
loss_list.append(tmp_loss)
tmp_acc = cal_acc(tran,batch_y)
accs.append(tmp_acc)
predict_list += [i for i in tran]
target_list += [i for i in batch_y]
source_list += [i for i in batch_x]
pb.complete(1)
return np.average(loss_list),np.average(accs),get_bleu_score(predict_list,target_list),predict_list,target_list,source_list
def _write_per_dir(self):
"""
write outputs of each directory to a different file
"""
bag_of_words = BagOfWordsParser(self._words_filter)
reviews_per_dir = defaultdict(list)
flags = Reviewer._get_flags_text(self._output_per_dir,
self._output_in_svm_light,
self._output_in_tfdidf)
for dir in map(os.path.abspath, self._dirs):
numoffiles = len(os.listdir(dir))
filename = Reviewer._get_filename(dir, flags)
reviews = reviews_per_dir[filename]
prefix = "crunching reviews for '{}'".format(filename)
with ProgressBar(prefix, numoffiles) as pb:
for review in bag_of_words.parse_dir(dir):
pb.report()
reviews.append(review)
assert numoffiles == len(reviews)
# we update here because the bag of words is not updated yet
# when we're still reading the files in the directories
for filename, reviews in reviews_per_dir.items():
self._write(filename, bag_of_words, reviews)
def test(model: nn.Module,
test_loader: DataLoader,
use_cuda: bool,
loss_func: Callable[[Tensor, Tensor], Tensor] = nnf.cross_entropy,
input_ops: Callable[[Tensor], Tensor] = None,
output_ops: Callable[..., Tensor] = None) -> Tuple[float, float]:
model.eval()
tot_loss = 0.
correct = 0
total = 0
print('\nTest started:')
sys.stdout.flush()
pb = ProgressBar()
for batch_idx, (data, target) in enumerate(test_loader):
(loss, correct, total,
tot_loss) = step(model, data, target, use_cuda, correct, total,
tot_loss, None, loss_func, input_ops, output_ops)
pb.progress(batch_idx / len(test_loader))
pb.complete()
avg_loss = tot_loss / total
accuracy = correct / total
print('\n{0}: Test result: mean loss = {1:f}, accuracy = {2:f}'.format(
get_timestamp(), avg_loss, accuracy))
sys.stdout.flush()
return avg_loss, accuracy
def export_to_xml_in_folder(source, destination=Defaults.MUNIN_XML_FOLDER):
"""
Calls "rrdtool dump" to convert RRD database files in "source" folder to XML representation
Converts all *.rrd files in source folder
"""
assert os.path.exists(source)
try:
os.makedirs(destination)
except OSError as e:
if e.errno != errno.EEXIST:
raise
filelist = [("", os.path.join(source, file)) for file in os.listdir(source)
if file.endswith(".rrd")]
nb_files = len(filelist)
progress_bar = ProgressBar(nb_files)
print("Exporting {0} RRD databases:".format(nb_files))
for domain, file in filelist:
src = os.path.join(source, domain, file)
dst = os.path.join(
destination, "{0}-{1}".format(domain,
file).replace(".rrd", ".xml"))
progress_bar.update()
code = subprocess.check_call(['rrdtool', 'dump', src, dst])
return nb_files
def train(self, ts, cm, dropout, batchsz=1):
total_loss = 0
start_time = time.time()
steps = int(math.floor(len(ts)/float(batchsz)))
shuffle = np.random.permutation(np.arange(steps))
pg = ProgressBar(steps)
cm.reset()
for i in range(steps):
si = shuffle[i]
ts_i = data.batch(ts, si, batchsz)
feed_dict = self.model.ex2dict(ts_i, 1.0-dropout)
_, step, summary_str, lossv, guess = self.sess.run([self.train_op, self.global_step, self.summary_op, self.loss, self.model.best], feed_dict=feed_dict)
self.train_writer.add_summary(summary_str, step)
total_loss += lossv
cm.add_batch(ts_i.y, guess)
pg.update()
pg.done()
total = cm.get_total()
total_corr = cm.get_correct()
duration = time.time() - start_time
print('Train (Loss %.4f) (Acc %d/%d = %.4f) (%.3f sec)' % (float(total_loss)/total, total_corr, total, float(total_corr)/total, duration))
print(cm)
