def __init__(self, model_size, group_id, environment_id=0, training=True):
self.model_size = model_size
self._training = training
self.environment_id = environment_id
self.group_id = group_id
# Build environment
self.environment = Environment.create_environment(flags.env_type, self.environment_id, self._training)
self.extrinsic_reward_manipulator = eval(flags.extrinsic_reward_manipulator)
self.terminal = True
self._composite_batch = CompositeBatch(maxlen=flags.replay_buffer_size if flags.replay_mean > 0 else 1)
# Statistics
self.__client_statistics = Statistics(flags.episode_count_for_evaluation)
if self._training:
#logs
if not os.path.isdir(flags.log_dir + "/performance"):
os.mkdir(flags.log_dir + "/performance")
if not os.path.isdir(flags.log_dir + "/episodes"):
os.mkdir(flags.log_dir + "/episodes")
formatter = logging.Formatter('%(asctime)s %(message)s')
# reward logger
self.__reward_logger = logging.getLogger('reward_{}_{}'.format(self.group_id, self.environment_id))
hdlr = logging.FileHandler(flags.log_dir + '/performance/reward_{}_{}.log'.format(self.group_id, self.environment_id))
hdlr.setFormatter(formatter)
self.__reward_logger.addHandler(hdlr)
self.__reward_logger.setLevel(logging.DEBUG)
self.__max_reward = float("-inf")
def report_training(self,
step,
num_steps,
learning_rate,
report_stats,
multigpu=False):
"""
This is the user-defined batch-level traing progress
report function.
Args:
step(int): current step count.
num_steps(int): total number of batches.
learning_rate(float): current learning rate.
report_stats(Statistics): old Statistics instance.
Returns:
report_stats(Statistics): updated Statistics instance.
"""
if self.start_time < 0:
raise ValueError("""ReportMgr needs to be started
(set 'start_time' or use 'start()'""")
if step % self.report_every == 0:
if multigpu:
report_stats = \
Statistics.all_gather_stats(report_stats)
self._report_training(step, num_steps, learning_rate, report_stats)
self.progress_step += 1
return Statistics()
else:
return report_stats
def __init__(self, args: Namespace, logger: HtmlLogger):
# init model
model = self.buildModel(args)
model = model.cuda()
# create DataParallel model instance
self.modelParallel = model
# self.modelParallel = DataParallel(model, args.gpu)
# assert (id(model) == id(self.modelParallel.module))
self.args = args
self.model = model
self.logger = logger
# load data
self.train_queue, self.valid_queue, self.createSearchQueue = load_data(
args)
# init train folder path, where to save loggers, checkpoints, etc.
self.trainFolderPath = '{}/{}'.format(args.save, args.trainFolder)
# build statistics containers
containers = self.buildStatsContainers()
# build statistics rules
rules = self.buildStatsRules()
# init statistics instance
self.statistics = Statistics(containers, rules, args.save)
# log parameters
logParameters(logger, args, model)
def __init__(self, cf):
self.cf = cf
self.net = None
self.loss = None
self.optimizer = None
self.scheduler = None
self.best_stats = Statistics()
def validate(self, valid_iter):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics()
for batch in valid_iter:
src = make_features(batch, 'src')
_, src_lengths = batch.src
tgt = make_features(batch, 'tgt')
# F-prop through the model.
outputs, attns = self.model(src, tgt, src_lengths)
# Compute loss.
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs, attns)
# Update statistics.
stats.update(batch_stats)
# Set model back to training mode.
self.model.train()
return stats
def validate(self, valid_iter):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics()
with torch.no_grad():
for batch in valid_iter:
src = make_features(batch, 'src')
src = src.transpose(0, 1).contiguous()
# _, src_lengths = batch.src
src_lengths = (torch.ones(batch.batch_size) * src.size(1)).long()
tgt = make_features(batch, 'tgt')
# F-prop through the model.
outputs, attns = self.model(src, tgt, src_lengths)
# Compute loss.
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs, attns)
# Update statistics.
stats.update(batch_stats)
# Set model back to training mode.
self.model.train()
return stats
def __init__(self, cf, model):
self.cf = cf
self.model = model
self.logger_stats = Logger(cf.log_file_stats)
self.stats = Statistics()
self.msg = Messages()
self.validator = self.validation(self.logger_stats, self.model, cf, self.stats, self.msg)
self.trainer = self.train(self.logger_stats, self.model, cf, self.validator, self.stats, self.msg)
self.predictor = self.predict(self.logger_stats, self.model, cf)
def validate_one_epoch(val_loader, model, trainer, args):
# switch to evaluate mode
model.eval()
val_stats = [Statistics() for i in range(5)]
# read-in global entity list
if args.dataset == 'kvr':
with open('data/KVR/kvret_entities.json') as f:
global_entity = json.load(f)
global_entity_list = []
for key in global_entity.keys():
if key != 'poi':
global_entity_list += [item.lower().replace(' ', '_') for item in global_entity[key]]
else:
for item in global_entity['poi']:
global_entity_list += [item[k].lower().replace(' ', '_') for k in item.keys()]
global_entity_list = list(set(global_entity_list))
else:
raise NotImplementedError('Not implemented this val for datasets other than kvr yet.')
with torch.no_grad():
end = time.time()
# for i, data in tqdm(enumerate(val_loader)):
cnt = 0
for data in tqdm(val_loader):
# data = to_device()
# end = time.time()
cnt += 1
decoded_words = trainer.evaluate_batch(model, data)
# logger.info("Decode Time cost: {}".format(str(time.time() - end)))
# end = time.time()
# update val states for each batch.
val_stats = compute_val_stat(data, decoded_words, global_entity_list, val_stats, args)
# logger.info("Val Compute Time cost: {}".format(str(time.time() - end)))
if args.distributed:
all_val_stats = Statistics.all_gather_stats_list(val_stats)
else:
all_val_stats = val_stats
f1 = all_val_stats[0].accuracy()
cal_f1 = all_val_stats[1].accuracy()
wet_f1 = all_val_stats[2].accuracy()
nav_f1 = all_val_stats[3].accuracy()
logger.info("F1 SCORE:\t{}".format(str(f1)))
logger.info("\tCAL F1:\t{}".format(str(cal_f1)))
logger.info("\tWET F1:\t{}".format(str(wet_f1)))
logger.info("\tNAV F1:\t{}".format(str(nav_f1)))
bleu_score = all_val_stats[4].accuracy() / 100.0
# not validated yet.
# bleu_score = 0.0
logger.info("\tBleu Score:\t{}".format(str(bleu_score)))
return bleu_score, [f1, cal_f1, wet_f1, nav_f1]
def before_epoch(self, _epoch):
cfg = self.cfg
self._current_epoch = _epoch
if cfg.n_gpu > 1:
torch.distributed.barrier()
self._model.train()
self._epoch_tr_loss = 0.0
self._epoch_n_tr_steps = 0.0
if cfg.is_master_node:
self._epoch_stats = Statistics(epoch_num=int(cfg.epoch_num),
total_training_steps=self._optimizer.total_training_steps)
def test(self):
result_file = '{}/test_results_{}.log'.format(flags.log_dir,
self.global_step)
if os.path.exists(result_file):
print('Test results already produced and evaluated for {}'.format(
result_file))
return
result_lock = RLock()
print('Start testing')
testers = []
threads = []
tf_session = tf.get_default_session()
tmp_environment = Environment.create_environment(
env_type=flags.env_type, training=False)
dataset_size = tmp_environment.get_dataset_size()
data_per_thread = max(1, dataset_size // self.thread_count)
for i in range(self.thread_count): # parallel testing
tester = Group(group_id=-(i + 1),
environment_count=data_per_thread,
global_network=self.global_network,
training=False)
data_range_start = i * data_per_thread
data_range_end = data_range_start + data_per_thread
# print(data_range_start, data_per_thread, dataset_size)
thread = Thread(target=self.test_function,
args=(result_file, result_lock, tester,
(data_range_start,
data_range_end), tf_session))
thread.start()
threads.append(thread)
testers.append(tester)
print('Test Set size:', dataset_size)
print('Tests per thread:', data_per_thread)
time.sleep(5)
for thread in threads: # wait for all threads to end
thread.join()
print('End testing')
# get overall statistics
test_statistics = Statistics(self.thread_count)
for group in testers:
test_statistics.add(group.get_statistics())
info = test_statistics.get()
# write results to file
stats_file = '{}/test_statistics.log'.format(flags.log_dir)
with open(stats_file, "a",
encoding="utf-8") as file: # write stats to file
file.write('{}\n'.format([
"{}={}".format(key, value)
for key, value in sorted(info.items(), key=lambda t: t[0])
]))
print('Test statistics saved in {}'.format(stats_file))
print('Test results saved in {}'.format(result_file))
return tmp_environment.evaluate_test_results(result_file)
def _report_training(self, step, num_steps, learning_rate, report_stats):
"""
See base class method `ReportMgrBase.report_training`.
"""
report_stats.output(step, num_steps, learning_rate, self.start_time)
# Log the progress using the number of batches on the x-axis.
self.maybe_log_tensorboard(report_stats, "progress", learning_rate,
self.progress_step)
report_stats = Statistics()
return report_stats
def main():
ITER_MAX = 30
list_statistics = []
e = Export(list_statistics)
for i in range(1, 17):
name_file = ""
if i < 11:
name_file = "./data/files/f{}.txt".format(i)
else:
name_file = "./data/files/Knapsack{}.txt".format(i - 10)
statistics = Statistics(name_file, i, ITER_MAX)
k = Knapsack(name_file)
hcc = HillclimbingClassic()
hcm = RandomSearch()
vns = VNS(0)
algorithms = []
algorithms.append(hcm)
algorithms.append(hcc)
algorithms.append(vns)
hcm.max_efos = 1000
hcc.max_efos = 1000
vns.max_efos = 1000
information = [0] * 2
sublist_statistics = [0] * len(algorithms)
print(name_file)
j = 0
for algorithm in algorithms:
vector = []
successfull_count = 0
start_time = time()
for l in range(ITER_MAX):
random.seed(l)
k_max = random.randint(
2, int(math.log10(k.total_items) +
2)) if k.total_items < 6 else random.randint(
3, int(math.log10(k.total_items) + 3))
vns.k_max = k_max
algorithm.execute(k, None)
vector.append(algorithm.best_solution.fitness)
successfull_count += 1 if algorithm.successfull else 0
end_time = time()
information[0] = algorithm.__str__()
information[1] = vector
statistics.set_vector(information)
statistics.successfull_count = successfull_count
sublist_statistics[j] = copy.deepcopy(statistics)
print("{}min\t\t{}".format(round((end_time - start_time) / 60, 3),
algorithm))
j += 1
list_statistics.append(copy.deepcopy(sublist_statistics))
e.writeCSV()
e.writeHTML()
def validate(test_loader, model, criterion, device):
statistics = Statistics()
# switch to evaluate mode
model.eval()
with torch.no_grad():
for batch_idx, (input_val, target_val) in enumerate(test_loader):
loss, (prec1, prec5), y_pred, y_true = execute_batch(
model, criterion, input_val, target_val, device)
statistics.update(loss.data.cpu().numpy(), prec1, prec5, y_pred,
y_true)
return statistics
def train(self, train_steps, train_steps2, valid_steps):
logger.info('Start training...')
task_step = self.optim._task_step + 1
task2_step = self.optim._task2_step + 1
self.train_iter = self.get_task_batch(task_step, task_type='task')
self.train_iter2 = self.get_task_batch(task2_step, task_type='task2')
self.total_stats = Statistics(task_type='task')
self.report_stats = Statistics(task_type='task')
self._start_report_manager(self.report_manager,
start_time=self.total_stats.start_time)
self.total_stats2 = Statistics(task_type='task2')
self.report_stats2 = Statistics(task_type='task2')
self._start_report_manager(self.report_manager2,
start_time=self.total_stats2.start_time)
while task_step <= train_steps or task2_step <= train_steps2:
self.save = False
# self.save = True
if task_step <= train_steps:
task_step = self.train_task(task_step,
train_steps,
valid_steps,
task_type='task')
# self.save = False
# self.save = True
if task2_step <= train_steps2:
task2_step = self.train_task(task2_step,
train_steps2,
valid_steps,
task_type='task2')
return self.total_stats, self.total_stats2
def _stats(self, loss, scores, target):
"""
Args:
loss (:obj:`FloatTensor`): the loss computed by the loss criterion.
scores (:obj:`FloatTensor`): a score for each possible output
target (:obj:`FloatTensor`): true targets
Returns:
:obj:`onmt.utils.Statistics` : statistics for this batch.
"""
pred = scores.max(1)[1]
non_padding = target.ne(self.padding_idx)
num_correct = pred.eq(target).masked_select(non_padding).sum().item()
num_non_padding = non_padding.sum().item()
return Statistics(loss.item(), num_non_padding, num_correct)
def sharded_compute_loss(self, batch, output, attns, shard_size,
normalization, ratio):
"""Compute the forward loss and backpropagate. Computation is done
with shards and optionally truncation for memory efficiency.
Also supports truncated BPTT for long sequences by taking a
range in the decoder output sequence to back propagate in.
Range is from `(cur_trunc, cur_trunc + trunc_size)`.
Note sharding is an exact efficiency trick to relieve memory
required for the generation buffers. Truncation is an
approximate efficiency trick to relieve the memory required
in the RNN buffers.
Args:
batch (batch) : batch of labeled examples
output (:obj:`FloatTensor`) :
output of decoder model `[tgt_len x batch x hidden]`
attns (dict) : dictionary of attention distributions
`[tgt_len x batch x src_len]`
cur_trunc (int) : starting position of truncation window
trunc_size (int) : length of truncation window
shard_size (int) : maximum number of examples in a shard
normalization (int) : Loss is divided by this number
Returns:
:obj:`onmt.utils.Statistics`: validation loss statistics
"""
batch_stats = Statistics()
shard_state = self._make_shard_state(batch, output, attns)
for shard in shards({k: v[0]
for k, v in shard_state.items()},
shard_size,
retain_graph=True,
ratio=ratio):
loss, stats = self._compute_loss(batch, **shard)
loss.div(float(normalization[0])).backward(retain_graph=True)
batch_stats.update(stats)
for shard in shards({k: v[1]
for k, v in shard_state.items()},
shard_size,
ratio=1 - ratio):
loss, stats = self._compute_loss(batch, **shard)
loss *= ratio
loss.div(float(normalization[1])).backward()
batch_stats.update(stats)
return batch_stats
def sharded_compute_loss(self, batch, output, attns,
cur_trunc, trunc_size, shard_size,
normalization):
"""Compute the forward loss and backpropagate. Computation is done
with shards and optionally truncation for memory efficiency.
Also supports truncated BPTT for long sequences by taking a
range in the decoder output sequence to back propagate in.
Range is from `(cur_trunc, cur_trunc + trunc_size)`.
Note sharding is an exact efficiency trick to relieve memory
required for the generation buffers. Truncation is an
approximate efficiency trick to relieve the memory required
in the RNN buffers.
Args:
batch (batch) : batch of labeled examples
output (:obj:`FloatTensor`) :
output of decoder model `[tgt_len x batch x hidden]`
attns (dict) : dictionary of attention distributions
`[tgt_len x batch x src_len]`
cur_trunc (int) : starting position of truncation window
trunc_size (int) : length of truncation window
shard_size (int) : maximum number of examples in a shard
normalization (int) : Loss is divided by this number
Returns:
:obj:`onmt.utils.Statistics`: validation loss statistics
"""
batch_stats = Statistics()
# 0-len_tgt_size
range_ = (cur_trunc, cur_trunc + trunc_size)
shard_state = self._make_shard_state(batch, output, range_, attns)
'''
return {
"output": output,
"target": batch.tgt[1:len_tgt_size]
}
'''
# shard_size被设置为2
# shard: {"output": output_, "target": target_}
for shard in shards(shard_state, shard_size):
loss, stats = self._compute_loss(batch, **shard)
# backward, 这里的div?
loss.div(float(normalization)).backward()
batch_stats.update(stats)
return batch_stats
def go(request):
product_id = request.GET.get('id')
try:
obj = Product.objects.get(pk=product_id, is_active=True)
except ObjectDoesNotExist:
return HttpResponseNotFound()
product_id = obj.id
identify = get_request_identify(request)
r = Statistics(product_id).add_uv(
dict(meta=request.META, identify=identify))
if r == 'success':
Product.objects.filter(pk=product_id).update(uv=F('uv') + 1)
return HttpResponseRedirect(obj.url)
def setUp(self):
super().setUp()
listeners = set()
statistics = Statistics()
self.source_server_clients = []
self.listener_server_clients = []
self.source_server = TornadoTCPServer(statistics, listeners,
source.SourceHandler)
sock, self.source_server_port = bind_unused_port()
self.source_server.add_socket(sock)
self.listener_server = TornadoTCPServer(statistics, listeners,
listener.ListenerHandler)
sock, self.listener_server_port = bind_unused_port()
self.listener_server.add_socket(sock)
self.source_client_msg_one = bytes(
b'\x01' + int.to_bytes(1, 2, byteorder='big', signed=False) +
'abcdefgh'.encode() + b'\x01' +
int.to_bytes(1, 1, byteorder='big', signed=False) +
'foofield'.encode() +
int.to_bytes(1, 4, byteorder='big', signed=False))
self.source_client_msg_one += _get_xor(self.source_client_msg_one)
self.source_client_msg_two = bytes(
b'\x01' + int.to_bytes(1, 2, byteorder='big', signed=False) +
'ijklmnop'.encode() + b'\x01' +
int.to_bytes(1, 1, byteorder='big', signed=False) +
'fieldfoo'.encode() +
int.to_bytes(2, 4, byteorder='big', signed=False))
self.source_client_msg_two += _get_xor(self.source_client_msg_two)
self.source_client_invalid_msg = bytes(
b'\x01' + int.to_bytes(10, 2, byteorder='big', signed=False) +
'ijklmnop'.encode() + b'\x01' +
int.to_bytes(1, 1, byteorder='big', signed=False) +
'fieldfoo'.encode() +
int.to_bytes(2, 4, byteorder='big', signed=False) +
int.to_bytes(1, 1, byteorder='big', signed=False))
def _stats(self, loss, scores, target):
"""
Args:
loss (:obj:`FloatTensor`): the loss computed by the loss criterion.
scores (:obj:`FloatTensor`): a score for each possible output
target (:obj:`FloatTensor`): true targets
1 / x * vocab_size / x
Returns:
:obj:`onmt.utils.Statistics` : statistics for this batch.
"""
# 返回行的最大值的索引即预测的单词
# (shard_size*batch)
pred = scores.max(1)[1]
# (shard_size*batch)
non_padding = target.ne(self.padding_idx)
# 排除掉填充字符，得到正确的单词个数
num_correct = pred.eq(target).masked_select(non_padding).sum().item()
# 总的单词个数
num_non_padding = non_padding.sum().item()
return Statistics(loss.item(), num_non_padding, num_correct)
def __init__(self, group_id, model_id, environment_info, beta=None, training=True, parent=None, sibling=None):
self.parameters_type = eval('tf.{}'.format(flags.parameters_type))
self.beta = beta if beta is not None else flags.beta
self.value_count = 2 if flags.split_values else 1
# initialize
self.training = training
self.group_id = group_id
self.model_id = model_id
self.id = '{0}_{1}'.format(self.group_id,self.model_id) # model id
self.parent = parent if parent is not None else self # used for sharing with other models in hierarchy, if any
self.sibling = sibling if sibling is not None else self # used for sharing with other models in hierarchy, if any
# Environment info
action_shape = environment_info['action_shape']
self.policy_heads = [
{
'size':head[0], # number of actions to take
'depth':head[1] if len(head) > 1 else 0 # number of discrete action types: set 0 for continuous control
}
for head in action_shape
]
state_shape = environment_info['state_shape']
self.state_heads = [
{'shape':head}
for head in state_shape
]
self.state_scaler = environment_info['state_scaler'] # state scaler, for saving memory (eg. in case of RGB input: uint8 takes less memory than float64)
self.has_masked_actions = environment_info['has_masked_actions']
# Create the network
self.build_input_placeholders()
self.initialize_network()
self.build_network()
# Stuff for building the big-batch and optimize training computations
self._big_batch_feed = [{},{}]
self._batch_count = [0,0]
self._train_batch_size = flags.batch_size*flags.big_batch_size
# Statistics
self._train_statistics = Statistics(flags.episode_count_for_evaluation)
#=======================================================================
# self.loss_distribution_estimator = RunningMeanStd(batch_size=flags.batch_size)
#=======================================================================
self.actor_loss_is_too_small = False
def validate(self, valid_iter, task_type='task'):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics(task_type=task_type)
with torch.no_grad():
for batch in valid_iter:
src = make_features(batch, 'src')
_, src_lengths = batch.src
if task_type == 'task':
tgt = make_features(batch, 'tgt')
else:
tgt = make_features(batch, 'tgt2')
# F-prop through the model.
outputs, attns = self.model(src,
tgt,
src_lengths,
task_type=task_type)
# Compute loss.
if task_type == 'task':
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs, attns)
else:
batch_stats = self.valid_loss2.monolithic_compute_loss(
batch, outputs, attns)
# Update statistics.
stats.update(batch_stats)
# Set model back to training mode.
self.model.train()
return stats
def validate(self, valid_iter):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics()
for batch in valid_iter:
src = make_features(batch, 'src')
_, src_lengths = batch.src
tgt = make_features(batch, 'tgt')
structure1 = make_features(batch, 'structure1')
structure1 = structure1.transpose(0, 1)
structure1 = structure1.transpose(1, 2)
structure2 = make_features(batch, 'structure2')
structure2 = structure2.transpose(0, 1)
structure2 = structure2.transpose(1, 2)
structure3 = make_features(batch, 'structure3')
structure3 = structure3.transpose(0, 1)
structure3 = structure3.transpose(1, 2)
structure4 = make_features(batch, 'structure4')
structure4 = structure4.transpose(0, 1)
structure4 = structure4.transpose(1, 2)
structure5 = make_features(batch, 'structure5')
structure5 = structure5.transpose(0, 1)
structure5 = structure5.transpose(1, 2)
# structure6 = make_features(batch, 'structure6')
# structure6 = structure6.transpose(0, 1)
# structure6 = structure6.transpose(1, 2)
#
# structure7 = make_features(batch, 'structure7')
# structure7 = structure7.transpose(0, 1)
# structure7 = structure7.transpose(1, 2)
#
# structure8 = make_features(batch, 'structure8')
# structure8 = structure8.transpose(0, 1)
# structure8 = structure8.transpose(1, 2)
# F-prop through the model.
outputs, attns = self.model(src, tgt, structure1, structure2,
structure3, structure4, structure5,
src_lengths)
# Compute loss.
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs, attns)
# Update statistics.
stats.update(batch_stats)
# Set model back to training mode.
self.model.train()
return stats
def train(self, train_iter_fct, valid_iter_fct, train_steps, valid_steps):
"""
The main training loops.
by iterating over training data (i.e. `train_iter_fct`)
and running validation (i.e. iterating over `valid_iter_fct`
Args:
train_iter_fct(function): a function that returns the train
iterator. e.g. something like
train_iter_fct = lambda: generator(*args, **kwargs)
valid_iter_fct(function): same as train_iter_fct, for valid data
train_steps(int):
valid_steps(int):
save_checkpoint_steps(int):
Return:
None
"""
logger.info('Start training...')
step = self.optim._step + 1
true_batchs = []
accum = 0
normalization = 0
train_iter = train_iter_fct()
total_stats = Statistics()
report_stats = Statistics()
self._start_report_manager(start_time=total_stats.start_time)
while step <= train_steps:
reduce_counter = 0
for i, batch in enumerate(train_iter):
if self.n_gpu == 0 or (i % self.n_gpu == self.gpu_rank):
if self.gpu_verbose_level > 1:
logger.info("GpuRank %d: index: %d accum: %d" %
(self.gpu_rank, i, accum))
true_batchs.append(batch)
if self.norm_method == "tokens":
num_tokens = batch.tgt[1:].ne(
self.train_loss.padding_idx).sum()
normalization += num_tokens.item()
else:
normalization += batch.batch_size
accum += 1
if accum == self.grad_accum_count:
reduce_counter += 1
if self.gpu_verbose_level > 0:
logger.info("GpuRank %d: reduce_counter: %d \
n_minibatch %d" % (self.gpu_rank, reduce_counter,
len(true_batchs)))
if self.n_gpu > 1:
normalization = sum(all_gather_list(normalization))
self._gradient_accumulation(true_batchs, normalization,
total_stats, report_stats)
report_stats = self._maybe_report_training(
step, train_steps, self.optim.learning_rate,
report_stats)
true_batchs = []
accum = 0
normalization = 0
if (step % valid_steps == 0):
if self.gpu_verbose_level > 0:
logger.info('GpuRank %d: validate step %d' %
(self.gpu_rank, step))
valid_iter = valid_iter_fct()
valid_stats = self.validate(valid_iter)
if self.gpu_verbose_level > 0:
logger.info('GpuRank %d: gather valid stat \
step %d' % (self.gpu_rank, step))
valid_stats = self._maybe_gather_stats(valid_stats)
if self.gpu_verbose_level > 0:
logger.info('GpuRank %d: report stat step %d' %
(self.gpu_rank, step))
self._report_step(self.optim.learning_rate,
step,
valid_stats=valid_stats)
if self.gpu_rank == 0:
self._maybe_save(step)
step += 1
if step > train_steps:
break
if self.gpu_verbose_level > 0:
logger.info('GpuRank %d: we completed an epoch \
at step %d' % (self.gpu_rank, step))
train_iter = train_iter_fct()
return total_stats
def __init__(self, cf):
super(Model, self).__init__()
self.url = None
self.cf = cf
self.best_stats = Statistics()
def train(model, criterion, optimizer, train_loader, val_loader, args,
single_view):
# Best prediction
best_prec1 = 0
epoch_no_improve = 0
for epoch in range(args.epochs):
if not single_view:
# Give random permutation to the images
indices = train_loader.dataset.indices
inds = random_permute_indices(np.array(indices), args.nview)
train_loader.dataset.indices = np.asarray(inds)
del indices
del inds
statistics = Statistics()
# switch to train mode
model.train()
start_time = time.time()
for batch_idx, (input_val, target_val) in enumerate(train_loader):
loss, (prec1, prec5), y_pred, y_true = execute_batch(
model, criterion, input_val, target_val, args, single_view)
statistics.update(loss.detach().cpu().numpy(), prec1, prec5,
y_pred, y_true)
# compute gradient and do optimizer step
optimizer.zero_grad()
loss.backward()
optimizer.step()
del loss
torch.cuda.empty_cache()
elapsed_time = time.time() - start_time
# Evaluate on validation set
val_statistics = validate(val_loader, model, criterion, args,
single_view)
# statistics.compute(args.num_classes)
# val_statistics.compute(args.num_classes)
log_data(statistics, "train", val_loader.dataset.dataset.classes,
epoch)
log_data(val_statistics, "internal_val",
val_loader.dataset.dataset.classes, epoch)
wandb.log({"Epoch elapsed time": elapsed_time}, step=epoch)
# Save best model and best prediction
if val_statistics.top1.avg > best_prec1:
best_prec1 = val_statistics.top1.avg
save_model(args.arch, model, optimizer, args.fname_best)
epoch_no_improve = 0
else:
# Early stopping
epoch_no_improve += 1
if epoch_no_improve == args.patience:
wandb.run.summary[
"best_internal_val_top1_accuracy"] = best_prec1
wandb.run.summary[
"best_internal_val_top1_accuracy_epoch"] = epoch - args.patience
return
def __init__(self, socket_mock=None):
super().__init__()
self.statistics = Statistics()
self._socket_mock = socket_mock
def train(model, criterion, optimizer, train_loader, val_loader, args):
"""
Train the model on the train data loader data and stop when the top1 precision did not increased for args.patience
epochs. The early stopping is done on the validation data loader.
All the data are sent to wandb or logged on a text file. More precisely for each epoch the validation and training
performance are sent to wandb and every args.print_freq the batch performance are logged on a file. At the end of
the training the best top1 validation accuracy is sent to wandb.
Parameters
----------
model : RotaitonNet model
criterion : Pytorch criterion (CrossEntropy for RotationNet)
optimizer : Pytorch optimizer (e.g. SGD)
train_loader : Data loader with training data (this must be created with a subset)
val_loader : Data loader with validation data for early stopping (this must be created with a subset)
args : Input args from the parser
Returns
-------
Nothing
"""
# Best prediction
best_prec1 = 0
# Using lr_scheduler for learning rate decay
# scheduler = StepLR(optimizer, step_size=args.learning_rate_decay, gamma=0.1)
epoch_no_improve = 0
for epoch in range(args.epochs):
# Give random permutation to the images
indices = train_loader.dataset.indices
inds = random_permute_indices(np.array(indices), args.nview, False)
train_loader.dataset.indices = np.asarray(inds)
del indices
del inds
statistics = Statistics()
# switch to train mode
model.train()
start_time = time.time()
for batch_idx, (input_val, target_val) in enumerate(train_loader):
# loss, (prec1, prec5), y_pred, y_true = execute_batch(model, criterion, input_val,
# target_val, args)
loss, (prec1, prec5), y_pred, y_true = execute_batch_aligned(model, criterion, input_val,
target_val, args)
statistics.update(loss.detach().cpu().numpy(), prec1, prec5, y_pred, y_true)
# compute gradient and do optimizer step
optimizer.zero_grad()
loss.backward()
optimizer.step()
if batch_idx % args.print_freq == 0:
logger.debug('Batch: [{0}/{1}]\t'
'Loss {loss:.4f} \t'
'Prec@1 {top1:.3f} \t'
'Prec@5 {top5:.3f}'.format(batch_idx, len(train_loader), loss=loss.data, top1=prec1,
top5=prec5))
del loss
torch.cuda.empty_cache()
elapsed_time = time.time() - start_time
logger.debug("Evaluating epoch {}".format(epoch))
# permute indices
# indices = val_loader.dataset.indices
# inds = random_permute_indices(np.array(indices), args.nview)
# val_loader.dataset.indices = np.asarray(inds)
# del indices
# del inds
# Evaluate on validation set
val_statistics = validate(val_loader, model, criterion, args)
# statistics.compute(args.num_classes)
# val_statistics.compute(args.num_classes)
log_data(statistics, "train", val_loader.dataset.dataset.classes, epoch)
log_data(val_statistics, "internal_val", val_loader.dataset.dataset.classes, epoch)
wandb.log({"Epoch elapsed time": elapsed_time}, step=epoch)
# Save best model and best prediction
if val_statistics.top1.avg > best_prec1:
best_prec1 = val_statistics.top1.avg
save_model(args.arch, model, optimizer, args.fname_best)
epoch_no_improve = 0
else:
# Early stopping
epoch_no_improve += 1
if epoch_no_improve == args.patience:
wandb.run.summary["best_internal_val_top1_accuracy"] = best_prec1
wandb.run.summary["best_internal_val_top1_accuracy_epoch"] = epoch - args.patience
logger.debug("Stopping at epoch {} for early stopping (best was at epoch {})".format(epoch,
epoch - args.patience))
return
def compute_val_stat(data_dev, words, global_entity_list, stats, args):
w = 0
temp_gen = []
ref = []
hyp = []
src = []
ref_s = ""
hyp_s = ""
src_s = ""
microF1_PRED, microF1_PRED_cal, microF1_PRED_nav, microF1_PRED_wet = 0, 0, 0, 0
microF1_TRUE, microF1_TRUE_cal, microF1_TRUE_nav, microF1_TRUE_wet = 0, 0, 0, 0
for i, row in enumerate(np.transpose(words)):
st = ''
for e in row:
if e == '<EOS>':
break
else:
st += e + ' '
temp_gen.append(st)
correct = data_dev['trg_plain'][i]
### compute F1 SCORE
st = st.lstrip().rstrip()
correct = correct.lstrip().rstrip()
if args.dataset == 'kvr':
f1_true, count = Tree2Seq.compute_prf(data_dev['entity'][i], st.split(), global_entity_list,
data_dev['kb_plain'][i])
microF1_TRUE += f1_true
microF1_PRED += count
f1_true, count = Tree2Seq.compute_prf(data_dev['entity_cal'][i], st.split(), global_entity_list,
data_dev['kb_plain'][i])
microF1_TRUE_cal += f1_true
microF1_PRED_cal += count
f1_true, count = Tree2Seq.compute_prf(data_dev['entity_nav'][i], st.split(), global_entity_list,
data_dev['kb_plain'][i])
microF1_TRUE_nav += f1_true
microF1_PRED_nav += count
f1_true, count = Tree2Seq.compute_prf(data_dev['entity_wet'][i], st.split(), global_entity_list,
data_dev['kb_plain'][i])
microF1_TRUE_wet += f1_true
microF1_PRED_wet += count
conv_src = [item[0] for item in data_dev['src_plain'][i] if '$' in item[1]]
conv_src_s = " ".join(conv_src)
src_s += conv_src_s + '\n'
src.append(src_s)
# w += wer(correct, st)
ref.append(str(correct))
hyp.append(str(st))
ref_s += str(correct) + "\n"
hyp_s += str(st) + "\n"
with open('./tmp/{}_ref_s.txt'.format(args.experiment), 'a+') as f:
f.write(ref_s)
with open('./tmp/{}_hyp_s.txt'.format(args.experiment), 'a+') as f:
f.write(hyp_s)
with open('./tmp/{}_src_s.txt'.format(args.experiment), 'a+') as f:
f.write(src_s)
# compute the bleu score
bleu_score = moses_multi_bleu(np.array(hyp), np.array(ref), lowercase=True)
bleu_stat = Statistics(n_correct=bleu_score, n_words=1)
entity_stat = Statistics(n_correct=microF1_TRUE, n_words=microF1_PRED)
entity_cal_stat = Statistics(n_correct=microF1_TRUE_cal, n_words=microF1_PRED_cal)
entity_nav_stat = Statistics(n_correct=microF1_TRUE_nav, n_words=microF1_PRED_nav)
entity_wet_stat = Statistics(n_correct=microF1_TRUE_wet, n_words=microF1_PRED_wet)
new_stats = [entity_stat, entity_cal_stat, entity_nav_stat, entity_wet_stat, bleu_stat]
for i in range(5):
stats[i].update(new_stats[i])
return stats
def train(model, criterion, optimizer, train_loader, val_loader, args):
best_prec1 = 0
epoch_no_improve = 0
for epoch in range(1000):
statistics = Statistics()
model.train()
start_time = time.time()
for i, (input, target) in enumerate(train_loader):
loss, (prec1, prec5), y_pred, y_true = execute_batch(
model, criterion, input, target, args.device)
statistics.update(loss.detach().cpu().numpy(), prec1, prec5,
y_pred, y_true)
# compute gradient and do optimizer step
optimizer.zero_grad() #
loss.backward()
optimizer.step()
# if args.net_version == 2:
# model.camera_position = model.camera_position.clamp(0, 1)
del loss
torch.cuda.empty_cache()
elapsed_time = time.time() - start_time
# Evaluate on validation set
val_statistics = validate(val_loader, model, criterion, args.device)
log_data(statistics, "train", val_loader.dataset.dataset.classes,
epoch)
log_data(val_statistics, "internal_val",
val_loader.dataset.dataset.classes, epoch)
wandb.log({"Epoch elapsed time": elapsed_time}, step=epoch)
# print(model.camera_position)
if epoch % 1 == 0:
vertices = []
if args.net_version == 1:
R = look_at_rotation(model.camera_position, device=args.device)
T = -torch.bmm(R.transpose(1, 2),
model.camera_position[:, :, None])[:, :, 0]
else:
t = Transform3d(device=model.device).scale(
model.camera_position[3] *
model.distance_range).rotate_axis_angle(
model.camera_position[0] * model.angle_range,
axis="X",
degrees=False).rotate_axis_angle(
model.camera_position[1] * model.angle_range,
axis="Y",
degrees=False).rotate_axis_angle(
model.camera_position[2] * model.angle_range,
axis="Z",
degrees=False)
vertices = t.transform_points(model.vertices)
R = look_at_rotation(vertices[:model.nviews],
device=model.device)
T = -torch.bmm(R.transpose(1, 2), vertices[:model.nviews, :,
None])[:, :, 0]
cameras = OpenGLPerspectiveCameras(R=R, T=T, device=args.device)
wandb.log(
{
"Cameras":
[wandb.Image(plot_camera_scene(cameras, args.device))]
},
step=epoch)
plt.close()
images = render_shape(model, R, T, args, vertices)
wandb.log(
{
"Views": [
wandb.Image(
image_grid(images,
rows=int(np.ceil(args.nviews / 2)),
cols=2))
]
},
step=epoch)
plt.close()
# Save best model and best prediction
if val_statistics.top1.avg > best_prec1:
best_prec1 = val_statistics.top1.avg
save_model("views_net", model, optimizer, args.fname_best)
epoch_no_improve = 0
else:
# Early stopping
epoch_no_improve += 1
if epoch_no_improve == 20:
wandb.run.summary[
"best_internal_val_top1_accuracy"] = best_prec1
wandb.run.summary[
"best_internal_val_top1_accuracy_epoch"] = epoch - 20
return