def distributions():
conditions.clear()
dice1 = 8
dice2 = 6
reroll_equal_to = []
reroll_lowest = 0
roll_min = 0
drop_lowest = 0
details = [
dice1, dice2, reroll_equal_to, reroll_lowest, roll_min, drop_lowest
]
d20 = d20Set(1)
diceset = Diceset([(dice1, dice2)])
action = Action(d20, 0, diceset, crit_numbers=[], fail_dmg_scale=0.5)
stats = Statistics(action)
stats.collect_statistics()
dummystats.clear()
dummystats.append(stats.report_statistics())
return render_template('distributions.html',
imagepath='static/img/placeholder.png',
stats=dummystats,
conditions=conditions,
collecting=False)
def home():
dice1 = 8
dice2 = 6
d20 = d20Set(1)
diceset = Diceset([(dice1, dice2)])
action = Action(d20, 0, diceset, crit_numbers=[], fail_dmg_scale=0.5)
stats = Statistics(action)
stats.collect_statistics()
return render_template(
'home.html'
) #, imagepath='static/img/placeholder.png', stats=stats.report_statistics())
def initialize_simulation(self):
"""Initialize the simulation. This initialization is required when the
simulator.simulate function is being called. The initialization deals with
the simulator clock, database initiation, various simulation events, etc.
"""
self.events = list()
self.initialize_db()
self.initialize_people()
self.statistics = Statistics(simulator=self)
self.clock = Time(delta_time=timedelta(0), init_date_time=self.end_time.init_date_time)
self.initialize_plan_day_events(self.end_time)
self.initialize_virus_spread_events(self.end_time, self.spread_period)
self.initialize_infections(self.initialized_infected_ids)
async def task_queue_processing():
"""Функция обработки вычислительных задач."""
while True:
if not task_queue.empty():
logging.debug('Starting processing task.')
on_processing[0] = task_queue.qsize()
future, task = task_queue.get()
future.set_result(await task)
logging.debug('Processing task was completed.')
Statistics.process_request()
else:
on_processing[0] = 0
await sleep(1)
def evaluate(self, dataloader, threshold, label_by_block=False):
"""Evaluate outputs."""
stats = Statistics()
validation_loss = 0
for inputs, outputs, labels in dataloader:
inputs = inputs.to(device=self.device)
outputs = outputs.to(device=self.device)
labels = labels.to(device='cpu')
results = self.predict(inputs)
loss = self.criterion(results, outputs)
results = results.to(device='cpu')
validation_loss += loss.to(device='cpu').item()
for i in range(results.size()[0]):
LogAnomalyDetection.evaluate_threshold(threshold, stats, results[i], labels[i], label_by_block)
validation_loss = validation_loss / len(dataloader)
data = {'stats': stats, 'validation_loss': validation_loss}
return "Validation loss: %.3f, %s" % (validation_loss, stats.as_string()), data
def evaluate_threshold(errors, labels, threshold):
stats = Statistics()
skip = 0
for e, l in zip(errors, labels):
# if len(e) < 5:
# skip+=1
# continue
if l == 1:
if np.max(e) > threshold:
stats.add_tp()
else:
stats.add_fn()
else:
if np.max(e) > threshold:
stats.add_fp()
else:
stats.add_tn()
# print('skippend',skip)
return stats
def test_detection(self, dataloader, label_by_block=False, dataset_name=None):
stats = Statistics()
detection_data = {
'stats': stats,
'label_by_block': label_by_block,
'block_split': [],
'labels': [],
'threshold': self.threshold,
'thresholds': [(t, Statistics()) for t in np.linspace(0, 1.5 * self.threshold, num=50)],
'e': []
}
detection_data['thresholds'].append((self.threshold, stats))
last_block_index = 0
logger.info(f" Computing and comparing predictions ({'labeled by block' if label_by_block else 'labeled by sample'})")
for inputs, outputs, labels in dataloader:
inputs = inputs.to(device=self.device)
outputs = outputs.to(device='cpu')
labels = labels.to(device='cpu')
results = self.predict(inputs).to(device='cpu')
losses = self.criterion(results, outputs, labels) # labels only used with cos loos + labeled anomal training data experiment
for i in range(losses.size()[0]):
loss = losses[i].numpy()
label = labels[i].numpy()
for t, s in detection_data['thresholds']:
self.evaluate_threshold(t, s, loss, label, label_by_block)
last_block_index += len(loss)
detection_data['block_split'].append(last_block_index)
detection_data['labels'].extend(label)
detection_data['e'].extend(loss)
detection_data['block_split'].pop()
self.env.add_test_result(self.epoch, dataset_name, detection_data)
return detection_data
def test(self, dataloader, threshold=0.5, label_by_block=False):
stats = Statistics()
detection_data = {
'stats': stats,
'label_by_block': label_by_block,
'block_split': [],
'labels': [],
'threshold': threshold,
'thresholds': [(t, Statistics()) for t in np.linspace(0, 1.5 * threshold, num=50)],
'e': []
}
detection_data['thresholds'].append((threshold, stats))
last_block_index = 0
logger.info(f" Computing classification ({'labeled by block' if label_by_block else 'labeled by sample'})")
for inputs, outputs, labels in dataloader:
inputs = inputs.to(device=self.device)
labels = labels.to(device='cpu')
results = self.predict(inputs).to(device='cpu')
for i in range(results.size()[0]):
loss = results[i].numpy()
label = labels[i].numpy()
for t, s in detection_data['thresholds']:
LogAnomalyDetection.evaluate_threshold(t, s, loss, label, label_by_block)
last_block_index += len(loss)
detection_data['block_split'].append(last_block_index)
detection_data['labels'].extend(label)
detection_data['e'].extend(loss)
detection_data['block_split'].pop()
return detection_data
def run(args=None):
device = 'cuda' if torch.cuda.is_available() and (not args.no_cuda) else 'cpu'
num_train, train_loader, test_loader, input_size, input_channel, n_class = get_loaders(args)
lossFn = nn.CrossEntropyLoss(reduction='none')
def evalFn(x): return torch.max(x, dim=1)[1]
## initialize SpecNet
dTNet = MyDeepTrunkNet.get_deepTrunk_net(args, device, lossFn, evalFn, input_size, input_channel, n_class)
## setup logging and checkpointing
timestamp = int(time.time())
model_signature = '%s/%s/%d/%s_%.5f/%d' % (args.dataset, args.exp_name, args.exp_id, args.net, args.train_eps, timestamp)
model_dir = args.root_dir + 'models_new/%s' % (model_signature)
args.model_dir = model_dir
print("Saving model to: %s" % model_dir)
count_vars(args, dTNet)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
tb_writer = SummaryWriter(model_dir)
stats = Statistics(len(train_loader), tb_writer, model_dir)
args_file = os.path.join(model_dir, 'args.json')
with open(args_file, 'w') as fou:
json.dump(vars(args), fou, indent=4)
write_config(args, os.path.join(model_dir, 'run_config.txt'))
## main part depending on training mode
if 'train' in args.train_mode:
epoch = train_deepTrunk(dTNet, args, device, stats, train_loader, test_loader)
if args.cert:
with torch.no_grad():
cert_deepTrunk_net(dTNet, args, device, test_loader if args.test_set == "test" else train_loader,
stats, log_ind=True, break_on_failure=False, epoch=epoch)
elif args.train_mode == 'test':
with torch.no_grad():
test_deepTrunk_net(dTNet, args, device, test_loader if args.test_set == "test" else train_loader, stats,
log_ind=True)
elif args.train_mode == "cert":
with torch.no_grad():
cert_deepTrunk_net(dTNet, args, device, test_loader if args.test_set == "test" else train_loader, stats,
log_ind=True, break_on_failure=False)
else:
assert False, 'Unknown mode: {}!'.format(args.train_mode)
exit(0)
async def statistics_handler(message: types.Message, state: FSMContext):
user_data = await state.get_data()
await BotStates.DEFAULT.set()
stat = Statistics.get()
info = 'Начиная с ' + stat['date'] + ' я обработал ' + str(stat['counter'])
mod = stat['counter'] % 10
if mod == 1:
info += ' изображение.'
elif mod in [2, 3, 4]:
info += ' изображения.'
else:
info += ' изображений.'
info += '\nНа текущий момент длина очереди на обработку равна ' + str(
on_processing[0]) + '.'
await message.answer(info, reply_markup=init_main_keyboard(user_data))
def save_simulation_data(self):
# manipulate time to store in db
date_time = self.simulator.clock.unix_time
# save simulation and observer id
observer_id = id(self)
observation_id = self.observation_id
stats = Statistics.get_people_statistics(self.simulator.people)
# extract the statistics
active_cases = stats[Health_Condition.IS_INFECTED]
confirmed_cases = stats[Health_Condition.HAS_BEEN_INFECTED]
death_cases = stats[Health_Condition.DEAD]
# insert people's data into database
self.simulator.database.insert(
table_name='simulator',
data=[confirmed_cases, death_cases, active_cases])
self.simulator.database.commit()
def is_satisfied(self, simulator, end_time: Time):
"""Check whether the condition is satisfied or not.
Args:
simulator (Simulator): The simulator object.
end_time (Time): The final time of the simulation.
Returns:
List: The clock of the simulation when the condition is
satisfied, otherwise none.
"""
self.family_stat = Statistics.get_family_statistics(
simulator.people, simulator.families)
current_ratio = self.family_stat[self.stat_type] / len(
simulator.families)
if Comparison.compare(current_ratio, self.target_ratio,
self.comparison_type) and self.max_satisfaction:
self.max_satisfaction -= 1
return [simulator.clock]
return []
def calculate():
root = os.path.dirname(__file__)
dir = os.path.join(root, 'static/img/temp/')
files = os.listdir(dir)
for file in files:
os.remove(os.path.join(dir, file))
alpha = 0.9**len(conditions)
statistics = []
i = 0
for details in conditions:
d20 = d20Set(1)
diceset = Diceset([(details[0], details[1])], details[2], details[3],
details[4], details[5])
action = Action(d20, 0, diceset, crit_numbers=[], fail_dmg_scale=0.0)
stats = Statistics(action)
stats.collect_statistics()
stats.plot_histogram(alpha, str(i))
i += 1
statistics.append(stats.report_statistics())
plotname = 'static/img/temp/'
for detail in details:
plotname = plotname + str(detail)
plotname = plotname + '.png'
copyconditions = conditions.copy()
conditions.clear()
plt.savefig(os.path.join(root, plotname))
plt.clf()
return render_template('distributions.html',
imagepath=plotname,
stats=statistics,
conditions=copyconditions,
collecting=False)
def main(args, local_rank):
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO)
vocabs = dict()
vocabs['src'] = Vocab(args.src_vocab, 0, [BOS, EOS])
vocabs['tgt'] = Vocab(args.tgt_vocab, 0, [BOS, EOS])
if args.world_size == 1 or (dist.get_rank() == 0):
logger.info(args)
for name in vocabs:
logger.info("vocab %s, size %d, coverage %.3f", name,
vocabs[name].size, vocabs[name].coverage)
set_seed(19940117)
#device = torch.device('cpu')
torch.cuda.set_device(local_rank)
device = torch.device('cuda', local_rank)
if args.resume_ckpt:
model = MatchingModel.from_pretrained(vocabs, args.resume_ckpt)
else:
model = MatchingModel.from_params(vocabs, args.layers, args.embed_dim,
args.ff_embed_dim, args.num_heads,
args.dropout, args.output_dim,
args.bow)
if args.world_size > 1:
set_seed(19940117 + dist.get_rank())
model = model.to(device)
if args.resume_ckpt:
dev_data = DataLoader(vocabs,
args.dev_data,
args.dev_batch_size,
addition=args.additional_negs)
acc = validate(model, dev_data, device)
logger.info("initialize from %s, initial acc %.2f", args.resume_ckpt,
acc)
optimizer = Adam(model.parameters(),
lr=args.lr,
betas=(0.9, 0.98),
eps=1e-9)
lr_schedule = get_linear_schedule_with_warmup(optimizer, args.warmup_steps,
args.total_train_steps)
train_data = DataLoader(vocabs,
args.train_data,
args.per_gpu_train_batch_size,
worddrop=args.worddrop,
addition=args.additional_negs)
global_step, step, epoch = 0, 0, 0
tr_stat = Statistics()
logger.info("start training")
model.train()
while global_step <= args.total_train_steps:
for batch in train_data:
batch = move_to_device(batch, device)
loss, acc, bsz = model(batch['src_tokens'], batch['tgt_tokens'],
args.label_smoothing)
tr_stat.update({
'loss': loss.item() * bsz,
'nsamples': bsz,
'acc': acc * bsz
})
tr_stat.step()
loss.backward()
step += 1
if not (step % args.gradient_accumulation_steps
== -1 % args.gradient_accumulation_steps):
continue
if args.world_size > 1:
average_gradients(model)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
lr_schedule.step()
optimizer.zero_grad()
global_step += 1
if args.world_size == 1 or (dist.get_rank() == 0):
if global_step % args.print_every == -1 % args.print_every:
logger.info("epoch %d, step %d, loss %.3f, acc %.3f",
epoch, global_step,
tr_stat['loss'] / tr_stat['nsamples'],
tr_stat['acc'] / tr_stat['nsamples'])
tr_stat = Statistics()
if global_step > args.warmup_steps and global_step % args.eval_every == -1 % args.eval_every:
dev_data = DataLoader(vocabs,
args.dev_data,
args.dev_batch_size,
addition=args.additional_negs)
acc = validate(model, dev_data, device)
logger.info("epoch %d, step %d, dev, dev acc %.2f", epoch,
global_step, acc)
save_path = '%s/epoch%d_batch%d_acc%.2f' % (
args.ckpt, epoch, global_step, acc)
model.save(args, save_path)
model.train()
if global_step > args.total_train_steps:
break
epoch += 1
logger.info('rank %d, finish training after %d steps', local_rank,
global_step)
def run_experiment(params, vd_environment, trial_out_dir, num_dimensions, n_generations=100,
save_results=False, silent=False, args=None):
"""
The function to run the experiment against hyper-parameters
defined in the provided configuration file.
The winner genome will be rendered as a graph as well as the
important statistics of neuroevolution process execution.
Arguments:
params: The NEAT parameters
vd_environment: The environment to test visual discrimination
trial_out_dir: The directory to store outputs for this trial
num_dimensions: The dimensionsionality of visual field
n_generations: The number of generations to execute.
save_results: The flag to control if intermdiate results will be saved.
silent: If True than no intermediary outputs will be
presented until solution is found.
args: The command line arguments holder.
Returns:
True if experiment finished with successful solver found.
"""
# random seed
seed = int(time.time())
# Create substrate
substrate = create_substrate(num_dimensions)
# Create CPPN genome and population
g = NEAT.Genome(0,
substrate.GetMinCPPNInputs(),
0,
substrate.GetMinCPPNOutputs(),
False,
NEAT.ActivationFunction.UNSIGNED_SIGMOID,
NEAT.ActivationFunction.UNSIGNED_SIGMOID,
0,
params, 0)
pop = NEAT.Population(g, params, True, 1.0, seed)
pop.RNG.Seed(seed)
# Run for up to N generations.
start_time = time.time()
best_genome_ser = None
best_ever_goal_fitness = 0
best_id = -1
solution_found = False
stats = Statistics()
for generation in range(n_generations):
print("\n****** Generation: %d ******\n" % generation)
gen_time = time.time()
# get list of current genomes
genomes = NEAT.GetGenomeList(pop)
# evaluate genomes
genome, fitness, distances = eval_genomes(genomes, vd_environment=vd_environment,
substrate=substrate, generation=generation)
stats.post_evaluate(max_fitness=fitness, distances=distances)
solution_found = fitness >= FITNESS_THRESHOLD
# store the best genome
if solution_found or best_ever_goal_fitness < fitness:
best_genome_ser = pickle.dumps(genome)
best_ever_goal_fitness = fitness
best_id = genome.GetID()
if solution_found:
print('Solution found at generation: %d, best fitness: %f, species count: %d' % (generation, fitness, len(pop.Species)))
break
# advance to the next generation
pop.Epoch()
# print statistics
gen_elapsed_time = time.time() - gen_time
print("Best fitness: %f, genome ID: %d" % (fitness, best_id))
print("Species count: %d" % len(pop.Species))
print("Generation elapsed time: %.3f sec" % (gen_elapsed_time))
print("Best fitness ever: %f, genome ID: %d" % (best_ever_goal_fitness, best_id))
elapsed_time = time.time() - start_time
best_genome = pickle.loads(best_genome_ser)
# write best genome to the file
best_genome_file = os.path.join(trial_out_dir, "best_genome.pickle")
with open(best_genome_file, 'wb') as genome_file:
pickle.dump(best_genome, genome_file)
# Print experiment statistics
print("\nBest ever fitness: %f, genome ID: %d" % (best_ever_goal_fitness, best_id))
print("\nTrial elapsed time: %.3f sec" % (elapsed_time))
print("Random seed:", seed)
# Visualize the experiment results
show_results = not silent
if save_results or show_results:
# Draw CPPN network graph
net = NEAT.NeuralNetwork()
best_genome.BuildPhenotype(net)
visualize.draw_net(net, view=show_results, node_names=None, directory=trial_out_dir, fmt='svg')
print("\nCPPN nodes: %d, connections: %d" % (len(net.neurons), len(net.connections)))
# Visualize activations from the best genome
net = NEAT.NeuralNetwork()
best_genome.BuildHyperNEATPhenotype(net, substrate)
# select random visual field
index = random.randint(0, len(vd_environment.data_set) - 1)
print("\nRunning test evaluation against random visual field:", index)
print("Substrate nodes: %d, connections: %d" % (len(net.neurons), len(net.connections)))
vf = vd_environment.data_set[index]
# draw activations
outputs, x, y = vd_environment.evaluate_net_vf(net, vf)
visualize.draw_activations(outputs, found_object=(x, y), vf=vf,
dimns=num_dimensions, view=show_results,
filename=os.path.join(trial_out_dir, "best_activations.svg"))
# Visualize statistics
visualize.plot_stats(stats, ylog=False, view=show_results, filename=os.path.join(trial_out_dir, 'avg_fitness.svg'))
return solution_found
socket = context.socket(zmq.REP)
port = socket.bind_to_random_port(ZMQ_ADDRESS)
# Start processes.
processes = [
multiprocessing.Process(target=environment_process,
args=(
i,
port,
),
daemon=True) for i in range(NUM_ENVIRONMENTS)
]
for p in processes:
p.start()
steps_statistics = Statistics()
total_reward_statistics = Statistics()
collision_statistics = Statistics()
num_nodes_statistics = Statistics()
depth_statistics = Statistics()
stats_statistics = [Statistics() for _ in range(5)]
planner_value_statistics = defaultdict(lambda: Statistics())
value_statistics = defaultdict(lambda: Statistics())
while len(total_reward_statistics) < 100 or total_reward_statistics.stderr(
) > TARGET_SE:
# Read request and process.
(steps, total_reward, collision, num_nodes, depth,
stats) = socket.recv_pyobj()
socket.send_pyobj(0)
stats])
socket.recv_pyobj()
if __name__ == '__main__':
# Prepare zmq server.
context = zmq.Context()
socket = context.socket(zmq.REP)
port = socket.bind_to_random_port(ZMQ_ADDRESS)
# Start processes.
processes = [multiprocessing.Process(target=environment_process, args=(i, port,), daemon=True) for i in range(NUM_ENVIRONMENTS)]
for p in processes:
p.start()
steps_statistics = Statistics()
total_reward_statistics = Statistics()
collision_statistics = Statistics()
num_nodes_statistics = Statistics()
depth_statistics = Statistics()
stats_statistics = [Statistics() for _ in range(5)]
while True:
# Read request and process.
(steps, total_reward, collision, num_nodes, depth, stats) = socket.recv_pyobj()
socket.send_pyobj(0)
if args.task not in ['DriveHard']:
if not collision:
steps_statistics.append(steps)
else:
hidden_ans_dim=args.hidden_ans_dim,
hidden_hist_dim=args.hidden_hist_dim,
hidden_cap_dim=args.hidden_cap_dim,
hidden_img_dim=img_features_dim)
# Multiple GPUs batch parallel
if torch.cuda.is_available():
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
model.cuda()
else:
sys.exit("Only GPU version is currently available.")
#for n, p in model.named_parameters():
# print(n, p.numel())
print("Total params:", sum(p.numel() for p in model.parameters()))
stats = Statistics(args)
ndcg = NDCG()
if args.submission:
if args.model_pathname:
args.mrr_pathname = os.path.join(args.model_pathname,
'best_model_mrr.pth.tar')
args.ndcg_pathname = os.path.join(args.model_pathname,
'best_model_ndcg.pth.tar')
else:
args.mrr_pathname = os.path.join(dir_path(args),
'best_model_mrr.pth.tar')
args.ndcg_pathname = os.path.join(dir_path(args),
'best_model_ndcg.pth.tar')
print('Creating submissions')
print("loading best MRR: {}".format(args.mrr_pathname))
def main(args, local_rank):
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO)
vocabs = dict()
vocabs['src'] = Vocab(args.src_vocab, 0, [BOS, EOS])
vocabs['tgt'] = Vocab(args.tgt_vocab, 0, [BOS, EOS])
if args.world_size == 1 or (dist.get_rank() == 0):
logger.info(args)
for name in vocabs:
logger.info("vocab %s, size %d, coverage %.3f", name,
vocabs[name].size, vocabs[name].coverage)
set_seed(19940117)
#device = torch.device('cpu')
torch.cuda.set_device(local_rank)
device = torch.device('cuda', local_rank)
if args.arch == 'vanilla':
model = Generator(vocabs, args.embed_dim, args.ff_embed_dim,
args.num_heads, args.dropout, args.enc_layers,
args.dec_layers, args.label_smoothing)
elif args.arch == 'mem':
model = MemGenerator(vocabs, args.embed_dim, args.ff_embed_dim,
args.num_heads, args.dropout, args.mem_dropout,
args.enc_layers, args.dec_layers,
args.mem_enc_layers, args.label_smoothing,
args.use_mem_score)
elif args.arch == 'rg':
logger.info("start building model")
logger.info("building retriever")
retriever = Retriever.from_pretrained(
args.num_retriever_heads,
vocabs,
args.retriever,
args.nprobe,
args.topk,
local_rank,
use_response_encoder=(args.rebuild_every > 0))
logger.info("building retriever + generator")
model = RetrieverGenerator(vocabs, retriever, args.share_encoder,
args.embed_dim, args.ff_embed_dim,
args.num_heads, args.dropout,
args.mem_dropout, args.enc_layers,
args.dec_layers, args.mem_enc_layers,
args.label_smoothing)
if args.resume_ckpt:
model.load_state_dict(torch.load(args.resume_ckpt)['model'])
else:
global_step = 0
if args.world_size > 1:
set_seed(19940117 + dist.get_rank())
model = model.to(device)
retriever_params = [
v for k, v in model.named_parameters() if k.startswith('retriever.')
]
other_params = [
v for k, v in model.named_parameters()
if not k.startswith('retriever.')
]
optimizer = Adam([{
'params': retriever_params,
'lr': args.embed_dim**-0.5 * 0.1
}, {
'params': other_params,
'lr': args.embed_dim**-0.5
}],
betas=(0.9, 0.98),
eps=1e-9)
lr_schedule = get_inverse_sqrt_schedule_with_warmup(
optimizer, args.warmup_steps, args.total_train_steps)
train_data = DataLoader(vocabs,
args.train_data,
args.per_gpu_train_batch_size,
for_train=True,
rank=local_rank,
num_replica=args.world_size)
model.eval()
#dev_data = DataLoader(vocabs, cur_dev_data, args.dev_batch_size, for_train=False)
#bleu = validate(device, model, dev_data, beam_size=5, alpha=0.6, max_time_step=10)
step, epoch = 0, 0
tr_stat = Statistics()
logger.info("start training")
model.train()
best_dev_bleu = 0.
while global_step <= args.total_train_steps:
for batch in train_data:
#step_start = time.time()
batch = move_to_device(batch, device)
if args.arch == 'rg':
loss, acc = model(
batch,
update_mem_bias=(global_step >
args.update_retriever_after))
else:
loss, acc = model(batch)
tr_stat.update({
'loss': loss.item() * batch['tgt_num_tokens'],
'tokens': batch['tgt_num_tokens'],
'acc': acc
})
tr_stat.step()
loss.backward()
#step_cost = time.time() - step_start
#print ('step_cost', step_cost)
step += 1
if not (step % args.gradient_accumulation_steps
== -1 % args.gradient_accumulation_steps):
continue
if args.world_size > 1:
average_gradients(model)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
lr_schedule.step()
optimizer.zero_grad()
global_step += 1
if args.world_size == 1 or (dist.get_rank() == 0):
if global_step % args.print_every == -1 % args.print_every:
logger.info("epoch %d, step %d, loss %.3f, acc %.3f",
epoch, global_step,
tr_stat['loss'] / tr_stat['tokens'],
tr_stat['acc'] / tr_stat['tokens'])
tr_stat = Statistics()
if global_step % args.eval_every == -1 % args.eval_every:
model.eval()
max_time_step = 256 if global_step > 2 * args.warmup_steps else 5
bleus = []
for cur_dev_data in args.dev_data:
dev_data = DataLoader(vocabs,
cur_dev_data,
args.dev_batch_size,
for_train=False)
bleu = validate(device,
model,
dev_data,
beam_size=5,
alpha=0.6,
max_time_step=max_time_step)
bleus.append(bleu)
bleu = sum(bleus) / len(bleus)
logger.info("epoch %d, step %d, dev bleu %.2f", epoch,
global_step, bleu)
if bleu > best_dev_bleu:
testbleus = []
for cur_test_data in args.test_data:
test_data = DataLoader(vocabs,
cur_test_data,
args.dev_batch_size,
for_train=False)
testbleu = validate(device,
model,
test_data,
beam_size=5,
alpha=0.6,
max_time_step=max_time_step)
testbleus.append(testbleu)
testbleu = sum(testbleus) / len(testbleus)
logger.info("epoch %d, step %d, test bleu %.2f", epoch,
global_step, testbleu)
torch.save({
'args': args,
'model': model.state_dict()
}, '%s/best.pt' % (args.ckpt, ))
if not args.only_save_best:
torch.save(
{
'args': args,
'model': model.state_dict()
},
'%s/epoch%d_batch%d_devbleu%.2f_testbleu%.2f' %
(args.ckpt, epoch, global_step, bleu,
testbleu))
best_dev_bleu = bleu
model.train()
if args.rebuild_every > 0 and (global_step % args.rebuild_every
== -1 % args.rebuild_every):
model.retriever.drop_index()
torch.cuda.empty_cache()
next_index_dir = '%s/batch%d' % (args.ckpt, global_step)
if args.world_size == 1 or (dist.get_rank() == 0):
model.retriever.rebuild_index(next_index_dir)
dist.barrier()
else:
dist.barrier()
model.retriever.update_index(next_index_dir, args.nprobe)
if global_step > args.total_train_steps:
break
epoch += 1
logger.info('rank %d, finish training after %d steps', local_rank,
global_step)
class Simulator:
"""This class handles the operation and preparations of simulation.
This object provides a set of methods to initialize and run a simulation.
Some of its objectives are to generate/save/load the simulation model, initialize
the simulation properties like database, and to run the main simulation based on
the given population structure and disease properties.
Attributes
----------
population_generator (Population_Generator): The population generator object containing
the data required to establish the population model.
clock (Time): The main simulation clock.
end_time (Time): The final time of the simulation.
spread_period (Time): The period that disease transmission is being investigated.
database (Database): The main database of the simulation to store the simulation data.
people (list[Person]): The people created out of the population generator.
graph (Dict[int, List]): The graph of all the people in the simulation.
families (List[Family]): The families built out of the population generator.
communities (List[Community]): The communities involved in the
simulation population structure.
events (List[Event]): The list of all the events in the simulation, e.g., Transition,
Virus Spread.
disease_properties (Disease_Properties): The properties of the disease are stored inside
this property of the simulation.
commands (List[Command]): A list of commands to be executed during the simulation time.
This is also known as simulation policies.
observers (List[Observer]): A list of all observers in the simulation.
initialized_infected_ids (List[int]): A list of people that initially have the disease.
"""
def __init__(self, population_generator: Population_Generator,
disease_properties: Disease_Properties):
"""Initialize a simulator object.
Args:
population_generator (Population_Generator): The population generator object
required to create families, communities, and people.
disease_properties (Disease_Properties): The disease properties object is
necessary to obtain a correct estimation of disease characteristics in the
simulation.
"""
self.population_generator = population_generator
self.disease_properties = disease_properties
self.clock, self.end_time = None, None
self.statistics = None
self.people, self.graph, self.families, self.communities = list(), dict(), list(), dict()
self.events = list()
def initialize_people(self):
"""Initialize every person in the simulator.people object.
"""
for person in self.people:
person.initialize()
def initialize_db(self):
"""Initialize the database associated with the simulation environment,
along with creating the tables for people, families, and communities.
"""
if self.database_name is None:
self.database = Database('simulator')
else:
self.database = \
Database('simulator_' + str(self.database_name[0]) + '_' + str(self.database_name[1]))
# a table to simulation statistics
self.database.create_table(name='simulation',
columns=[
('simulator_id', 'integer'),
('people_confirmed_cases', 'integer'),
('people_death_cases', 'integer'),
('people_active_cases', 'integer')
])
# a table to keep people information
self.database.create_table(name='people',
columns=[
('id_number', 'integer'),
('age', 'integer'),
('health_condition', 'real'),
('gender', 'integer'),
('infection_status', 'integer'),
('is_alive', 'integer'),
('has_profession', 'integer'),
('times_of_infection', 'integer'),
('is_quarantined', 'integer'),
('location', 'str'),
('observation_id', 'integer'),
('observer_id', 'integer'),
('date_time', 'integer')
])
# TODO: remove and create a consistent database
self.database.delete_data('people')
# TODO: tables reserved for later in case of use, fill accordingly
self.database.create_table(name='families',
columns=[('family_id', 'integer')])
self.database.create_table(name='communities',
columns=[('community_id', 'integer')])
def initialize_simulation(self):
"""Initialize the simulation. This initialization is required when the
simulator.simulate function is being called. The initialization deals with
the simulator clock, database initiation, various simulation events, etc.
"""
self.events = list()
self.initialize_db()
self.initialize_people()
self.statistics = Statistics(simulator=self)
self.clock = Time(delta_time=timedelta(0), init_date_time=self.end_time.init_date_time)
self.initialize_plan_day_events(self.end_time)
self.initialize_virus_spread_events(self.end_time, self.spread_period)
self.initialize_infections(self.initialized_infected_ids)
def initialize_plan_day_events(self, end_time: Time):
"""Initialize plan day event is the place that the plan_day_event task is
pushed into the simulation heap structure.
Args:
end_time (Time): The simulation end time.
"""
total_simulation_days = end_time.get_total_days()
for i in range(total_simulation_days + 1):
plan_time = Time(timedelta(days=i))
plan_day_event = Plan_Day_Event(plan_time.get_minutes(), Simulation_Event.PLAN_DAY)
heapq.heappush(self.events, plan_day_event)
def initialize_virus_spread_events(self, end_time: Time, spread_period: Time):
"""Create and push the virus spread events into the simulation heap.
Args:
end_time (Time): The simulator final time.
spread_period (Time): The spread period of the simulation on which the virus
spread events are processed.
"""
for i in range(0, end_time.get_minutes() + 1, spread_period.get_minutes()):
virus_spread_event = Virus_Spread_Event(i, Simulation_Event.VIRUS_SPREAD)
heapq.heappush(self.events, virus_spread_event)
def initialize_infections(self, initialized_infected_ids: List[int]):
"""Initialize the infections by building an infection class and
adding the corresponding event to the heap.
Args:
initialized_infected_ids (List[int]): The list of id numbers related
to the people first having the disease.
"""
Infection.start_infections(new_infected_ids=initialized_infected_ids,
simulator=self)
def generate_model(self, is_parallel: bool = False, show_progress: bool = True):
"""Generates a the simulation model, including the people, graph, families,
and communities.
Args:
is_parallel (bool): If true, the function uses multiprocessing to boost
the computations. Otherwise, the method runs in normal single processing
mode.
show_progress (bool, optional): Whether to show the progress bar or not.
Defaults to True.
"""
self.people, self.graph, self.families, self.communities = \
self.population_generator.generate_population(is_parallel, show_progress)
logger.info('Simulation model generated')
def save_model(self, model_name: str):
"""Save the simulation model generated in generate model method for later use.
This function saves the model in a pickle file located in the data/pickle
folder.
Warning: Using save model will change the reference of the objects, and leads to
creation of different communities, people, families, etc. Only use it once to
maintain consistency of the references.
Args:
model_name (str): The name of the pickle file to be saved.
"""
if basename(os.path.abspath(os.path.join(os.getcwd(), os.pardir))) == 'Pyfectious':
path = os.path.join(os.getcwd(), os.pardir, 'data', 'pickle', model_name)
elif basename(os.getcwd()) == 'Pyfectious':
path = os.path.join(os.getcwd(), 'data', 'pickle', model_name)
else:
logger.warning('Failed to save model, change directory to src, example, or \
project main directory and try again!')
return
os.makedirs(path, exist_ok=True)
graph_path = os.path.join(path, 'graph.pickle')
people_path = os.path.join(path, 'people.pickle')
families_path = os.path.join(path, 'families.pickle')
communities_path = os.path.join(path, 'communities.pickle')
with open(people_path, 'wb') as f:
pickle.dump(self.people, f)
with open(graph_path, 'wb') as f:
pickle.dump(self.graph, f)
with open(families_path, 'wb') as f:
pickle.dump(self.families, f)
with open(communities_path, 'wb') as f:
pickle.dump(self.communities, f)
logger.info(f'Simulator model {model_name} saved')
def load_model(self, model_name: str):
"""Load a saved simulation model from the pickle folder.
This function automatically sets the simulation main entities to
the data loaded from the pickle file.
Args:
model_name (str): The name of the model to be loaded.
"""
if basename(os.getcwd()) == 'Pyfectious':
sys.path.insert(1, 'src')
path = os.path.join(os.getcwd(), 'data', 'pickle', model_name)
elif basename(os.path.abspath(os.path.join(os.getcwd(), os.pardir))) == 'Pyfectious':
sys.path.insert(1, os.path.join(os.pardir, 'src'))
path = os.path.join(os.getcwd(), os.pardir, 'data', 'pickle', model_name)
else:
raise FileNotFoundError('Failed to save model, change directory to src, example, or \
project main directory and try again!')
graph_path = os.path.join(path, 'graph.pickle')
people_path = os.path.join(path, 'people.pickle')
families_path = os.path.join(path, 'families.pickle')
communities_path = os.path.join(path, 'communities.pickle')
with open(people_path, 'rb') as f:
self.people = pickle.load(f)
with open(graph_path, 'rb') as f:
self.graph = pickle.load(f)
with open(families_path, 'rb') as f:
self.families = pickle.load(f)
with open(communities_path, 'rb') as f:
self.communities = pickle.load(f)
logger.info(f'Simulator model {model_name} loaded')
def execute_observers(self):
"""This function is used to parallelize the observation process.
The user-defined observation classes run here and store what they want
into the database, using their own observer id.
"""
for observer in self.observers:
if not observer.is_deleted:
observer.observe(self, self.end_time)
if observer.observation_is_done():
observer.is_deleted = True
def execute_commands(self):
"""Executes the commands defined by user as the simulator input.
A command gets deleted from the list whenever successfully committed.
"""
for command in self.commands:
if not command.is_deleted:
command.take_action(self, self.end_time)
if command.action_is_done():
command.is_deleted = True
def simulate(self, end_time: Time, spread_period: Time, initialized_infected_ids: List[int],
commands: List, observers: List, report_statistics: int = 1, database_name: Tuple = None,
show_progress: bool = True):
"""The main simulator function that starts the simulation given a certain criteria.
The simulation main loop is located inside this function, in addition to collecting
the observers and commands (policy).
Args:
end_time (Time): The simulation end time. Determines the total period
of the simulation.
spread_period (Time): The spread period determines the period of running the
virus spread function.
initialized_infected_ids (List[int]): A list of infected people at the beginning
of the simulation.
commands (List[Command]): A list of actions given to the simulator to contract the
spread of the virus. The acceptable commands are located in commands.py module.
observers (List[Observer]): The observer object to record the data from simulation
into the database.
report_statistics (int, optional): This parameter determines the degree of statistics
shown after the simulation concluded. Starting from 0, meaning no statistics at all,
and goes all the way to 2, where all the available statistics are shown in the console.
Defaults to 1.
database_name (Tuple, optional): The name of the database in case a change is required.
Default to None.
show_progress (bool, optional): Whether to show the progress bar or not.
Defaults to True.
"""
# Initialize simulation
logger.info('Initializing the simulation')
self.initialized_infected_ids = initialized_infected_ids
self.end_time, self.spread_period = end_time, spread_period
self.commands, self.observers = commands, observers
self.database_name = database_name
self.initialize_simulation()
# Print a log and the progress indicator
logger.info('Starting the simulation')
with tqdm(total=self.end_time.get_minutes(), file=sys.stdout, disable=not show_progress) \
as progress_bar:
# Simulation main loop
while Time.check_less(self.clock, self.end_time):
# Events
event = heapq.heappop(self.events)
event.activate(self)
# Observations
self.execute_observers()
# Commands
self.execute_commands()
# Update clock
self.clock.set_minutes(event.minute)
# Show progress
progress_bar.update(self.clock.get_minutes() - progress_bar.n)
logger.info('Simulation completed')
# Show statistics
self.statistics.report(self, report_statistics)
# Database termination
self.database.commit()
def clear(self):
"""Clear the main objects of the simulation.
"""
self.people.clear()
self.families.clear()
self.communities.clear()
self.observers.clear()
self.commands.clear()
self.database.close()
self.disease_properties = None
self.population_generator = None
def to_json(self):
"""Convert object fields to a JSON dictionary.
Returns:
dict: The JSON dictionary.
"""
return dict(name=self.__class__.__name__,
end_time=self.end_time,
spread_period=self.spread_period,
initialized_infected_ids=self.initialized_infected_ids,
commands=self.commands,
observers=self.observers)
# Prepare zmq server.
zmq_context = zmq.Context()
socket = zmq_context.socket(zmq.REP)
port = socket.bind_to_random_port(ZMQ_ADDRESS)
# Start processes.
processes = [
multiprocessing.Process(target=environment_process,
args=(port, ),
daemon=True) for _ in range(NUM_ENVIRONMENTS)
]
for p in processes:
p.start()
steps_statistics = Statistics()
total_reward_statistics = Statistics()
collision_statistics = Statistics()
macro_length_statistics = Statistics()
num_nodes_statistics = Statistics()
depth_statistics = Statistics()
stats_statistics = [Statistics() for _ in range(5)]
planner_value_statistics = defaultdict(lambda: Statistics())
value_statistics = defaultdict(lambda: Statistics())
while True:
# Read request and process.
request = socket.recv_pyobj()
instruction = request[0]
instruction_data = request[1:]
def run(autoencoder, classifier, optimizer, loader, split):
predictions, targets = list(), list()
tot_ce_loss, tot_stat_par, tot_eq_odds = Statistics.get_stats(3)
progress_bar = tqdm(loader)
if args.adversarial:
attack = PGD(classifier,
args.delta,
F.binary_cross_entropy_with_logits,
clip_min=float('-inf'),
clip_max=float('inf'))
for data_batch, targets_batch, protected_batch in progress_bar:
batch_size = data_batch.shape[0]
data_batch = data_batch.to(device)
targets_batch = targets_batch.to(device)
protected_batch = protected_batch.to(device)
x_batches, y_batches = list(), list()
assert batch_size % oracle.constraint.n_tvars == 0
k = batch_size // oracle.constraint.n_tvars
for i in range(oracle.constraint.n_tvars):
x_batches.append(data_batch[i:i + k])
y_batches.append(targets_batch[i:i + k])
if split == 'train':
classifier.train()
latent_data = autoencoder.encode(data_batch)
if args.adversarial:
latent_data = attack.attack(args.delta / 10,
latent_data,
20,
targets_batch,
targeted=False,
num_restarts=1,
random_start=True)
logits = classifier(latent_data)
ce_loss = binary_cross_entropy(logits, targets_batch)
predictions_batch = classifier.predict(latent_data)
stat_par = statistical_parity(predictions_batch, protected_batch)
eq_odds = equalized_odds(targets_batch, predictions_batch,
protected_batch)
predictions.append(predictions_batch.detach().cpu())
targets.append(targets_batch.detach().cpu())
if split == 'train':
optimizer.zero_grad()
ce_loss.mean().backward()
optimizer.step()
tot_ce_loss.add(ce_loss.mean().item())
tot_stat_par.add(stat_par.mean().item())
tot_eq_odds.add(eq_odds.mean().item())
progress_bar.set_description(
f'[{split}] epoch={epoch:d}, ce_loss={tot_ce_loss.mean():.4f}')
predictions = torch.cat(predictions)
targets = torch.cat(targets)
accuracy = accuracy_score(targets, predictions)
balanced_accuracy = balanced_accuracy_score(targets, predictions)
tn, fp, fn, tp = confusion_matrix(targets, predictions).ravel()
f1 = f1_score(targets, predictions)
writer.add_scalar('Accuracy/%s' % split, accuracy, epoch)
writer.add_scalar('Balanced Accuracy/%s' % split, balanced_accuracy, epoch)
writer.add_scalar('Cross Entropy/%s' % split, tot_ce_loss.mean(), epoch)
writer.add_scalar('True Positives/%s' % split, tp, epoch)
writer.add_scalar('False Negatives/%s' % split, fn, epoch)
writer.add_scalar('True Negatives/%s' % split, tn, epoch)
writer.add_scalar('False Positives/%s' % split, fp, epoch)
writer.add_scalar('F1 Score/%s' % split, f1, epoch)
writer.add_scalar('Learning Rate', optimizer.param_groups[0]['lr'], epoch)
writer.add_scalar('Stat. Parity/%s' % split, tot_stat_par.mean(), epoch)
writer.add_scalar('Equalized Odds/%s' % split, tot_eq_odds.mean(), epoch)
return tot_ce_loss
def run_experiment(maze_env,
trial_out_dir,
args=None,
n_generations=100,
save_results=False,
silent=False):
"""
The function to run the experiment against hyper-parameters
defined in the provided configuration file.
The winner genome will be rendered as a graph as well as the
important statistics of neuroevolution process execution.
Arguments:
maze_env: The maze environment to use in simulation.
trial_out_dir: The directory to store outputs for this trial
n_generations: The number of generations to execute.
save_results: The flag to control if intermdiate results will be saved.
silent: If True than no intermediary outputs will be
presented until solution is found.
args: The command line arguments holder.
Returns:
True if experiment finished with successful solver found.
"""
# set random seed
seed = int(time.time()) #1571021768#
print("Random seed: %d" % seed)
# Create Population of Robots and objective functions
robot = create_robot(maze_env, seed=seed)
obj_func = create_objective_fun(seed)
# Run for up to N generations.
start_time = time.time()
best_robot_genome_ser = None
best_robot_id = -1
solution_found = False
best_obj_func_coeffs = None
best_solution_novelty = 0
best_solution_distance = 0
stats = Statistics()
for generation in range(n_generations):
print("\n****** Generation: %d ******\n" % generation)
gen_time = time.time()
# evaluate objective function population
obj_func_coeffs, max_obj_func_fitness = evaluate_obj_functions(
obj_func, generation)
# evaluate robots population
robot_genome, solution_found, robot_fitness, distances, \
obj_coeffs, best_distance, best_novelty = evaluate_solutions(
robot=robot, obj_func_coeffs=obj_func_coeffs, generation=generation)
stats.post_evaluate(max_fitness=robot_fitness, errors=distances)
# store the best genome
if solution_found or robot.population.GetBestFitnessEver(
) < robot_fitness:
best_robot_genome_ser = pickle.dumps(robot_genome)
best_robot_id = robot_genome.GetID()
best_obj_func_coeffs = obj_coeffs
best_solution_novelty = best_novelty
best_solution_distance = best_distance
if solution_found:
print(
'\nSolution found at generation: %d, best fitness: %f, species count: %d\n'
% (generation, robot_fitness, len(robot.population.Species)))
break
# advance to the next generation
robot.population.Epoch()
obj_func.population.Epoch()
# print statistics
gen_elapsed_time = time.time() - gen_time
print("Generation fitness -> solution: %f, objective function: %f" %
(robot_fitness, max_obj_func_fitness))
print(
"Gen. species count -> solution: %d, objective function: %d" %
(len(robot.population.Species), len(obj_func.population.Species)))
print("Gen. archive size -> solution: %d, objective function: %d" %
(robot.archive.size(), obj_func.archive.size()))
print("Objective function coeffts: %s" % obj_coeffs)
print(
"Gen. best solution genome ID: %d, distance to exit: %f, novelty: %f"
% (robot_genome.GetID(), best_distance, best_novelty))
print("->")
print("Best fitness ever -> solution: %f, objective function: %f" %
(robot.population.GetBestFitnessEver(),
obj_func.population.GetBestFitnessEver()))
print(
"Best ever solution genome ID: %d, distance to exit: %f, novelty: %f"
% (best_robot_id, best_solution_distance, best_solution_novelty))
print("------------------------------")
print("Generation elapsed time: %.3f sec\n" %
(gen_elapsed_time))
elapsed_time = time.time() - start_time
# Load serialized best robot genome
best_robot_genome = pickle.loads(best_robot_genome_ser)
# write best genome to the file
best_genome_file = os.path.join(trial_out_dir, "best_robot_genome.pickle")
with open(best_genome_file, 'wb') as genome_file:
pickle.dump(best_robot_genome, genome_file)
# write the record store data
rs_file = os.path.join(trial_out_dir, "data.pickle")
robot.record_store.dump(rs_file)
print("==================================")
print("Record store file: %s" % rs_file)
print("Random seed: %d" % seed)
print("............")
print("Best solution fitness: %f, genome ID: %d" %
(robot.population.GetBestFitnessEver(), best_robot_genome.GetID()))
print("Best objective func coefficients: %s" % best_obj_func_coeffs)
print("------------------------------")
# Visualize the experiment results
show_results = not silent
if save_results or show_results:
if args is None:
visualize.draw_maze_records(maze_env,
robot.record_store.records,
view=show_results)
else:
visualize.draw_maze_records(maze_env,
robot.record_store.records,
view=show_results,
width=args.width,
height=args.height,
filename=os.path.join(
trial_out_dir, 'maze_records.svg'))
# store NoveltyItems archive data
robot.archive.write_to_file(
path=os.path.join(trial_out_dir, 'ns_items_all.txt'))
# create the best genome simulation path and render
maze_env = copy.deepcopy(robot.orig_maze_environment)
multi_net = NEAT.NeuralNetwork()
best_robot_genome.BuildPhenotype(multi_net)
depth = 8
try:
best_robot_genome.CalculateDepth()
depth = genome.GetDepth()
except:
pass
control_net = ANN(multi_net, depth=depth)
path_points = []
distance = maze.maze_simulation_evaluate(env=maze_env,
net=control_net,
time_steps=SOLVER_TIME_STEPS,
path_points=path_points)
print("Best solution distance to maze exit: %.2f, novelty: %.2f" %
(distance, best_solution_novelty))
visualize.draw_agent_path(robot.orig_maze_environment,
path_points,
best_robot_genome,
view=show_results,
width=args.width,
height=args.height,
filename=os.path.join(
trial_out_dir, 'best_solver_path.svg'))
# Draw the best agent phenotype ANN
visualize.draw_net(multi_net,
view=show_results,
filename="best_solver_net",
directory=trial_out_dir)
# Visualize statistics
visualize.plot_stats(stats,
ylog=False,
view=show_results,
filename=os.path.join(trial_out_dir,
'avg_fitness.svg'))
print("------------------------")
print("Trial elapsed time: %.3f sec" % (elapsed_time))
print("==================================")
return solution_found
def run(args=None):
device = 'cuda' if torch.cuda.is_available() and (
not args.no_cuda) else 'cpu'
num_train, train_loader, test_loader, input_size, input_channel, n_class = get_loaders(
args)
lossFn = nn.CrossEntropyLoss(reduction='none')
evalFn = lambda x: torch.max(x, dim=1)[1]
net = get_net(device,
args.dataset,
args.net,
input_size,
input_channel,
n_class,
load_model=args.load_model,
net_dim=args.cert_net_dim
) #, feature_extract=args.core_feature_extract)
timestamp = int(time.time())
model_signature = '%s/%s/%d/%s_%.5f/%d' % (args.dataset, args.exp_name,
args.exp_id, args.net,
args.train_eps, timestamp)
model_dir = args.root_dir + 'models_new/%s' % (model_signature)
args.model_dir = model_dir
count_vars(args, net)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
if isinstance(net, UpscaleNet):
relaxed_net = None
relu_ids = None
else:
relaxed_net = RelaxedNetwork(net.blocks, args.n_rand_proj).to(device)
relu_ids = relaxed_net.get_relu_ids()
if "nat" in args.train_mode:
cnet = CombinedNetwork(net,
relaxed_net,
lossFn=lossFn,
evalFn=evalFn,
device=device,
no_r_net=True).to(device)
else:
dummy_input = torch.rand((1, ) + net.dims[0],
device=device,
dtype=torch.float32)
cnet = CombinedNetwork(net,
relaxed_net,
lossFn=lossFn,
evalFn=evalFn,
device=device,
dummy_input=dummy_input).to(device)
n_epochs, test_nat_loss, test_nat_acc, test_adv_loss, test_adv_acc = args.n_epochs, None, None, None, None
if 'train' in args.train_mode:
tb_writer = SummaryWriter(model_dir)
stats = Statistics(len(train_loader), tb_writer, model_dir)
args_file = os.path.join(model_dir, 'args.json')
with open(args_file, 'w') as fou:
json.dump(vars(args), fou, indent=4)
write_config(args, os.path.join(model_dir, 'run_config.txt'))
eps = 0
epoch = 0
lr = args.lr
n_epochs = args.n_epochs
if "COLT" in args.train_mode:
relu_stable = args.relu_stable
# if args.layers is None:
# args.layers = [-2, -1] + relu_ids
layers = get_layers(args.train_mode,
cnet,
n_attack_layers=args.n_attack_layers,
protected_layers=args.protected_layers)
elif "adv" in args.train_mode:
relu_stable = None
layers = [-1, -1]
args.mix = False
elif "natural" in args.train_mode:
relu_stable = None
layers = [-2, -2]
args.nat_factor = 1
args.mix = False
elif "diffAI" in args.train_mode:
relu_stable = None
layers = [-2, -2]
else:
assert False, "Unknown train mode %s" % args.train_mode
print('Saving model to:', model_dir)
print('Training layers: ', layers)
for j in range(len(layers) - 1):
opt, lr_scheduler = get_opt(cnet.net,
args.opt,
lr,
args.lr_step,
args.lr_factor,
args.n_epochs,
train_loader,
args.lr_sched,
fixup="fixup" in args.net)
curr_layer_idx = layers[j + 1]
eps_old = eps
eps = get_scaled_eps(args, layers, relu_ids, curr_layer_idx, j)
kappa_sched = Scheduler(0.0 if args.mix else 1.0, 1.0,
num_train * args.mix_epochs, 0)
beta_sched = Scheduler(
args.beta_start if args.mix else args.beta_end, args.beta_end,
args.train_batch * len(train_loader) * args.mix_epochs, 0)
eps_sched = Scheduler(eps_old if args.anneal else eps, eps,
num_train * args.anneal_epochs, 0)
layer_dir = '{}/{}'.format(model_dir, curr_layer_idx)
if not os.path.exists(layer_dir):
os.makedirs(layer_dir)
print('\nnew train phase: eps={:.5f}, lr={:.2e}, curr_layer={}\n'.
format(eps, lr, curr_layer_idx))
for curr_epoch in range(n_epochs):
train(device,
epoch,
args,
j + 1,
layers,
cnet,
eps_sched,
kappa_sched,
opt,
train_loader,
lr_scheduler,
relu_ids,
stats,
relu_stable,
relu_stable_protected=args.relu_stable_protected,
beta_sched=beta_sched)
if isinstance(lr_scheduler, optim.lr_scheduler.StepLR
) and curr_epoch >= args.mix_epochs:
lr_scheduler.step()
if (epoch + 1) % args.test_freq == 0:
with torch.no_grad():
test_nat_loss, test_nat_acc, test_adv_loss, test_adv_acc = test(
device,
args,
cnet,
test_loader if args.test_set == "test" else
train_loader, [curr_layer_idx],
stats=stats,
log_ind=(epoch + 1) % n_epochs == 0)
if (epoch + 1) % args.test_freq == 0 or (epoch +
1) % n_epochs == 0:
torch.save(
net.state_dict(),
os.path.join(layer_dir, 'net_%d.pt' % (epoch + 1)))
torch.save(
opt.state_dict(),
os.path.join(layer_dir, 'opt_%d.pt' % (epoch + 1)))
stats.update_tb(epoch)
epoch += 1
relu_stable = None if relu_stable is None else relu_stable * args.relu_stable_layer_dec
lr = lr * args.lr_layer_dec
if args.cert:
with torch.no_grad():
diffAI_cert(
device,
args,
cnet,
test_loader if args.test_set == "test" else train_loader,
stats=stats,
log_ind=True,
epoch=epoch,
domains=args.cert_domain)
elif args.train_mode == 'print':
print('printing network to:', args.out_net_file)
dummy_input = torch.randn(1,
input_channel,
input_size,
input_size,
device='cuda')
net.skip_norm = True
torch.onnx.export(net, dummy_input, args.out_net_file, verbose=True)
elif args.train_mode == 'test':
with torch.no_grad():
test(device,
args,
cnet,
test_loader if args.test_set == "test" else train_loader,
[-1],
log_ind=True)
elif args.train_mode == "cert":
tb_writer = SummaryWriter(model_dir)
stats = Statistics(len(train_loader), tb_writer, model_dir)
args_file = os.path.join(model_dir, 'args.json')
with open(args_file, 'w') as fou:
json.dump(vars(args), fou, indent=4)
write_config(args, os.path.join(model_dir, 'run_config.txt'))
print('Saving results to:', model_dir)
with torch.no_grad():
diffAI_cert(
device,
args,
cnet,
test_loader if args.test_set == "test" else train_loader,
stats=stats,
log_ind=True,
domains=args.cert_domain)
exit(0)
else:
assert False, 'Unknown mode: {}!'.format(args.train_mode)
return test_nat_loss, test_nat_acc, test_adv_loss, test_adv_acc
def run_experiment(params,
rt_environment,
trial_out_dir,
n_generations=100,
save_results=False,
silent=False,
args=None):
"""
The function to run the experiment against hyper-parameters
defined in the provided configuration file.
The winner genome will be rendered as a graph as well as the
important statistics of neuroevolution process execution.
Arguments:
params: The NEAT parameters
rt_environment: The test environment for detector ANN evaluations
trial_out_dir: The directory to store outputs for this trial
n_generations: The number of generations to execute.
save_results: The flag to control if intermdiate results will be saved.
silent: If True than no intermediary outputs will be
presented until solution is found.
args: The command line arguments holder.
Returns:
True if experiment finished with successful solver found.
"""
# random seed
seed = 1569777981 #int(time.time())
# Create substrate
substrate = create_substrate()
# Create CPPN genome and population
g = NEAT.Genome(
0,
substrate.GetMinCPPNInputs(),
2, # hidden units
substrate.GetMinCPPNOutputs(),
False,
NEAT.ActivationFunction.TANH,
NEAT.ActivationFunction.
SIGNED_GAUSS, # The initial activation type for hidden
1, # hidden layers seed
params,
1) # one hidden layer
pop = NEAT.Population(g, params, True, 1.0, seed)
pop.RNG.Seed(seed)
# Run for up to N generations.
start_time = time.time()
best_genome_ser = None
best_ever_goal_fitness = 0
best_id = -1
solution_found = False
stats = Statistics()
for generation in range(n_generations):
print("\n****** Generation: %d ******\n" % generation)
gen_time = time.time()
# get list of current genomes
genomes = NEAT.GetGenomeList(pop)
# evaluate genomes
genome, fitness, errors = eval_genomes(genomes,
rt_environment=rt_environment,
substrate=substrate,
params=params)
stats.post_evaluate(max_fitness=fitness, errors=errors)
solution_found = fitness >= FITNESS_THRESHOLD
# store the best genome
if solution_found or best_ever_goal_fitness < fitness:
best_genome_ser = pickle.dumps(
genome) # dump to pickle to freeze the genome state
best_ever_goal_fitness = fitness
best_id = genome.GetID()
if solution_found:
print(
'Solution found at generation: %d, best fitness: %f, species count: %d'
% (generation, fitness, len(pop.Species)))
break
# advance to the next generation
pop.Epoch()
# print statistics
gen_elapsed_time = time.time() - gen_time
print("Best fitness: %f, genome ID: %d" % (fitness, best_id))
print("Species count: %d" % len(pop.Species))
print("Generation elapsed time: %.3f sec" % (gen_elapsed_time))
print("Best fitness ever: %f, genome ID: %d" %
(best_ever_goal_fitness, best_id))
# Find the experiment elapsed time
elapsed_time = time.time() - start_time
# Restore the freezed best genome from pickle
best_genome = pickle.loads(best_genome_ser)
# write best genome to the file
best_genome_file = os.path.join(trial_out_dir, "best_genome.pickle")
with open(best_genome_file, 'wb') as genome_file:
pickle.dump(best_genome, genome_file)
# Print experiment statistics
print("\nBest ever fitness: %f, genome ID: %d" %
(best_ever_goal_fitness, best_id))
print("\nTrial elapsed time: %.3f sec" % (elapsed_time))
print("Random seed:", seed)
# Visualize the experiment results
show_results = not silent
if save_results or show_results:
# Draw CPPN network graph
net = NEAT.NeuralNetwork()
best_genome.BuildPhenotype(net)
visualize.draw_net(net,
view=False,
node_names=None,
filename="cppn_graph.svg",
directory=trial_out_dir,
fmt='svg')
print("\nCPPN nodes: %d, connections: %d" %
(len(net.neurons), len(net.connections)))
# Draw the substrate network graph
net = NEAT.NeuralNetwork()
best_genome.BuildESHyperNEATPhenotype(net, substrate, params)
visualize.draw_net(net,
view=False,
node_names=None,
filename="substrate_graph.svg",
directory=trial_out_dir,
fmt='svg')
print("\nSubstrate nodes: %d, connections: %d" %
(len(net.neurons), len(net.connections)))
inputs = net.NumInputs()
outputs = net.NumOutputs()
hidden = len(net.neurons) - net.NumInputs() - net.NumOutputs()
print("\n\tinputs: %d, outputs: %d, hidden: %d" %
(inputs, outputs, hidden))
# Test against random retina configuration
l_index = random.randint(0, 15)
r_index = random.randint(0, 15)
left = rt_environment.visual_objects[l_index]
right = rt_environment.visual_objects[r_index]
err, outputs = rt_environment._evaluate(net, left, right, 3)
print("Test evaluation error: %f" % err)
print("Left flag: %f, pattern: %s" % (outputs[0], left))
print("Right flag: %f, pattern: %s" % (outputs[1], right))
# Test against all visual objects
fitness, avg_error, total_count, false_detetctions = rt_environment.evaluate_net(
net, debug=True)
print(
"Test evaluation against full data set [%d], fitness: %f, average error: %f, false detections: %f"
% (total_count, fitness, avg_error, false_detetctions))
# Visualize statistics
visualize.plot_stats(stats,
ylog=False,
view=show_results,
filename=os.path.join(trial_out_dir,
'avg_fitness.svg'))
return solution_found
def run(autoencoder, classifier, optimizer, loader, split, epoch):
predictions, targets, l_inf_diffs = list(), list(), list()
tot_mix_loss, tot_ce_loss, tot_dl2_loss = Statistics.get_stats(3)
tot_l2_loss, tot_stat_par, tot_eq_odds = Statistics.get_stats(3)
progress_bar = tqdm(loader)
for data_batch, targets_batch, protected_batch in progress_bar:
batch_size = data_batch.shape[0]
data_batch = data_batch.to(device)
targets_batch = targets_batch.to(device)
protected_batch = protected_batch.to(device)
x_batches, y_batches = list(), list()
assert batch_size % oracle.constraint.n_tvars == 0
k = batch_size // oracle.constraint.n_tvars
for i in range(oracle.constraint.n_tvars):
x_batches.append(data_batch[i: i + k])
y_batches.append(targets_batch[i: i + k])
if split == 'train':
autoencoder.train()
classifier.train()
latent_data = autoencoder.encode(data_batch)
data_batch_dec = autoencoder.decode(latent_data)
l2_loss = torch.norm(data_batch_dec - data_batch, dim=1)
logits = classifier(latent_data)
cross_entropy = binary_cross_entropy(logits, targets_batch)
predictions_batch = classifier.predict(latent_data)
stat_par = statistical_parity(predictions_batch, protected_batch)
eq_odds = equalized_odds(
targets_batch, predictions_batch, protected_batch
)
predictions.append(predictions_batch.detach().cpu())
targets.append(targets_batch.detach().cpu())
autoencoder.eval()
classifier.eval()
if oracle.constraint.n_gvars > 0:
domains = oracle.constraint.get_domains(x_batches, y_batches)
z_batches = oracle.general_attack(
x_batches, y_batches, domains, num_restarts=1,
num_iters=args.dl2_iters, args=args
)
else:
z_batches = None
latent_adv = autoencoder.encode(z_batches[0]).detach()
l_inf_diffs.append(
torch.abs(latent_data - latent_adv).max(1)[0].detach().cpu()
)
if split == 'train':
autoencoder.train()
classifier.train()
_, dl2_loss, _ = oracle.evaluate(
x_batches, y_batches, z_batches, args
)
mix_loss = torch.mean(
cross_entropy + args.dl2_weight * dl2_loss +
args.dec_weight * l2_loss
)
if split == 'train':
optimizer.zero_grad()
mix_loss.backward()
optimizer.step()
tot_ce_loss.add(cross_entropy.mean().item())
tot_dl2_loss.add(dl2_loss.mean().item())
tot_mix_loss.add(mix_loss.mean().item())
tot_l2_loss.add(l2_loss.mean().item())
tot_stat_par.add(stat_par.mean().item())
tot_eq_odds.add(eq_odds.mean().item())
progress_bar.set_description(
f'[{split}] epoch={epoch:d}, ce_loss={tot_ce_loss.mean():.4f}, '
f'dl2_loss={tot_dl2_loss.mean():.4f}, '
f'mix_loss={tot_mix_loss.mean():.4f}'
)
predictions = torch.cat(predictions)
targets = torch.cat(targets)
l_inf_diffs = torch.cat(l_inf_diffs)
accuracy = accuracy_score(targets, predictions)
balanced_accuracy = balanced_accuracy_score(targets, predictions)
tn, fp, fn, tp = confusion_matrix(targets, predictions).ravel()
f1 = f1_score(targets, predictions)
writer.add_scalar('Accuracy/%s' % split, accuracy, epoch)
writer.add_scalar('Balanced Accuracy/%s' % split, balanced_accuracy, epoch)
writer.add_scalar('Cross Entropy/%s' % split, tot_ce_loss.mean(), epoch)
writer.add_scalar('Decoder Loss/%s' % split, tot_l2_loss.mean(), epoch)
writer.add_scalar('DL2 Loss/%s' % split, tot_dl2_loss.mean(), epoch)
writer.add_scalar('Loss/%s' % split, tot_mix_loss.mean(), epoch)
writer.add_scalar('True Positives/%s' % split, tp, epoch)
writer.add_scalar('False Negatives/%s' % split, fn, epoch)
writer.add_scalar('True Negatives/%s' % split, tn, epoch)
writer.add_scalar('False Positives/%s' % split, fp, epoch)
writer.add_scalar('F1 Score/%s' % split, f1, epoch)
writer.add_scalar('Learning Rate', optimizer.param_groups[0]['lr'], epoch)
writer.add_scalar('Stat. Parity/%s' % split, tot_stat_par.mean(), epoch)
writer.add_scalar('Equalized Odds/%s' % split, tot_eq_odds.mean(), epoch)
writer.add_histogram('L-inf Differences/%s' % split, l_inf_diffs, epoch)
return tot_mix_loss
def main(args):
wandb_logger = WandbLogger(project=args.project)
checkpoint_callback = ModelCheckpoint(
save_top_k=1,
verbose=True,
monitor="val_loss",
mode="min",
)
datamodule = RosslerAttractorDataModule(
n_iter_train=args.n_iter_train,
n_iter_valid=args.n_iter_valid,
n_iter_test=args.n_iter_test,
init_pos_train=args.init_pos_train,
init_pos_test=args.init_pos_train,
init_pos_valid=args.init_pos_valid,
batch_size=args.batch_size,
delta_t=args.delta_t,
)
datamodule.setup()
criterion = nn.L1Loss(reduction="mean")
# use_cuda = False if args.gpus is None else True
# criterion_2 = SoftDTW(use_cuda=use_cuda, gamma=0.1, normalize=True)
criterion_2 = nn.MSELoss(reduction="mean")
# checkpoint_path = "Data/checkpoints/model_dtw.ckpt"
# model = DiscreteModel.load_from_checkpoint(checkpoint_path=checkpoint_path)
# model.hparams.criterion_2 = criterion_2
# model.configure_optimizers()
# model.hparams.lr = args.lr
model = DiscreteModel(
criterion=criterion,
criterion_2=criterion_2,
lr=args.lr,
delta_t=args.delta_t,
mean=datamodule.dataset_train.mean,
std=datamodule.dataset_train.std,
hidden_size=15,
)
trainer = Trainer(
gpus=args.gpus,
logger=wandb_logger,
max_epochs=args.epochs,
callbacks=[checkpoint_callback],
# auto_lr_find=True,
)
# trainer.tune(model=model, datamodule=datamodule)
trainer.fit(model=model, datamodule=datamodule)
trainer.test(model=model, datamodule=datamodule)
# Tests
TRAJECTORY_DUR = 1000
nb_steps = int(TRAJECTORY_DUR // args.delta_t)
checkpoint_path = Path(checkpoint_callback.best_model_path)
save_dir_path = checkpoint_path.parent
trained_model = DiscreteModel.load_from_checkpoint(
checkpoint_path=checkpoint_path)
trained_model.normalize = False
true_model = RosslerMap(delta_t=args.delta_t)
statstics_calculator = Statistics(wandb_logger)
dynamics_calculator = Dynamics(wandb_logger, true_model, trained_model,
nb_steps)
# TRAIN set
traj_pred, traj_true, time_list = compute_traj(trained_model, true_model,
args.init_pos_train,
nb_steps)
np.save(os.path.join(save_dir_path, "traj_pred_train.npy"), traj_pred)
np.save(os.path.join(save_dir_path, "traj_true_train.npy"), traj_true)
np.save(os.path.join(save_dir_path, "time_list_train.npy"), time_list)
statstics_calculator.add_traj(traj_true,
traj_pred,
time_list,
prefix="train ")
statstics_calculator.plot_all()
dynamics_calculator.add_traj(traj_true, traj_pred)
dynamics_calculator.plot_all()
# VAL set
# traj_pred, traj_true, time_list = compute_traj(
# trained_model, true_model, args.init_pos_valid, nb_steps
# )
# np.save(os.path.join(save_dir_path, "traj_pred_valid.npy"), traj_pred)
# np.save(os.path.join(save_dir_path, "traj_true_valid.npy"), traj_true)
# np.save(os.path.join(save_dir_path, "time_list_valid.npy"), time_list)
# statstics_calculator.add_traj(traj_true, traj_pred, time_list, prefix="valid ")
# statstics_calculator.plot_all()
# dynamics_calculator.add_traj(traj_true, traj_pred)
# dynamics_calculator.plot_all()
# TEST set
traj_pred, traj_true, time_list = compute_traj(trained_model, true_model,
args.init_pos_test,
nb_steps)
np.save(os.path.join(save_dir_path, "traj_pred_test.npy"), traj_pred)
np.save(os.path.join(save_dir_path, "traj_true_test.npy"), traj_true)
np.save(os.path.join(save_dir_path, "time_list_test.npy"), time_list)
statstics_calculator.add_traj(traj_true,
traj_pred,
time_list,
prefix="test ")
statstics_calculator.plot_all()
dynamics_calculator.add_traj(traj_true, traj_pred)
dynamics_calculator.plot_all()
