def run():
"""
Prepares and runs the whole system.
"""
args = parse_args()
logger = logging.getLogger("brc")
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
if args.log_path:
file_handler = logging.FileHandler(args.log_path)
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
else:
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
# string=",".join([g_num for g_num in args.gpus])
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"] = string
logger.info('Running with args : {}'.format(args))
if args.prepare:
prepare(args)
if args.train:
train(args)
if args.evaluate:
evaluate(args)
if args.predict:
predict(args)
def main():
# Create Metadata
metadata = Metadata()
# Create Replay Buffer
g_buffer = ReplayMem(metadata)
with Game(metadata) as game:
# Create Agent
agent = DDQLAgent(game.env.action_space.n)
# Load files
file_path = Path(Constants.MODEL_PATH)
metadata_file = (file_path / "metadata.p")
if all([x.validate_load()
for x in [g_buffer, agent]]) and metadata_file.is_file():
print("Loading state...")
with metadata_file.open("rb") as metadata_file:
metadata = pickle.load(metadata_file) # Metadata
game.load_metadata(metadata)
g_buffer.load(metadata)
agent.load()
print("Running Eval...")
evaluate(game, metadata, agent, 0)
def evaluate_generations(name,
experiment_id,
folder=None,
hops: int = 10,
unused_cpu: int = 2):
"""
Evaluate the population across its lifetime. At each generation, the ten best genomes are evaluated together with
the elite genome of the past five generations.
:param name: Name of the population
:param experiment_id: Experiment for which the population is trained (and now will be evaluated)
:param folder: Population-folder (~experiment level)
:param hops: Number of generations between each saved population
:param unused_cpu: Number of CPU cores not used
"""
# Fetch population and evaluation games
folder = folder if folder else get_folder(experiment_id)
pop = Population(
name=name,
folder_name=folder,
log_print=False,
use_backup=True,
)
_, game_ids_eval = get_game_ids(experiment_id=experiment_id)
# Perform the evaluations
max_gen = pop.generation
for gen in tqdm(range(0, max_gen + 1, hops)):
# Load in the current generation
if not pop.load(gen=gen):
raise Exception(
f"Population {name} is not trained for generation {gen}")
# Collect the used genomes
if gen > 5:
genomes = sorted([g for g in pop.population.values()],
key=lambda x: x.fitness if x.fitness else 0,
reverse=True)[:10]
for i in range(1, 6):
keys = [g.key for g in genomes]
g = copy.deepcopy(pop.best_genome_hist[gen - i]
[1]) # Copy since chance of mutation
while g.key in keys: # Already added to genomes, update keys
g.key += 1
genomes.append(g)
else:
# No history yet, use only the ten most fit genomes from the current generation
genomes = sorted([g for g in pop.population.values()],
key=lambda x: x.fitness if x.fitness else 0,
reverse=True)[:15]
# Evaluate the selected genomes
evaluate(
population=pop,
games=game_ids_eval,
genomes=genomes,
unused_cpu=unused_cpu,
overwrite=True,
)
def test_evaluate(self):
"""> Test if the population can be evaluated."""
# Folder must be root to load in make_net properly
if os.getcwd().split('\\')[-1] == 'tests': os.chdir('..')
pop = get_population()
evaluate(
population=pop,
games=[-2],
debug=True,
)
def test_evaluate_expressions_with_sublevel():
expressions = {
'(4+4) * 10': 80,
'(10/10) * (-5+1)': -4,
}
for expr, result in expressions.items():
assert result == main.evaluate(expr)
def test_evaluate_expressions_with_two_sublevels():
expressions = {
'(4+(16/4)) * 10': 80,
'((4x2+2)/(5x2)) * ((5-10)+(10-9))': -4,
}
for expr, result in expressions.items():
assert result == main.evaluate(expr)
def test_evaluate_simple_values():
expressions = {
' + 1 ': 1,
'-1': -1,
}
for expr, result in expressions.items():
assert result == main.evaluate(expr)
def test_evaluate_expressions_more_complex():
"all expressions from https://teresachiacchio.wixsite.com/matemagica/expressoes"
expressions = {
'4 – [– (6 + 4) + (3 – 2 – 1)]': 14,
'15 x 2 – 30 ÷ 3 + 7': 27,
'10 x [30 ÷ (2 x 3 + 4) + 15]': 180,
'25 + {14 – [25 x 4 + 40 – (20 ÷ 2 + 10)]}': -81,
'37 – 14 + 35 – 10': 48,
'32 ÷ 2 . 3 ÷ 4 . 5': 3.091787439613527,
'32 ÷ 2 x 3 ÷ 4 x 5': 60,
'180 ÷ 4 * {9 ÷ [3 * (3 * 1)]}': 45,
'16 : (-4) x 2': -8,
'16 x(-4): 2': -32,
'10 + 2² x 3': 22,
'5² – 4 x 2 + 3': 20,
'20 – [4² + ( 2³ – 7 )]': 3,
'10 –{ 10 + [ 8² : ( 10 – 2 ) + 3 x 2 ] }': -14,
'27 + {14 + 3 x [100 : (18 – 4 x 2) + 7] } : 13': 32,
'{100 – 413 x (20 – 5 x 4) + 25} : 5': 25,
'25 + { 12 + [ 2 – ( 8 – 6 ) + 2 ]}': 39,
'38 – { 20 – [ 22 – ( 5 + 3) + ( 7 – 4 +1)]}': 36,
'26 + { 12 – [ ( 30 – 18) + ( 4 – 1) – 6 ] – 1 }': 28,
'(90+10)*2 / (90+(1000/ ( 50+25*2)))': 2,
}
for expr, result in expressions.items():
assert result == main.evaluate(expr)
def test_attribute_evaluation():
evaluation, model_cfgs = evaluation_builder.build(attribute_evaluation_cfg)
with torch.no_grad():
for model_cfg in model_cfgs:
model = model_builder.build(model_cfg)
model = DataParallel(model)
score = evaluate(evaluation, model, delete=True)
print(score)
def test_evaluate_sequence_expressions():
expressions = {
' 2 x 2 / 2 ': 2,
' 2 / 2 * 2 ': 2,
' 2 + 2 / 2 ': 3,
' 2 / 2 + 1 ': 2,
}
for expr, result in expressions.items():
assert result == main.evaluate(expr)
def test_evaluate_simple_expressions():
expressions = {
' 1 + 1 ': 2,
'100 / 4': 25,
'9 * 9': 81,
'15 - 5': 10
}
for expr, result in expressions.items():
assert result == main.evaluate(expr)
def attack_PGM(model,test_data_loader, p=2, epsilon = 0.01, alpha = 0.1, random=False) :
model.eval()
T_adv = 15
loss_function = nn.CrossEntropyLoss()
valid_data_x = torch.FloatTensor(len(test_data_loader.dataset),1,28,28)
valid_data_y = torch.LongTensor(len(test_data_loader.dataset))
count = 0
for x_, y_ in test_data_loader :
if USE_CUDA:
x_, y_ = x_.cuda(), y_.cuda()
input_var, target_var = Variable(x_, requires_grad=True), Variable(y_)
if random == True :
noise = torch.FloatTensor(x_.size()).uniform_(-epsilon, epsilon)
if USE_CUDA :
noise = noise.cuda()
input_var.data += noise
#generate attack data
for n in range(1, T_adv + 1) :
step_alpha = float(alpha /np.sqrt(n))
zero_gradients(input_var)
output = model(input_var)
loss = loss_function(output, target_var)
loss.backward()
x_grad = input_var.grad.data
if p == 2:
# delta_x = epsilon * x_grad / torch.norm(x_grad.view(len(x_),1),2,1)
grad_ = x_grad.view(len(x_),-1)
grad_ = grad_/torch.norm(grad_,2,1).view(len(x_),1).expand_as(grad_)
normed_grad = epsilon * grad_.view_as(x_grad)
else:
# infinity-norm
normed_grad = epsilon * torch.sign(x_grad)
# xi + alpha_t * delta_x
# normed_grad = step_alpha * normed_grad
# normed_grad.clamp_(-epsilon, epsilon)
# input_var.data += normed_grad
normed_grad.clamp_(-epsilon, epsilon)
step_adv = input_var.data + step_alpha * normed_grad # x^(t+1) = x^(t) + alpha * delta_x^t
total_adv = step_adv - x_ #x^t - x
total_adv.clamp_(-epsilon, epsilon) # ||x^t-x|| <= epsilon
input_adv = x_ + total_adv
input_adv.clamp_(-1.0, 1.0) #mnist data between -1,1
input_var.data = input_adv
# print (np.all(input_var.data.cpu() == x_.data))
valid_data_x[count:count+len(x_),:] = input_var.data.cpu()
valid_data_y[count:count+len(x_)] = y_.clone().cpu()
count += len(x_)
dataset = torch.utils.data.TensorDataset(valid_data_x, valid_data_y)
data_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)
return main.evaluate(model,data_loader)
def main(_):
# set up logger
tf.logging.set_verbosity(tf.logging.INFO)
params = global_params
# try loading parameters
# priority: command line > saver > default
# 1. load latest path to load parameters
params.parse(flags.FLAGS.parameters)
params = load_parameters(params, params.output_dir)
# 2. refine with command line parameters
params.parse(flags.FLAGS.parameters)
# set up random seed
random.seed(params.random_seed)
np.random.seed(params.random_seed)
tf.set_random_seed(params.random_seed)
# loading vocabulary
tf.logging.info("Begin Loading Vocabulary")
start_time = time.time()
params.src_vocab = Vocab(params.src_vocab_file)
params.tgt_vocab = Vocab(params.tgt_vocab_file)
tf.logging.info("End Loading Vocabulary, Source Vocab Size {}, "
"Target Vocab Size {}, within {} seconds".format(
params.src_vocab.size(), params.tgt_vocab.size(),
time.time() - start_time))
mode = flags.FLAGS.mode
if mode == "train":
# save parameters
save_parameters(params, params.output_dir)
# load the recorder
params = setup_recorder(params)
graph.train(params)
elif mode == "test":
graph.evaluate(params)
else:
tf.logging.error("Invalid mode: {}".format(mode))
def test_reid_evaluation():
dataloader = dataloader_builder.build(reid_cfg)
# restore
model_cfgs = evaluation_model_builder.build(baseline_model_cfg)
model = model_builder.build(model_cfgs[0])
model = DataParallel(model)
_run = {'config': {'device': torch.device('cuda')}}
score = evaluate([dataloader], model, _run, "test")
print(score)
def main(args):
"""Evaluation corresponding to given argparse arguments."""
# device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
agent, env, eval_env = configure(args)
agent = agent.to(device)
# load checkpoints if directory is not empty
agent_file = join(args.logdir, 'best_agent.pt')
R = - np.inf
if exists(agent_file):
state_dict = torch.load(agent_file)
R = state_dict["return"]
info(f"eval> Loading agent with return {R}...")
agent.load_state_dict(state_dict)
else:
raise ValueError(f"{agent_file} does not exists, no agent available...")
evaluate(args.dt, 0, eval_env, agent, args.time_limit,
eval_return=True, video=True, progress_bar=True, no_log=True)
def attack_WRM(model,test_data_loader, gamma, max_lr0, epsilon = 0.01, random=False, get_err=False) :
model.eval()
T_adv = 15
loss_function = nn.CrossEntropyLoss()
if get_err:
valid_data_x = torch.FloatTensor(len(test_data_loader.dataset),1,28,28)
valid_data_y = torch.LongTensor(len(test_data_loader.dataset))
count = 0
err=0
rhos=[]
for x_, y_ in test_data_loader :
if USE_CUDA:
x_, y_ = x_.cuda(), y_.cuda()
x_, y_ = Variable(x_), Variable(y_)
#initialize z_hat with x_
z_hat = x_.data.clone()
if USE_CUDA:
z_hat = z_hat.cuda()
if random :
noise = torch.FloatTensor(x_.size()).uniform_(-epsilon, epsilon)
if USE_CUDA :
noise = noise.cuda()
z_hat += noise
z_hat = Variable(z_hat,requires_grad=True)
#running the maximizer for z_hat
optimizer_zt = torch.optim.Adam([z_hat], lr=max_lr0)
loss_zt = 0 # phi(theta,z0)
rho = 0 #E[c(Z,Z0)]
for n in range(1,T_adv+1) :
optimizer_zt.zero_grad()
delta = z_hat - x_
rho = torch.mean((torch.norm(delta.view(len(x_),-1),2,1)**2))
loss_zt = - ( loss_function(model(z_hat),y_)- gamma * rho)
loss_zt.backward()
optimizer_zt.step()
main.adjust_lr_zt(optimizer_zt,max_lr0, n+1)
rhos.append(rho.data[0])
if get_err:
valid_data_x[count:count+len(x_),:] = z_hat.data.cpu().clone()
valid_data_y[count:count+len(x_)] = y_.data.cpu().clone()
count += len(x_)
if get_err:
dataset = torch.utils.data.TensorDataset(valid_data_x, valid_data_y)
data_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)
err=1.0-main.evaluate(model,data_loader)/100
return torch.mean(torch.FloatTensor(rhos)),err
def main(sys_args):
parser = argparse.ArgumentParser(description='Testing helper for DeNSe parser.')
parser.add_argument('--test_file', type=str, required=True,
help='test data')
parser.add_argument('--output_file', type=str, default=None,
help='output file')
parser.add_argument('--model_dir', type=str, default='model',
help='folder of the model')
parser.add_argument('--data_model', type=str, default='data.pkl',
help='data model file')
parser.add_argument('--classifier_model', type=str, default='model.pkl',
help='classifier model file')
parser.add_argument('--best_model', type=str, default='best.pkl',
help='best classifier model file')
parser.add_argument('--batch_size', type=int, default=50,
help='mini-batch size')
args = parser.parse_args(sys_args)
evaluate(args)
def play_models_and_update(black, white, elos):
output_dir = '/tmp/play_models2'
os.system('mkdir ' + output_dir);
white_win = main.evaluate(black, white, output_dir=output_dir, games=1, readouts=400)
black_elo = elos[black]
white_elo = elos[white]
if white_win > 0:
white_elo, black_elo = new_elos(white_elo, black_elo)
else:
black_elo, white_elo = new_elos(black_elo, white_elo)
elos[black] = black_elo
elos[white] = white_elo
def test():
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess=sess, save_path=save_path)
word2id = read_vocab(vocab_dir)
query_test, doc_test = process_file(test_query_dir, test_docs_dir, word2id)
print('Testing...')
loss_test, p_1_test, mrr_test = evaluate(sess, query_test, doc_test)
msg = 'Test Loss: {0:>6.2}, Test P@1: {1:>7.2%}, Test MRR: {2:8.2%}'
print(msg.format(loss_test, p_1_test, mrr_test))
def evaluate_population(population):
for count, chromosome in enumerate(population):
logger.info('Evaluating network %d of %d' % (count, len(population)))
parameter = chromosome['params']
# chromosome['fitness'] = 1 / parameter['learning_rate'] * parameter['batch_size']
# chromosome['fitness'] *= parameter['l2_reg_scale'] * parameter['dropout_rate']
# chromosome['fitness'] *= parameter['shuffle'] * 100
# chromosome['fitness'] *= parameter['hidden_units'][0]
try:
error = g_research.evaluate(max_steps=1.2e5, **parameter)
chromosome['fitness'] = 1.0 / error
except:
logger.exception('Error while evaluating {}'.format(parameter))
chromosome['fitness'] = 0.0
def play_models_and_update(black, white, elos):
output_dir = '/tmp/play_models2'
os.system('mkdir ' + output_dir)
white_win = main.evaluate(black,
white,
output_dir=output_dir,
games=1,
readouts=400)
black_elo = elos[black]
white_elo = elos[white]
if white_win > 0:
white_elo, black_elo = new_elos(white_elo, black_elo)
else:
black_elo, white_elo = new_elos(black_elo, white_elo)
elos[black] = black_elo
elos[white] = white_elo
def test_evaluate(self):
self.assertEqual(
main.evaluate({
'type': '/',
'left': {
'type': '+',
'left': {
'type': 'number',
'value': '1'
},
'right': {
'type': 'number',
'value': '2'
},
},
'right': {
'type': 'number',
'value': '3'
},
}), 1)
def forex_eval(params):
print(f'Using params: {params}')
now_str = str(datetime.now()).replace(':', '-').replace(' ',
'_').split('.')[0]
writer = SummaryWriter(os.path.join(log_dir, now_str))
config = helpers.get_config(config_path)
if 'n_dense_layers' in params and 'n_nodes_dense_layers' in params:
config['dense_params'] = [{
'out_features':
params['n_nodes_dense_layers']
} for _ in range(params['n_dense_layers'])]
if 'n_conv_layers' in params and 'conv_filter_size' in params and 'conv_kernel_size' in params:
config['conv_params'] = [{
'out_channels': params['conv_filter_size'],
'kernel_size': params['conv_kernel_size']
} for _ in range(params['n_conv_layers'])]
for param, value in params.items():
if param in config: config[param] = value
model = train(device, writer, config, data_dir)
metrics = evaluate(model, device, writer, config, data_dir, config_path)
save(model, device, metrics, now_str, config, model_dir, config_path)
return {k: (v, 0.0) for k, v in metrics.items() if v is not tuple}
def create_visuals(model_path):
from config import args
args['malnet_tiny'] = False
create_image_symlinks(args)
train_gen, val_gen, test_gen = get_generators(args)
args['y_train'] = train_gen.labels
args['class_indexes'] = list(val_gen.class_indices.values())
args['class_labels'] = list(val_gen.class_indices.keys())
args['num_classes'] = len(val_gen.class_indices.keys())
os.environ["CUDA_DEVICE_ORDER"] = 'PCI_BUS_ID'
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(
str(d) for d in args['devices'])
model = tf.keras.models.load_model(model_path, compile=False)
loss, _ = get_loss(args)
model.compile(loss=loss, optimizer='adam', metrics=[])
y_pred, y_scores = evaluate(test_gen, model)
y_true = test_gen.labels.tolist()
images_to_view = []
counts = defaultdict(int)
for idx, label in enumerate(y_true):
if counts[label] < 20:
images_to_view.append(idx)
counts[label] += 1
for idx in images_to_view:
if y_pred[idx] == y_true[idx]:
path = test_gen.filepaths[idx]
run_gradcam2(path, model)
def test_evaluate(self):
list = ['aber das nächste mal doch bitte im querformat']
result = mn.evaluate(list)
self.assertEqual(result, 0)
vtype=grb.GRB.CONTINUOUS,
name="movies")
# re-write obj function using constraints, since gurobi does not support mupltication of multiple variables
objective = 0
for i, j in zip(rows, cols):
z = opt_model.addVar(name=f'z_{i}_{j}')
opt_model.addConstr(u[i] * v[j] == z)
objective += (z - trainX[i, j]) * (z - trainX[i, j])
start = time.time()
# OPTIMIZE
opt_model.setObjective(objective, grb.GRB.MINIMIZE)
opt_model.optimize()
elapsed = time.time() - start
print(f'Optimization took {elapsed}s')
# get u and v vectors
u_arr = np.array([u[i].X for i in range(dataset.n_users)], dtype=np.float64)
v_arr = np.array([v[i].X for i in range(dataset.n_movies)], dtype=np.float64)
print('shapes', u_arr.shape, v_arr.shape)
# store latest feature vectors
np.save('./data/gurobi2_U.npy', u_arr)
np.save('./data/gurobi2_V.npy', v_arr)
# keep evaluation schema fixed wrt other experiments
train_mse = evaluate(u_arr, v_arr, trainX, 'sparse')
# mse = evaluate(u_arr, v_arr, testX, 'sparse')
als = ALSSparse(u=u_arr, v=v_arr, dataset=trainX)
fun_eval = als.function_eval()
print("Final function evaluation", fun_eval)
def main(_):
# set up logger
tf.logging.set_verbosity(tf.logging.INFO)
tf.logging.info("Welcome Using Zero :)")
params = global_params
# try loading parameters
# priority: command line > saver > default
# 1. load latest path to load parameters
if os.path.exists(flags.FLAGS.config):
params.override_from_dict(eval(open(flags.FLAGS.config).read()))
params = load_parameters(params, params.output_dir)
# 2. refine with command line parameters
if os.path.exists(flags.FLAGS.config):
params.override_from_dict(eval(open(flags.FLAGS.config).read()))
params.parse(flags.FLAGS.parameters)
# set up random seed
random.seed(params.random_seed)
np.random.seed(params.random_seed)
tf.set_random_seed(params.random_seed)
# loading vocabulary
tf.logging.info("Begin Loading Vocabulary")
start_time = time.time()
full_task = get_task(params, True)
if not os.path.exists(params.word_vocab_file):
params.word_vocab = Vocab(lower=params.lower)
params.word_vocab.make_vocab(
full_task,
use_char=False,
embedding_path=params.pretrain_word_embedding_file)
else:
params.word_vocab = Vocab(lower=params.lower,
vocab_file=params.word_vocab_file)
if params.use_char:
if not os.path.exists(params.char_vocab_file):
params.char_vocab = Vocab(lower=False)
params.char_vocab.make_vocab(full_task,
use_char=True,
embedding_path=None)
else:
params.char_vocab = Vocab(lower=False,
vocab_file=params.char_vocab_file)
tf.logging.info("End Loading Vocabulary, Word Vocab Size {}, "
"Char Vocab Size {}, within {} seconds".format(
params.word_vocab.size(),
params.char_vocab.size() if params.use_char else 0,
time.time() - start_time))
if flags.FLAGS.mode == "vocab":
save_parameters(params, params.output_dir)
return
# save parameters
if flags.FLAGS.mode == "train":
save_parameters(params, params.output_dir)
# loading bert config
if params.enable_bert:
bert_config = bert.load_config(params.bert_dir)
params.bert = tc.training.HParams(**bert_config)
# loading vocabulary
tf.logging.info("Begin Loading Vocabulary")
start_time = time.time()
params.bert.vocab = bert.load_vocab(params.bert_dir)
tf.logging.info(
"End Loading Vocabulary, Vocab Size {}, within {} seconds".format(
params.bert.vocab.size,
time.time() - start_time))
# loading task label information
params.label_size = full_task.get_label_size()
# print parameters
print_parameters(params)
# print the used datasets
tf.logging.info("Task {} is performed with data {}".format(
params.task, full_task.data_path))
mode = flags.FLAGS.mode
if mode == "train":
# load the recorder
params = setup_recorder(params)
graph.train(params)
elif mode == "test":
graph.evaluate(params)
else:
tf.logging.error("Invalid mode: {}".format(mode))
def test_evaluate_with_wrong_text():
with pytest.raises(SyntaxError):
main.evaluate('')
def main():
population = load_best_population()
individual = population[0]
evaluate(individual=individual, render=True)
def test_evaluate_multiple_evaluations():
assert 42 == main.evaluate(main.evaluate('5² – 4 x 2 + 3'), '+', main.evaluate('10 + 2² x 3'))
def test_evaluate_with_wrong_parentheses():
with pytest.raises(SyntaxError):
main.evaluate('(1+1')
def test3(self):
self.assertEqual(evaluate("5"), 5)
def test4(self):
self.assertEqual(evaluate("5+4-3"), 6)
def test4(self):
self.assertEqual(evaluate("5-(4-3)"), 4)
def test1(self):
self.assertEqual(evaluate("5+4"), 9)
def test2(self):
self.assertEqual(evaluate("5+4+3"), 12)
