def train_global(cluster_args):
'''
1. apply GMM or kmeans on the repr after lstm
returns
cluster_predictions (type:int): shape (n,)
'''
X = cluster_args['X']
y = cluster_args['y']
X_train = cluster_args['X_train']
y_train = cluster_args['y_train']
global_model_fn = cluster_args['global_model_fn']
FLAGS = cluster_args['FLAGS']
global_model = torch.load(global_model_fn)
global_model.cuda()
def extract_embedding(x):
'''x assumes to be pytorch tensor'''
o, (h, c) = global_model.lstm(x)
embeddings = h[-1]
return embeddings
# Get Embeddings:
embedded_train = get_output(global_model, create_loader(X_train, y_train))
embedded_all = get_output(global_model, create_loader(X, y))
return gmm_fit_and_predict(embedded_train,
embedded_all,
FLAGS,
savename_suffix="_global")
def decode(self, latent):
'''
Decode the latent variables and generate output.
'''
pose, appearance = latent['pose'].view(
-1, self.pose_latent_size), latent['appearance'].view(
-1, self.appearance_latent_size)
objects = self.object_decoder(appearance)
components = utils.object_to_image(objects, pose, self.image_size)
components = components.view(-1, self.n_frames_total,
self.total_components, self.n_channels,
self.image_size, self.image_size)
masks = latent['mask']
if masks is not None:
masked_components = components[:, :self.
n_frames_input] * masks.unsqueeze(
-1).unsqueeze(-1)
masked_output = utils.get_output(masked_components)
output = utils.get_output(components)
return output, masked_output, components
def main(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
variants = utils.make_variants(args.python_versions, args.build_types,
args.mpi_types, args.cuda_versions)
pr_branch = utils.get_output("git log -1 --format='%H'")
utils.run_and_log("git remote set-head origin -a")
default_branch = utils.get_output(
"git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'"
)
variant_build_results = dict()
for variant in variants:
utils.run_and_log("git checkout {}".format(default_branch))
master_build_config_data, master_config = _get_configs(variant)
master_build_numbers = _get_build_numbers(master_build_config_data,
master_config, variant)
utils.run_and_log("git checkout {}".format(pr_branch))
pr_build_config_data, pr_config = _get_configs(variant)
current_pr_build_numbers = _get_build_numbers(pr_build_config_data,
pr_config, variant)
print("Build Info for Variant: {}".format(variant))
print("Current PR Build Info: {}".format(current_pr_build_numbers))
print("Master Branch Build Info: {}".format(master_build_numbers))
#No build numbers can go backwards without a version change.
for package in master_build_numbers:
if package in current_pr_build_numbers and current_pr_build_numbers[
package]["version"] == master_build_numbers[package][
"version"]:
assert int(current_pr_build_numbers[package]["number"]) >= int(
master_build_numbers[package]["number"]
), "If the version doesn't change, the build number can't be reduced."
#If packages are added or removed, don't require a version change
if set(master_build_numbers.keys()) != set(
current_pr_build_numbers.keys()):
return
#At least one package needs to increase the build number or change the version.
checks = [
current_pr_build_numbers[package]["version"] !=
master_build_numbers[package]["version"]
or int(current_pr_build_numbers[package]["number"]) > int(
master_build_numbers[package]["number"])
for package in master_build_numbers
]
variant_build_results[utils.variant_string(
variant["python"], variant["build_type"], variant["mpi_type"],
variant["cudatoolkit"])] = any(checks)
assert any(
variant_build_results.values()
), "At least one package needs to increase the build number or change the version in at least one variant."
def train_val_curve(cluster_args):
# 1. learn validation curve: save as snapshot; reuse the code
# 2. train from x to validation curve cluster
# 3. apply this on Train, val and test to save
X = cluster_args['X']
y = cluster_args['y']
X_train = cluster_args['X_train']
y_train = cluster_args['y_train']
X_val = cluster_args['X_val']
y_val = cluster_args['y_val']
global_model_dir = cluster_args['global_model_dir']
global_model_fn = cluster_args['global_model_fn']
FLAGS = cluster_args['FLAGS']
def sorted_by_epoch(l):
return sorted(l,
key=lambda s: int(re.search("epoch(.*)\.m", s).group(1)))
curves = []
val_loader = create_loader(X_val, y_val, batch_size=64)
y_val = dataset2numpy(y_val)
criterion = nn.BCELoss(reduction='none')
for fn in sorted_by_epoch(glob.glob(global_model_dir + "/epoch*.m")):
net = torch.load(fn)
y_pred_val = get_output(net, val_loader).ravel() # (n_val,)
# collect the loss
curves.append(-criterion(
torch.from_numpy(y_pred_val),
torch.from_numpy(y_val)).detach().cpu().numpy()) # (epochs, n_val)
curves = np.array(curves).T # (n_val, epochs)
# joblib.dump(curves, 'val_curves.pkl')
# train a mapping from input to val_curve
k = FLAGS.num_clusters
assignment, experts_epochs = val_curve_kmeans(curves, k=k, niters=10)
if FLAGS.not_pt:
net = None
else:
net = torch.load(global_model_fn)
net.rest[-2] = nn.Linear(net.rest[-2].in_features, k).cuda()
net.rest = net.rest[:-1] # drop sigmoid layer
gate = train_assignment(FLAGS,
k,
assignment,
X_val,
net=net,
savename_suffix="_val_curve")
# Get cluster membership: from (n, k) -> (n,)
cluster_preds = get_output(gate, create_loader(X, y)).argmax(1)
return cluster_preds
def test_udp_denial_of_service():
utils.get_output("wireshark/s5-eth1-iperf-udp-h1-h5.pcap",
"test/s5-udp-h1-h5.txt")
def callback(x):
return x['dst'] == '10.0.0.5' and x['protocol'] == 'UDP'
df_1 = utils.get_df("test/s5-udp-h1-h5.txt")
df_1.columns = ['src', 'dst', 'protocol']
df_1 = utils.filter_by(df_1, ['dst', 'protocol'], callback)
size_1 = len(df_1.index)
assert size_1 < 893
os.system("rm test/s5-udp-h1-h5.txt")
def main(opt):
logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO)
for pref in opt.prefixes:
ms_input = MsInput.basedir_form(opt.base_dir, pref)
LG.info("testing on input %s", ms_input)
res1 = get_output(fast(opt, ms_input))
res2 = get_output(slow(opt.slow_prg, opt.slow_load_dir, ms_input))
err_lst = check_res(res1, res2)
if any(err_lst):
for err in err_lst:
if err:
print "\t" + err
print
def main(opt):
logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO)
base_dict = dict(lazy_wl=False,
rank_fail=True,
use_maxrep=opt.use_maxrep,
load_maxrep=opt.load_maxrep,
load_cst=opt.load_cst,
space_usage=True,
time_usage=True,
answer=False)
for i, pref in enumerate(opt.prefixes):
ispec = MsInput.basedir_form(opt.base_dir, pref)
command = MsInterface.command_from_dict(
dict(s_path=ispec.s_path, t_path=ispec.t_path, **base_dict))
for j in range(opt.repeat):
with open(opt.output, 'a') as fd:
res = get_output(command)
if i + j == 0:
fd.write(res[0] + ",label,b_path\n")
for line in res[1:]:
fd.write(
line.replace(" ", "") + ("," + opt.label) +
("," + pref) + "\n")
def linear_search(file_path_P, file_path_T):
read_accesses = 0
write_accesses = 0
to_write = []
P = get_P(file_path_P)
file_T = get_T(file_path_T)
length_T = get_length_file(file_path_T)
output = get_output('output_files/output_linear.txt')
for iteration_index in range(0, length_T, B):
start_reading_from = LINE_SIZE * iteration_index
if(iteration_index + B > length_T):
number_of_lines = length_T - iteration_index
else:
number_of_lines = B
lines = read_many_lines(start_reading_from, number_of_lines, file_T)
read_accesses += 1
for element in lines:
if element in P:
found_element = element.zfill(9) + '\n'
to_write.append(found_element)
if len(to_write) * LINE_SIZE >= BLOCK_SIZE:
output.write(''.join(to_write))
write_accesses += 1
to_write = []
file_T.close()
if(len(to_write) != 0):
output.write(''.join(to_write))
write_accesses += 1
output.close()
return read_accesses, write_accesses
def _eval_model(self, dev_data_loader):
all_uuids = []
all_labels = []
all_preds = []
all_probs = []
loss_vals = []
for batch in dev_data_loader.batch_iter():
uuids = batch['uuids']
label_ids, pred_ids, probs, loss_val = self._get_preds(batch)
all_uuids += uuids
all_labels += label_ids
all_preds += pred_ids
all_probs += probs
loss_vals.append(loss_val)
print(len(all_labels))
print(len(all_preds))
gold, output = get_output(all_uuids, all_labels, all_preds, all_probs)
score = get_hit_score(gold, output)
#score = scorer(all_labels, all_preds)
loss_val = sum(loss_vals) / len(loss_vals)
return score, loss_val
def indexed_search(path_p, path_t):
read_accesses = 0
write_accesses = 0
P = get_P(path_p)
T = get_T(path_t)
out = get_output('output_files/output_indexed.txt')
S = []
to_write = []
len_t = get_length_file(path_t)
n_blocks = math.ceil(len_t / B)
for i in range(n_blocks):
line_str = read_a_line_from_file(T, i * B)
read_accesses += 1
S.append(line_str)
for searched in P:
s_interval_id = binary_search_modified(S, searched, len(S) - 1)
if s_interval_id != -1:
start_reading_at = LINE_SIZE * s_interval_id * B
many_lines = read_many_lines(start_reading_at, B, T)
read_accesses += 1
for block_number in many_lines:
if block_number != '' and searched == block_number:
num = searched.zfill(9) + '\n'
to_write.append(num)
if len(to_write) * LINE_SIZE >= BLOCK_SIZE:
out.write(''.join(to_write))
write_accesses += 1
to_write = []
T.close()
if (len(to_write) != 0):
out.write(''.join(to_write))
write_accesses += 1
out.close()
return read_accesses, write_accesses
def test_tcp_balance_of_charges():
utils.get_output("wireshark/s2-eth1-iperf-h1-h4.pcap", "test/s2-h1-h4.txt")
utils.get_output("wireshark/s3-eth1-iperf-h1-h4.pcap", "test/s3-h1-h4.txt")
def callback(x):
return x['dst'] == '10.0.0.4' and x['protocol'] == 'TCP'
df_1 = utils.get_df("test/s2-h1-h4.txt")
df_1.columns = ['src', 'dst', 'protocol']
df_1 = utils.filter_by(df_1, ['dst', 'protocol'], callback)
size_1 = len(df_1.index)
df_2 = utils.get_df("test/s3-h1-h4.txt")
df_2.columns = ['src', 'dst', 'protocol']
df_2 = utils.filter_by(df_2, ['dst', 'protocol'], callback)
size_2 = len(df_2.index)
os.system("rm test/s2-h1-h4.txt")
os.system("rm test/s3-h1-h4.txt")
assert (size_1 == 0 and size_2 != 0) or (size_1 != 0 and size_2 == 0)
def train_ae_pytorch(cluster_args):
'''
pytorch version of train ae
returns
cluster_predictions (type:int): shape (n,)
'''
X = cluster_args['X']
X_train = cluster_args['X_train']
X_val = cluster_args['X_val']
FLAGS = cluster_args['FLAGS']
# Train autoencoder
encoder = train_seq_ae_pytorch(X_train, X_val, FLAGS)
# Get Embeddings
embedded_train = get_output(encoder,
create_loader(X_train, X_train, batch_size=64))
embedded_all = get_output(encoder, create_loader(X, X, batch_size=64))
return gmm_fit_and_predict(embedded_train,
embedded_all,
FLAGS,
savename_suffix="_ae")
def save_output(model, model_name, X, y, cohorts, all_tasks, fname, FLAGS):
if 'mtl' in model_name:
y_pred = get_output_mtl(model, X, y, cohorts).ravel()
else:
loader = create_loader(X, y)
y_pred = get_output(model, loader).ravel()
# secondary mark for future change
if FLAGS.runname is not None:
fname += "_" + FLAGS.runname
joblib.dump((y_pred, y, cohorts),
'{}/logs'.format(FLAGS.result_dir) + '/results/' +
fname + FLAGS.result_suffix + '.pkl')
def testing(self, image_name, image_i, dataset_tests=False):
"""Método de teste da rede"""
mlp_input = None
image = None
# bias
bias_0 = self.bias_0
bias_1 = self.bias_1
table = 'treinamento'
if dataset_tests:
table = 'testes'
try:
self.cursor. execute(
'SELECT {} FROM {} WHERE image_name = "{}"'.format(self.descriptor,
table, image_name))
row = self.cursor.fetchone()
image = np.frombuffer(row[0])
except TypeError:
print('Erro na obtenção da matriz {} da imagem {} da tabela "testes"'.format(
self.descriptor, image_name))
print('Execução encerrada')
exit()
mlp_input = np.array(image.reshape(1, np.size(image)))
self.l0_neurons = len(mlp_input)
expected_output = np.array(u.get_output(image_name, self.part_2))
# print ("Test: {}\tImage: {}".format(str(image_i + 1).zfill(4),
# u.get_letter(image_name, self.part_2)))
layer_0 = mlp_input
layer_1 = self.activFunction(np.dot(layer_0, self.weights_0) + bias_0)
layer_2 = self.activFunction(np.dot(layer_1, self.weights_1) + bias_1).T
# se o teste estiver sendo rodado para o dataset de testes,
# armazena os acertos e erros para a matriz de confusao
if dataset_tests:
resulting_letter = u.get_resulting_letter(layer_2, self.part_2)
self.test_predicted.append(u.get_letter(image_name, self.part_2))
self.test_results.append(resulting_letter)
# erros: segunda camada
y_error = (expected_output - layer_2)
# erro quadrático médio da imagem
avg_y_error = np.sum(np.power(y_error, 2)) / 2
self.error_test_avg = self.error_test_avg + avg_y_error
self.test_number = self.test_number + 1
def create_snapshot_model(model_args):
"""
Create snapshot models with LSTM layer(s), shared dense layer(s), and sigmoided output.
model_args: a dictionary with the following keys
n_layers (int): Number of initial LSTM layers.
units (int): Number of units in each LSTM layer.
num_dense_shared_layers (int): Number of dense layers following LSTM layer(s).
dense_shared_layer_size (int): Number of units in each dense layer.
input_dim (int): Number of features in the input.
output_dim (int): Number of outputs (1 for binary tasks).
tasks (list): list of the tasks.
global_model_dir (str): directory in which each epoch's performance is saved
X_val: validation X (n, T, d)
y_val: validation y (n,)
cohorts_val: cluster assignment val (n,)
FLAGS: arguments in this file
Returns:
PyTorch model
"""
# similar to create_separate_model but with experts pretrained
# 1. get model directory path with models at each epoch for a global model
# 2. choose the model at epochs that gives best validation performance for each cohort
# as starting point
# 3. finetune the resulting model
tasks = model_args['tasks']
X_val, y_val, cohorts_val = model_args['X_val'], model_args['y_val'], model_args['cohorts_val']
val_loader = create_loader(X_val, y_val, batch_size=100, shuffle=False)
# convert y_val and cohorts_val to numpy
y_val, cohorts_val = dataset2numpy(y_val).astype(int), dataset2numpy(cohorts_val).astype(int)
experts_auc = [(None, 0) for _ in range(len(tasks))] # init to (n model, 0 auc)
for fn in glob.glob(model_args['global_model_dir'] + "/epoch*.m"):
net = torch.load(fn)
y_pred = get_output(net, val_loader).ravel()
for i, task in enumerate(tasks):
y_val_in_task = y_val[cohorts_val == task]
y_pred_in_task = y_pred[cohorts_val == task]
try:
auc = roc_auc_score(y_val_in_task, y_pred_in_task)
except:
auc = 0.1 # slightly larger than 0 but shouldn't be selected
if auc > experts_auc[i][1]:
experts_auc[i] = (net, auc)
experts = nn.ModuleList([expert for expert, auc in experts_auc])
# currently is inefficient by running all models for all tasks
# I should be able to just run the required expert
model = Separate_MIMIC_Model(experts)
return model
def pmt_importance(net, X_orig, y_orig, n_pmt=10, bs=None, device='cuda'):
'''
DEPRECATED: not used
X: numpy array of size (n, T, d)
y: numpy array of size (n,)
net: pytorch model
return feature importance array
'''
n, T, d = X_orig.shape
if bs == None or bs >= n:
bs = n
X, y = X_orig, y_orig
y_pred_orig = get_output(net, create_loader(X, y))
fps = 0
for _ in range(n_pmt):
if bs != n: # small sample to boost speed
indices = np.random.choice(len(X_orig), bs)
X, y = X_orig[indices], y_orig[indices]
y_pred_orig = get_output(net, create_loader(X, y))
fp = []
indices = np.random.choice(len(X), bs)
X_ = X[indices]
for i in tqdm.tqdm(range(d)):
X_p = copy.deepcopy(X)
X_p[:, :, i] = X_[:, :, i]
y_pred_pmt = get_output(net, create_loader(X_p, y))
fimp = y_pred_orig - y_pred_pmt # (bs, 1); no softmax b/c already after sigmoid
fp.append(fimp.ravel()) # fp: length d list of (bs,)
fp = np.vstack(fp).T # (bs, d)
fps += np.abs(fp)
return np.abs(fps / n_pmt).mean(0) # (d,)
def interface(self, data):
data_loader = self._get_data_loader(data)
all_uuids = []
all_labels = []
all_preds = []
all_probs = []
for batch in data_loader.batch_iter():
uuids = batch['uuids']
label_ids = batch['label_ids']
logits = self.model(batch)
label_ids, pred_ids, probs = decode(label_ids, logits)
all_uuids += uuids
all_labels += label_ids
all_preds += pred_ids
all_probs += probs
gold, output = get_output(all_uuids, all_labels, all_preds, all_probs)
return gold, output
def main(opt):
logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO)
for i, pref in enumerate(opt.prefixes):
ispec = MsInput.basedir_form(opt.base_dir, pref)
command = MaxrepInterface.command_from_dict(
dict(s_path=ispec.s_path, load_cst=opt.load_cst, answer=True))
res = get_output(command)[0].split()
with open(ispec.s_path) as fd:
sidx = FullIndex(fd.read().strip()[::-1])
for ((si, sj), substr, (i, j), is_max) in sidx.maxrep_iter():
LG.debug("s[%d:%d]=%s, I=[%d, %d], is_max = %s", si, sj, substr, i,
j, is_max)
bin_is_max = (res[i] == res[j - 1] == '1')
assert is_max == bin_is_max
def test_ping_h1_h5_and_h2_h5_balance_of_charges():
utils.get_output("wireshark/s2-eth1-h1-ping-h5.pcap",
"test/s2-ping-h1-h5.txt")
utils.get_output("wireshark/s2-eth1-h2-ping-h5.pcap",
"test/s2-ping-h2-h5.txt")
utils.get_output("wireshark/s3-eth1-h1-ping-h5.pcap",
"test/s3-ping-h1-h5.txt")
utils.get_output("wireshark/s3-eth1-h2-ping-h5.pcap",
"test/s3-ping-h2-h5.txt")
def callback(x):
return x['dst'] == '10.0.0.5' and x['protocol'] == 'ICMP'
df_1 = utils.get_df("test/s2-ping-h1-h5.txt")
df_1.columns = ['src', 'dst', 'protocol']
df_1 = utils.filter_by(df_1, ['dst', 'protocol'], callback)
size_1 = len(df_1.index)
df_2 = utils.get_df("test/s2-ping-h2-h5.txt")
df_2.columns = ['src', 'dst', 'protocol']
df_2 = utils.filter_by(df_2, ['dst', 'protocol'], callback)
size_2 = len(df_2.index)
df_3 = utils.get_df("test/s3-ping-h1-h5.txt")
df_3.columns = ['src', 'dst', 'protocol']
df_3 = utils.filter_by(df_3, ['dst', 'protocol'], callback)
size_3 = len(df_3.index)
df_4 = utils.get_df("test/s3-ping-h2-h5.txt")
df_4.columns = ['src', 'dst', 'protocol']
df_4 = utils.filter_by(df_4, ['dst', 'protocol'], callback)
size_4 = len(df_4.index)
os.system("rm test/s2-ping-h1-h5.txt")
os.system("rm test/s2-ping-h2-h5.txt")
os.system("rm test/s3-ping-h1-h5.txt")
os.system("rm test/s3-ping-h2-h5.txt")
ecmp_h1_h5 = (size_1 == 0 and size_3 != 0) or (size_1 != 0 and size_3 == 0)
ecmp_h2_h5 = (size_2 == 0 and size_4 != 0) or (size_2 != 0 and size_4 == 0)
assert ecmp_h1_h5 and ecmp_h2_h5
def main(opt):
logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO)
if opt.output != '/dev/stdout' and os.path.exists(opt.output):
LG.error("output file (%s) exsts. Exiting ...", opt.output)
return 1
command = MsInterface.command_from_dict(vars(opt))
pref = os.path.basename(opt.s_path).replace(".s", "")
header_suff = ",".join(['label', opt.repeat_colname, 'b_path'])
for j in range(opt.repeat):
with open(opt.output, 'a') as fd:
res = get_output(command)
if j == 0:
fd.write(res[0] + "," + header_suff + "\n")
for line in res[1:]:
fd.write(line.replace(" ", "") +
("," + opt.label) +
("," + str(j + 1)) +
("," + pref) + "\n")
def linear_search_plus_merge(file_path_P, file_path_T):
read_accesses = 0
write_accesses = 0
to_write = []
P = get_P(file_path_P)
P.sort()
file_T = get_T(file_path_T)
length_T = get_length_file(file_path_T)
output = get_output('output_files/output_linear_search_plus_merge.txt')
for iteration_index in range(0, length_T, B):
start_reading_from = LINE_SIZE * iteration_index
if iteration_index + B > length_T:
number_of_lines = length_T - iteration_index
else:
number_of_lines = B
lines = read_many_lines(start_reading_from, number_of_lines, file_T)
read_accesses += 1
index_P = 0
index_T = 0
while index_P < len(P) and index_T < len(lines):
element_P = P[index_P]
element_T = lines[index_T]
if element_P > element_T:
index_T += 1
elif element_P < element_T:
index_P += 1
else:
found_element = element_T.zfill(9) + '\n'
to_write.append(found_element)
index_P += 1
index_T += 1
if len(to_write) * LINE_SIZE >= BLOCK_SIZE:
output.write(''.join(to_write))
write_accesses += 1
to_write = []
file_T.close()
if (len(to_write) != 0):
output.write(''.join(to_write))
write_accesses += 1
output.close()
return read_accesses, write_accesses
def main():
print("hello world")
interpreter = make_interpreter("models/quantized_vae.tflite")
interpreter.allocate_tensors()
size = utils.input_size(interpreter)
print(size)
#image = cv2.imread("0.0722970757_0.223937735123.png", 0)
#image = cv2.resize(image, (200, 120), interpolation=cv2.INTER_AREA) # width(x), height(y)
#with open("test.jpg", "rb") as f:
# b = io.BytesIO(f.read())
image = Image.open('test.jpg')
image = image.convert('L') #.resize(size, Image.ANTIALIAS)
image = np.expand_dims(np.array(image).astype(np.float32), axis=2)
print(image[image > 0])
utils.set_input(interpreter, image)
image_x = utils.get_output(interpreter)
print(image_x[image_x > 0])
print(image_x.shape)
image_x = Image.fromarray(image_x.reshape(120, 200))
new_p = image_x.convert("L")
new_p.save('out.jpg')
def binary_search(path_p, path_t):
read_accesses = 0
write_accesses = 0
to_write = []
P = get_P(path_p)
ft = get_T(path_t)
output = get_output('output_files/output_binary.txt')
# We need to iterate over P loaded in memory
for p in P:
l = 0
h = get_length_file(path_t) - 1
m = 0
stop = False
while (l <= h and not stop):
m = (l + h) // 2
current_num = read_a_line_from_file(ft, m)
read_accesses += 1
if p < current_num:
h = m - 1
elif p > current_num:
l = m + 1
else:
num = current_num.zfill(9) + '\n'
to_write.append(num)
if len(to_write) * LINE_SIZE >= BLOCK_SIZE:
output.write(''.join(to_write))
write_accesses += 1
to_write = []
stop = True
ft.close()
if (len(to_write) != 0):
output.write(''.join(to_write))
write_accesses += 1
output.close()
return read_accesses, write_accesses
def _prepare_solution(self, identifier, configuration, mode='task'):
"""
Generate a working directory, configuration files and multiprocessing Process object to be ready to just run it.
:param identifier: Job or task identifier.
:param configuration: A dictionary with a cinfiguration or description.
:param mode: 'task' or 'job'.
:raise SchedulerException: Raised if the preparation fails and task or job cannot be scheduled.
"""
self.logger.info("Going to prepare execution of the {} {}".format(mode, identifier))
node_status = self._manager.node_info(self._node_name)
if mode == 'task':
subdir = 'tasks'
client_conf = self._get_task_configuration()
self._manager.check_resources(configuration, job=False)
else:
subdir = 'jobs'
client_conf = self._job_conf_prototype.copy()
self._manager.check_resources(configuration, job=True)
args = [self._client_bin, mode]
self._create_work_dir(subdir, identifier)
client_conf["Klever Bridge"] = self.conf["Klever Bridge"]
client_conf["identifier"] = identifier
work_dir = os.path.join(self.work_dir, subdir, identifier)
file_name = os.path.join(work_dir, 'client.json')
args.extend(['--file', file_name])
self._reserved[subdir][identifier] = dict()
# Check disk space limitation
if "keep working directory" in self.conf["scheduler"] and self.conf["scheduler"]["keep working directory"] and \
'disk memory size' in configuration["resource limits"] and \
configuration["resource limits"]['disk memory size']:
current_space = int(utils.get_output('du -bs {} | cut -f1'.format(work_dir)))
if current_space > configuration["resource limits"]['disk memory size']:
raise schedulers.SchedulerException(
"Clean manually existing working directory of {} since its size on the disk is {}B which is "
"greater than allowed limitation of {}B".
format(os.path.abspath(work_dir), current_space,
configuration["resource limits"]['disk memory size']))
if configuration["resource limits"].get("CPU time"):
# This is emergency timer if something will hang
timeout = int((configuration["resource limits"]["CPU time"] * 1.5) / 100)
else:
timeout = None
process = multiprocessing.Process(None, self._process_starter, identifier, [timeout, args])
if mode == 'task':
client_conf["Klever Bridge"] = self.conf["Klever Bridge"]
client_conf["identifier"] = identifier
client_conf["common"]["working directory"] = work_dir
for name in ("verifier", "upload input files of static verifiers"):
client_conf[name] = configuration[name]
# Speculative flag
if configuration.get('speculative'):
client_conf["speculative"] = True
# Do verification versions check
if client_conf['verifier']['name'] not in client_conf['client']['verification tools']:
raise schedulers.SchedulerException(
'Use another verification tool or install and then specify verifier {!r} with its versions at {!r}'.
format(client_conf['verifier']['name'], self.conf["scheduler"]["task client configuration"]))
if 'version' not in client_conf['verifier']:
raise schedulers.SchedulerException('Cannot find any given {!r} version at at task description'.
format(client_conf['verifier']['name']))
if client_conf['verifier']['version'] not in \
client_conf['client']['verification tools'][client_conf['verifier']['name']]:
raise schedulers.SchedulerException(
'Use another version of {!r} or install given version {!r} and specify it at scheduler client '
'configuration {!r}'.format(client_conf['verifier']['name'], client_conf['verifier']['version'],
self.conf["scheduler"]["task client configuration"]))
self._task_processes[identifier] = process
else:
klever_core_conf = configuration.copy()
del klever_core_conf["resource limits"]
klever_core_conf["Klever Bridge"] = self.conf["Klever Bridge"]
klever_core_conf["working directory"] = "klever-core-work-dir"
self._reserved["jobs"][identifier]["configuration"] = klever_core_conf
client_conf["common"]["working directory"] = work_dir
client_conf["Klever Core conf"] = self._reserved["jobs"][identifier]["configuration"]
self._job_processes[identifier] = process
client_conf["resource limits"] = configuration["resource limits"]
# Add particular cores
if "resource limits" not in client_conf:
client_conf["resource limits"] = {}
client_conf["resource limits"]["CPU cores"] = \
self._get_virtual_cores(int(node_status["available CPU number"]),
int(node_status["reserved CPU number"]),
int(configuration["resource limits"]["number of CPU cores"]))
if mode != "task":
if len(client_conf["resource limits"]["CPU cores"]) == 0:
data = utils.extract_cpu_cores_info()
client_conf["Klever Core conf"]["task resource limits"]["CPU Virtual cores"] = \
sum((len(data[a]) for a in data))
else:
client_conf["Klever Core conf"]["task resource limits"]["CPU Virtual cores"] = \
len(client_conf["resource limits"]["CPU cores"])
# Save Klever Core configuration to default configuration file
with open(os.path.join(work_dir, "core.json"), "w", encoding="utf8") as fh:
json.dump(client_conf["Klever Core conf"], fh, ensure_ascii=False, sort_keys=True, indent=4)
with open(file_name, 'w', encoding="utf8") as fp:
json.dump(client_conf, fp, ensure_ascii=False, sort_keys=True, indent=4)
def evaluation(model, model_name, X, y, cohorts, all_tasks, FLAGS):
'''
model: pytorch model
X: (n, d) pytorch dataset
y: (n,) pytorch dataset
cohorts: (n,) pytorch dataset
all_tasks (Numpy array/list): List of tasks
'''
cohort_aucs = []
loader = create_loader(X, y)
if 'mtl' in model_name:
y_pred = get_output_mtl(model, X, y, cohorts).ravel()
else:
y_pred = get_output(model, loader).ravel()
y = dataset2numpy(y).astype(int)
cohorts = dataset2numpy(cohorts).astype(int)
# all bootstrapped AUCs
for task in all_tasks:
y_pred_in_cohort = y_pred[cohorts == task]
y_in_cohort = y[cohorts == task]
if FLAGS.bootstrap:
all_aucs = bootstrap_metric(y_in_cohort, y_pred_in_cohort,
FLAGS.num_bootstrap_samples, roc_auc_score)
cohort_aucs.append(np.array(all_aucs))
min_auc, max_auc, avg_auc = np.min(all_aucs), np.max(all_aucs), np.mean(all_aucs)
print('{} Model AUC on {}: [min: {}, max: {}, avg: {}]'.format(model_name,
task, min_auc,
max_auc, avg_auc))
else:
auc = roc_auc_score(y_in_cohort, y_pred_in_cohort)
cohort_aucs.append(auc)
print('{} Model AUC on {}: {}'.format(model_name, task, cohort_aucs[-1]))
if FLAGS.bootstrap:
# Macro AUC
cohort_aucs = np.array(cohort_aucs)
cohort_aucs = np.concatenate(
(cohort_aucs, np.expand_dims(np.mean(cohort_aucs, axis=0), 0)))
all_aucs = cohort_aucs[-1]
min_auc, max_auc, avg_auc = np.min(all_aucs), np.max(all_aucs), np.mean(all_aucs)
print('{} Model AUC Macro: [min: {}, max: {}, avg: {}]'.format(model_name,
min_auc, max_auc, avg_auc))
# Micro AUC
all_micro_aucs = bootstrap_metric(y, y_pred,
FLAGS.num_bootstrap_samples,
roc_auc_score)
cohort_aucs = np.concatenate(
(cohort_aucs, np.array([all_micro_aucs])))
all_aucs = cohort_aucs[-1]
min_auc, max_auc, avg_auc = np.min(all_aucs), np.max(all_aucs), np.mean(all_aucs)
print('{} Model AUC Micro: [min: {}, max: {}, avg: {}]'.format(model_name,
min_auc, max_auc, avg_auc))
else:
# Macro AUC
macro_auc = np.mean(cohort_aucs)
cohort_aucs.append(macro_auc)
print('{} Model AUC Macro: {}'.format(model_name, cohort_aucs[-1]))
# Micro AUC
micro_auc = roc_auc_score(y, y_pred)
cohort_aucs.append(micro_auc)
print('{} Model AUC Micro: {}'.format(model_name, cohort_aucs[-1]))
return cohort_aucs
def get_rev():
return utils.get_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=BASE_DIR)
SPEC = os.path.join(BASE_DIR, 'win', 'rednotebook.spec')
WINE_BUILD = os.path.join(DRIVE_C, 'build')
WINE_DIST = os.path.join(DRIVE_C, 'dist')
LOCALE_DIR = os.path.join(WINE_DIST, 'share', 'locale')
WINE_RN_EXE = os.path.join(WINE_DIST, 'rednotebook.exe')
WINE_PYTHON = os.path.join(DRIVE_C, 'Python34', 'python.exe')
utils.confirm_overwrite(DIST_DIR)
os.environ['WINEPREFIX'] = DIST_DIR
utils.ensure_path(os.path.dirname(DIST_DIR))
print('Start copying {} to {}'.format(BUILD_DIR, DIST_DIR))
utils.fast_copytree(BUILD_DIR, DIST_DIR)
print('Finished copying')
archive = '/tmp/rednotebook-archive.tar'
stash_name = utils.get_output(['git', 'stash', 'create'], cwd=BASE_DIR)
stash_name = stash_name or 'HEAD'
run(['git', 'archive', stash_name, '-o', archive], cwd=BASE_DIR)
utils.ensure_path(WINE_RN_DIR)
run(['tar', '-xf', archive], cwd=WINE_RN_DIR)
os.mkdir(os.path.join(DRIVE_C, 'Python34/share'))
shutil.copytree(os.path.join(DRIVE_C, 'Python34/Lib/site-packages/gnome/share/gir-1.0/'), os.path.join(DRIVE_C, 'Python34/share/gir-1.0/'))
run(['wine', WINE_PYTHON, '-m', 'PyInstaller', '--workpath', WINE_BUILD,
'--distpath', DRIVE_C, SPEC]) # will be built at ...DRIVE_C/dist
run(['python3', 'build-translations.py', LOCALE_DIR], cwd=DIR)
if args.test:
run(['wine', WINE_RN_EXE])
def training(self, image_name, image_i, epoch, fold_num):
"""Método de treinamento da rede"""
mlp_input = None
image = None
try:
self.cursor. execute(
'SELECT {} FROM treinamento WHERE image_name = "{}"'.format(self.descriptor,
image_name))
row = self.cursor.fetchone()
image = np.frombuffer(row[0])
except TypeError:
print('Erro na obtenção da matriz {} da imagem {} da tabela "treinamento"'.format(
self.descriptor, image_name))
print('Execução encerrada')
exit()
# camada de entrada: preparação
mlp_input = np.array(image.reshape(1, np.size(image)))
self.l0_neurons = len(mlp_input)
expected_output = np.array(u.get_output(image_name, self.part_2))
# pesos: preparação dos pesos para esta execução
weights_0 = self.weights_0
weights_1 = self.weights_1
# bias
bias_0 = self.bias_0
bias_1 = self.bias_1
# feed forward
layer_0 = mlp_input # 1x576 (HOG)
layer_1 = self.activFunction(np.dot(layer_0, weights_0) + bias_0) # ->1x32 (1x576 por 576x32)
layer_2 = self.activFunction(np.dot(layer_1, weights_1) + bias_1).T # ->1X3 (1x32 por 32x3) (T)-> 3x1
y_error = (expected_output - layer_2) # 3x1 - 3x1 = 3x1
# erro quadrático médio de uma imagem
avg_y_error = np.sum(np.power(y_error, 2)) / 2
self.error_training_avg = self.error_training_avg + avg_y_error
self.training_number = self.training_number + 1
# erros: segunda camada
y_error = np.array(y_error) * self.derivative(np.array(layer_2)) # 3x1 (3x1 por 3x1)
y_error = y_error.T # 1x3
y_delta = self.alpha * layer_1.T.dot(y_error) # 32x3 (32x1 por 1x3)
bias_1_delta = self.alpha * y_error # 1x3
# erros: repasse para a camada escondida
z_error = y_error.dot(weights_1.T) # 1x32 (1x3 por 3x32)
z_error = z_error * self.derivative(np.array(layer_1)) # 1x32 (1x32 por 1x32)
z_delta = self.alpha * layer_0.T.dot(z_error) # 576x32 (576x1(HOG) por 1x32)
bias_0_delta = self.alpha * z_error
# pesos e bias: atualização
weights_1 += y_delta
weights_0 += z_delta
bias_1_delta += bias_1
bias_0_delta += bias_0
self.weights_1 = weights_1
self.weights_0 = weights_0
self.bias_1 = bias_1_delta
self.bias_0 = bias_0_delta
lr = learning_rate(args.lr, epoch)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# label-correction
if epoch > args.correction:
train_eval_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=256,
shuffle=False,
**kwargs)
noisy_targets = np.eye(args.num_class)[train_dataset.targets]
log(logpath, 'Updating labels.\n')
args.sigma = 0
output, losses = get_output(ema_net, device, train_eval_loader)
losses = (losses - min(losses)) / (max(losses) - min(losses)
) # normalize to [0,1]
losses = losses.reshape([len(losses), 1])
targets = losses * noisy_targets + (1 - losses) * output
train_dataset.targets = targets
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
**kwargs)
_, train_acc = train_noise(args, net, device, train_loader, optimizer,
epoch, ema_optimizer)
_, test_acc_NoEMA = test(args, net, device, test_loader)
_, val_acc = test(args, ema_net, device, val_loader)
SPEC = os.path.join(BASE_DIR, 'win', 'rednotebook.spec')
WINE_BUILD = os.path.join(DRIVE_C, 'build')
WINE_DIST = os.path.join(DRIVE_C, 'dist')
LOCALE_DIR = os.path.join(WINE_DIST, 'share', 'locale')
WINE_RN_EXE = os.path.join(WINE_DIST, 'rednotebook.exe')
WINE_PYTHON = os.path.join(DRIVE_C, 'Python34', 'python.exe')
utils.confirm_overwrite(DIST_DIR)
os.environ['WINEPREFIX'] = DIST_DIR
utils.ensure_path(os.path.dirname(DIST_DIR))
print('Start copying {} to {}'.format(BUILD_DIR, DIST_DIR))
utils.fast_copytree(BUILD_DIR, DIST_DIR)
print('Finished copying')
archive = '/tmp/rednotebook-archive.tar'
stash_name = utils.get_output(['git', 'stash', 'create'], cwd=BASE_DIR)
stash_name = stash_name or 'HEAD'
run(['git', 'archive', stash_name, '-o', archive], cwd=BASE_DIR)
utils.ensure_path(WINE_RN_DIR)
run(['tar', '-xf', archive], cwd=WINE_RN_DIR)
os.mkdir(os.path.join(DRIVE_C, 'Python34/share'))
shutil.copytree(
os.path.join(DRIVE_C, 'Python34/Lib/site-packages/gnome/share/gir-1.0/'),
os.path.join(DRIVE_C, 'Python34/share/gir-1.0/'))
run([
'wine', WINE_PYTHON, '-m', 'PyInstaller', '--workpath', WINE_BUILD,
'--distpath', DRIVE_C, SPEC
]) # will be built at ...DRIVE_C/dist
run(['python3', 'build-translations.py', LOCALE_DIR], cwd=DIR)
def __extract_description(self, solution_dir):
"""
Get directory with BenchExec output and extract results from there saving them to JSON file according to
provided path.
:param solution_dir: Path with BenchExec output.
:return: Identifier string of the solution.
"""
identifier = str(uuid.uuid4())
description = {"id": identifier, "resources": {}, "comp": {}}
# Import description
desc_file = os.path.join(solution_dir, "runDescription.txt")
self.logger.debug(
"Import description from the file {}".format(desc_file))
description["desc"] = ""
if os.path.isfile(desc_file):
with open(desc_file, encoding="utf8") as di:
for line in di:
key, value = line.strip().split("=")
if key == "tool":
description["desc"] += value
elif key == "revision":
description["desc"] += " {}".format(value)
else:
raise FileNotFoundError(
"There is no solution file {}".format(desc_file))
# Import general information
general_file = os.path.join(solution_dir, "runInformation.txt")
self.logger.debug(
"Import general information from the file {}".format(general_file))
termination_reason = None
number = re.compile("(\d.*\d)")
if os.path.isfile(general_file):
with open(general_file, encoding="utf8") as gi:
for line in gi:
key, value = line.strip().split("=", maxsplit=1)
if key == "terminationreason":
termination_reason = value
elif key == "command":
description["comp"]["command"] = value
elif key == "exitsignal":
description["signal num"] = int(value)
elif key == "exitcode":
description["return value"] = int(value)
elif key == "walltime":
sec = number.match(value).group(1)
if sec:
description["resources"]["wall time"] = int(
float(sec) * 1000)
else:
self.logger.warning(
"Cannot properly extract wall time from {}".
format(general_file))
elif key == "cputime":
sec = number.match(value).group(1)
if sec:
description["resources"]["CPU time"] = int(
float(sec) * 1000)
else:
self.logger.warning(
"Cannot properly extract CPU time from {}".
format(general_file))
elif key == "memory":
mem_bytes = number.match(value).group(1)
if mem_bytes:
description["resources"]["memory size"] = int(
mem_bytes)
else:
self.logger.warning(
"Cannot properly extract exhausted memory from {}"
.format(general_file))
elif key == "coreLimit":
cores = int(value)
description["resources"]["coreLimit"] = cores
else:
raise FileNotFoundError(
"There is no solution file {}".format(general_file))
# Set final status
if termination_reason:
if termination_reason == "cputime":
description["status"] = "TIMEOUT"
elif termination_reason == "memory":
description["status"] = 'OUT OF MEMORY'
else:
raise ValueError("Unsupported termination reason {}".format(
termination_reason))
elif "signal num" in description:
description["status"] = "killed by signal"
elif "return value" in description:
if description["return value"] == 0:
if glob.glob(
os.path.join(solution_dir, "output",
"witness.*.graphml")):
description["status"] = "false"
else:
# Check that soft limit has not activated
status_checker = 'grep -F "Verification result: UNKNOWN" -m 1 -c {}'.\
format(os.path.join(solution_dir, "output", "benchmark.logfiles", "*.log"))
number = int(utils.get_output(status_checker))
if number > 0:
description["status"] = "unknown"
else:
description["status"] = "true"
else:
description["status"] = "unknown"
else:
raise ValueError(
"Cannot determine termination reason according to the file {}".
format(general_file))
# Import Host information
host_file = os.path.join(solution_dir, "hostInformation.txt")
self.logger.debug(
"Import host information from the file {}".format(host_file))
lv_re = re.compile("Linux\s(\d.*)")
if os.path.isfile(host_file):
with open(host_file, encoding="utf8") as hi:
for line in hi:
key, value = line.strip().split("=", maxsplit=1)
if key == "name":
description["comp"]["node name"] = value
elif key == "os":
version = lv_re.match(value).group(1)
if version:
description["comp"][
"Linux kernel version"] = version
else:
self.logger.warning(
"Cannot properly extract Linux kernel version from {}"
.format(host_file))
elif key == "memory":
description["comp"]["mem size"] = value
elif key == "cpuModel":
description["comp"]["CPU model"] = value
elif key == "cores":
description["comp"]["number of CPU cores"] = value
else:
raise FileNotFoundError(
"There is no solution file {}".format(host_file))
return identifier, description
def get_rev():
return utils.get_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=BASE_DIR)
