def train_global(cluster_args): ''' 1. apply GMM or kmeans on the repr after lstm returns cluster_predictions (type:int): shape (n,) ''' X = cluster_args['X'] y = cluster_args['y'] X_train = cluster_args['X_train'] y_train = cluster_args['y_train'] global_model_fn = cluster_args['global_model_fn'] FLAGS = cluster_args['FLAGS'] global_model = torch.load(global_model_fn) global_model.cuda() def extract_embedding(x): '''x assumes to be pytorch tensor''' o, (h, c) = global_model.lstm(x) embeddings = h[-1] return embeddings # Get Embeddings: embedded_train = get_output(global_model, create_loader(X_train, y_train)) embedded_all = get_output(global_model, create_loader(X, y)) return gmm_fit_and_predict(embedded_train, embedded_all, FLAGS, savename_suffix="_global")
def decode(self, latent): ''' Decode the latent variables and generate output. ''' pose, appearance = latent['pose'].view( -1, self.pose_latent_size), latent['appearance'].view( -1, self.appearance_latent_size) objects = self.object_decoder(appearance) components = utils.object_to_image(objects, pose, self.image_size) components = components.view(-1, self.n_frames_total, self.total_components, self.n_channels, self.image_size, self.image_size) masks = latent['mask'] if masks is not None: masked_components = components[:, :self. n_frames_input] * masks.unsqueeze( -1).unsqueeze(-1) masked_output = utils.get_output(masked_components) output = utils.get_output(components) return output, masked_output, components
def main(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) variants = utils.make_variants(args.python_versions, args.build_types, args.mpi_types, args.cuda_versions) pr_branch = utils.get_output("git log -1 --format='%H'") utils.run_and_log("git remote set-head origin -a") default_branch = utils.get_output( "git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'" ) variant_build_results = dict() for variant in variants: utils.run_and_log("git checkout {}".format(default_branch)) master_build_config_data, master_config = _get_configs(variant) master_build_numbers = _get_build_numbers(master_build_config_data, master_config, variant) utils.run_and_log("git checkout {}".format(pr_branch)) pr_build_config_data, pr_config = _get_configs(variant) current_pr_build_numbers = _get_build_numbers(pr_build_config_data, pr_config, variant) print("Build Info for Variant: {}".format(variant)) print("Current PR Build Info: {}".format(current_pr_build_numbers)) print("Master Branch Build Info: {}".format(master_build_numbers)) #No build numbers can go backwards without a version change. for package in master_build_numbers: if package in current_pr_build_numbers and current_pr_build_numbers[ package]["version"] == master_build_numbers[package][ "version"]: assert int(current_pr_build_numbers[package]["number"]) >= int( master_build_numbers[package]["number"] ), "If the version doesn't change, the build number can't be reduced." #If packages are added or removed, don't require a version change if set(master_build_numbers.keys()) != set( current_pr_build_numbers.keys()): return #At least one package needs to increase the build number or change the version. checks = [ current_pr_build_numbers[package]["version"] != master_build_numbers[package]["version"] or int(current_pr_build_numbers[package]["number"]) > int( master_build_numbers[package]["number"]) for package in master_build_numbers ] variant_build_results[utils.variant_string( variant["python"], variant["build_type"], variant["mpi_type"], variant["cudatoolkit"])] = any(checks) assert any( variant_build_results.values() ), "At least one package needs to increase the build number or change the version in at least one variant."
def train_val_curve(cluster_args): # 1. learn validation curve: save as snapshot; reuse the code # 2. train from x to validation curve cluster # 3. apply this on Train, val and test to save X = cluster_args['X'] y = cluster_args['y'] X_train = cluster_args['X_train'] y_train = cluster_args['y_train'] X_val = cluster_args['X_val'] y_val = cluster_args['y_val'] global_model_dir = cluster_args['global_model_dir'] global_model_fn = cluster_args['global_model_fn'] FLAGS = cluster_args['FLAGS'] def sorted_by_epoch(l): return sorted(l, key=lambda s: int(re.search("epoch(.*)\.m", s).group(1))) curves = [] val_loader = create_loader(X_val, y_val, batch_size=64) y_val = dataset2numpy(y_val) criterion = nn.BCELoss(reduction='none') for fn in sorted_by_epoch(glob.glob(global_model_dir + "/epoch*.m")): net = torch.load(fn) y_pred_val = get_output(net, val_loader).ravel() # (n_val,) # collect the loss curves.append(-criterion( torch.from_numpy(y_pred_val), torch.from_numpy(y_val)).detach().cpu().numpy()) # (epochs, n_val) curves = np.array(curves).T # (n_val, epochs) # joblib.dump(curves, 'val_curves.pkl') # train a mapping from input to val_curve k = FLAGS.num_clusters assignment, experts_epochs = val_curve_kmeans(curves, k=k, niters=10) if FLAGS.not_pt: net = None else: net = torch.load(global_model_fn) net.rest[-2] = nn.Linear(net.rest[-2].in_features, k).cuda() net.rest = net.rest[:-1] # drop sigmoid layer gate = train_assignment(FLAGS, k, assignment, X_val, net=net, savename_suffix="_val_curve") # Get cluster membership: from (n, k) -> (n,) cluster_preds = get_output(gate, create_loader(X, y)).argmax(1) return cluster_preds
def test_udp_denial_of_service(): utils.get_output("wireshark/s5-eth1-iperf-udp-h1-h5.pcap", "test/s5-udp-h1-h5.txt") def callback(x): return x['dst'] == '10.0.0.5' and x['protocol'] == 'UDP' df_1 = utils.get_df("test/s5-udp-h1-h5.txt") df_1.columns = ['src', 'dst', 'protocol'] df_1 = utils.filter_by(df_1, ['dst', 'protocol'], callback) size_1 = len(df_1.index) assert size_1 < 893 os.system("rm test/s5-udp-h1-h5.txt")
def main(opt): logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO) for pref in opt.prefixes: ms_input = MsInput.basedir_form(opt.base_dir, pref) LG.info("testing on input %s", ms_input) res1 = get_output(fast(opt, ms_input)) res2 = get_output(slow(opt.slow_prg, opt.slow_load_dir, ms_input)) err_lst = check_res(res1, res2) if any(err_lst): for err in err_lst: if err: print "\t" + err print
def main(opt): logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO) base_dict = dict(lazy_wl=False, rank_fail=True, use_maxrep=opt.use_maxrep, load_maxrep=opt.load_maxrep, load_cst=opt.load_cst, space_usage=True, time_usage=True, answer=False) for i, pref in enumerate(opt.prefixes): ispec = MsInput.basedir_form(opt.base_dir, pref) command = MsInterface.command_from_dict( dict(s_path=ispec.s_path, t_path=ispec.t_path, **base_dict)) for j in range(opt.repeat): with open(opt.output, 'a') as fd: res = get_output(command) if i + j == 0: fd.write(res[0] + ",label,b_path\n") for line in res[1:]: fd.write( line.replace(" ", "") + ("," + opt.label) + ("," + pref) + "\n")
def linear_search(file_path_P, file_path_T): read_accesses = 0 write_accesses = 0 to_write = [] P = get_P(file_path_P) file_T = get_T(file_path_T) length_T = get_length_file(file_path_T) output = get_output('output_files/output_linear.txt') for iteration_index in range(0, length_T, B): start_reading_from = LINE_SIZE * iteration_index if(iteration_index + B > length_T): number_of_lines = length_T - iteration_index else: number_of_lines = B lines = read_many_lines(start_reading_from, number_of_lines, file_T) read_accesses += 1 for element in lines: if element in P: found_element = element.zfill(9) + '\n' to_write.append(found_element) if len(to_write) * LINE_SIZE >= BLOCK_SIZE: output.write(''.join(to_write)) write_accesses += 1 to_write = [] file_T.close() if(len(to_write) != 0): output.write(''.join(to_write)) write_accesses += 1 output.close() return read_accesses, write_accesses
def _eval_model(self, dev_data_loader): all_uuids = [] all_labels = [] all_preds = [] all_probs = [] loss_vals = [] for batch in dev_data_loader.batch_iter(): uuids = batch['uuids'] label_ids, pred_ids, probs, loss_val = self._get_preds(batch) all_uuids += uuids all_labels += label_ids all_preds += pred_ids all_probs += probs loss_vals.append(loss_val) print(len(all_labels)) print(len(all_preds)) gold, output = get_output(all_uuids, all_labels, all_preds, all_probs) score = get_hit_score(gold, output) #score = scorer(all_labels, all_preds) loss_val = sum(loss_vals) / len(loss_vals) return score, loss_val
def indexed_search(path_p, path_t): read_accesses = 0 write_accesses = 0 P = get_P(path_p) T = get_T(path_t) out = get_output('output_files/output_indexed.txt') S = [] to_write = [] len_t = get_length_file(path_t) n_blocks = math.ceil(len_t / B) for i in range(n_blocks): line_str = read_a_line_from_file(T, i * B) read_accesses += 1 S.append(line_str) for searched in P: s_interval_id = binary_search_modified(S, searched, len(S) - 1) if s_interval_id != -1: start_reading_at = LINE_SIZE * s_interval_id * B many_lines = read_many_lines(start_reading_at, B, T) read_accesses += 1 for block_number in many_lines: if block_number != '' and searched == block_number: num = searched.zfill(9) + '\n' to_write.append(num) if len(to_write) * LINE_SIZE >= BLOCK_SIZE: out.write(''.join(to_write)) write_accesses += 1 to_write = [] T.close() if (len(to_write) != 0): out.write(''.join(to_write)) write_accesses += 1 out.close() return read_accesses, write_accesses
def test_tcp_balance_of_charges(): utils.get_output("wireshark/s2-eth1-iperf-h1-h4.pcap", "test/s2-h1-h4.txt") utils.get_output("wireshark/s3-eth1-iperf-h1-h4.pcap", "test/s3-h1-h4.txt") def callback(x): return x['dst'] == '10.0.0.4' and x['protocol'] == 'TCP' df_1 = utils.get_df("test/s2-h1-h4.txt") df_1.columns = ['src', 'dst', 'protocol'] df_1 = utils.filter_by(df_1, ['dst', 'protocol'], callback) size_1 = len(df_1.index) df_2 = utils.get_df("test/s3-h1-h4.txt") df_2.columns = ['src', 'dst', 'protocol'] df_2 = utils.filter_by(df_2, ['dst', 'protocol'], callback) size_2 = len(df_2.index) os.system("rm test/s2-h1-h4.txt") os.system("rm test/s3-h1-h4.txt") assert (size_1 == 0 and size_2 != 0) or (size_1 != 0 and size_2 == 0)
def train_ae_pytorch(cluster_args): ''' pytorch version of train ae returns cluster_predictions (type:int): shape (n,) ''' X = cluster_args['X'] X_train = cluster_args['X_train'] X_val = cluster_args['X_val'] FLAGS = cluster_args['FLAGS'] # Train autoencoder encoder = train_seq_ae_pytorch(X_train, X_val, FLAGS) # Get Embeddings embedded_train = get_output(encoder, create_loader(X_train, X_train, batch_size=64)) embedded_all = get_output(encoder, create_loader(X, X, batch_size=64)) return gmm_fit_and_predict(embedded_train, embedded_all, FLAGS, savename_suffix="_ae")
def save_output(model, model_name, X, y, cohorts, all_tasks, fname, FLAGS): if 'mtl' in model_name: y_pred = get_output_mtl(model, X, y, cohorts).ravel() else: loader = create_loader(X, y) y_pred = get_output(model, loader).ravel() # secondary mark for future change if FLAGS.runname is not None: fname += "_" + FLAGS.runname joblib.dump((y_pred, y, cohorts), '{}/logs'.format(FLAGS.result_dir) + '/results/' + fname + FLAGS.result_suffix + '.pkl')
def testing(self, image_name, image_i, dataset_tests=False): """Método de teste da rede""" mlp_input = None image = None # bias bias_0 = self.bias_0 bias_1 = self.bias_1 table = 'treinamento' if dataset_tests: table = 'testes' try: self.cursor. execute( 'SELECT {} FROM {} WHERE image_name = "{}"'.format(self.descriptor, table, image_name)) row = self.cursor.fetchone() image = np.frombuffer(row[0]) except TypeError: print('Erro na obtenção da matriz {} da imagem {} da tabela "testes"'.format( self.descriptor, image_name)) print('Execução encerrada') exit() mlp_input = np.array(image.reshape(1, np.size(image))) self.l0_neurons = len(mlp_input) expected_output = np.array(u.get_output(image_name, self.part_2)) # print ("Test: {}\tImage: {}".format(str(image_i + 1).zfill(4), # u.get_letter(image_name, self.part_2))) layer_0 = mlp_input layer_1 = self.activFunction(np.dot(layer_0, self.weights_0) + bias_0) layer_2 = self.activFunction(np.dot(layer_1, self.weights_1) + bias_1).T # se o teste estiver sendo rodado para o dataset de testes, # armazena os acertos e erros para a matriz de confusao if dataset_tests: resulting_letter = u.get_resulting_letter(layer_2, self.part_2) self.test_predicted.append(u.get_letter(image_name, self.part_2)) self.test_results.append(resulting_letter) # erros: segunda camada y_error = (expected_output - layer_2) # erro quadrático médio da imagem avg_y_error = np.sum(np.power(y_error, 2)) / 2 self.error_test_avg = self.error_test_avg + avg_y_error self.test_number = self.test_number + 1
def create_snapshot_model(model_args): """ Create snapshot models with LSTM layer(s), shared dense layer(s), and sigmoided output. model_args: a dictionary with the following keys n_layers (int): Number of initial LSTM layers. units (int): Number of units in each LSTM layer. num_dense_shared_layers (int): Number of dense layers following LSTM layer(s). dense_shared_layer_size (int): Number of units in each dense layer. input_dim (int): Number of features in the input. output_dim (int): Number of outputs (1 for binary tasks). tasks (list): list of the tasks. global_model_dir (str): directory in which each epoch's performance is saved X_val: validation X (n, T, d) y_val: validation y (n,) cohorts_val: cluster assignment val (n,) FLAGS: arguments in this file Returns: PyTorch model """ # similar to create_separate_model but with experts pretrained # 1. get model directory path with models at each epoch for a global model # 2. choose the model at epochs that gives best validation performance for each cohort # as starting point # 3. finetune the resulting model tasks = model_args['tasks'] X_val, y_val, cohorts_val = model_args['X_val'], model_args['y_val'], model_args['cohorts_val'] val_loader = create_loader(X_val, y_val, batch_size=100, shuffle=False) # convert y_val and cohorts_val to numpy y_val, cohorts_val = dataset2numpy(y_val).astype(int), dataset2numpy(cohorts_val).astype(int) experts_auc = [(None, 0) for _ in range(len(tasks))] # init to (n model, 0 auc) for fn in glob.glob(model_args['global_model_dir'] + "/epoch*.m"): net = torch.load(fn) y_pred = get_output(net, val_loader).ravel() for i, task in enumerate(tasks): y_val_in_task = y_val[cohorts_val == task] y_pred_in_task = y_pred[cohorts_val == task] try: auc = roc_auc_score(y_val_in_task, y_pred_in_task) except: auc = 0.1 # slightly larger than 0 but shouldn't be selected if auc > experts_auc[i][1]: experts_auc[i] = (net, auc) experts = nn.ModuleList([expert for expert, auc in experts_auc]) # currently is inefficient by running all models for all tasks # I should be able to just run the required expert model = Separate_MIMIC_Model(experts) return model
def pmt_importance(net, X_orig, y_orig, n_pmt=10, bs=None, device='cuda'): ''' DEPRECATED: not used X: numpy array of size (n, T, d) y: numpy array of size (n,) net: pytorch model return feature importance array ''' n, T, d = X_orig.shape if bs == None or bs >= n: bs = n X, y = X_orig, y_orig y_pred_orig = get_output(net, create_loader(X, y)) fps = 0 for _ in range(n_pmt): if bs != n: # small sample to boost speed indices = np.random.choice(len(X_orig), bs) X, y = X_orig[indices], y_orig[indices] y_pred_orig = get_output(net, create_loader(X, y)) fp = [] indices = np.random.choice(len(X), bs) X_ = X[indices] for i in tqdm.tqdm(range(d)): X_p = copy.deepcopy(X) X_p[:, :, i] = X_[:, :, i] y_pred_pmt = get_output(net, create_loader(X_p, y)) fimp = y_pred_orig - y_pred_pmt # (bs, 1); no softmax b/c already after sigmoid fp.append(fimp.ravel()) # fp: length d list of (bs,) fp = np.vstack(fp).T # (bs, d) fps += np.abs(fp) return np.abs(fps / n_pmt).mean(0) # (d,)
def interface(self, data): data_loader = self._get_data_loader(data) all_uuids = [] all_labels = [] all_preds = [] all_probs = [] for batch in data_loader.batch_iter(): uuids = batch['uuids'] label_ids = batch['label_ids'] logits = self.model(batch) label_ids, pred_ids, probs = decode(label_ids, logits) all_uuids += uuids all_labels += label_ids all_preds += pred_ids all_probs += probs gold, output = get_output(all_uuids, all_labels, all_preds, all_probs) return gold, output
def main(opt): logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO) for i, pref in enumerate(opt.prefixes): ispec = MsInput.basedir_form(opt.base_dir, pref) command = MaxrepInterface.command_from_dict( dict(s_path=ispec.s_path, load_cst=opt.load_cst, answer=True)) res = get_output(command)[0].split() with open(ispec.s_path) as fd: sidx = FullIndex(fd.read().strip()[::-1]) for ((si, sj), substr, (i, j), is_max) in sidx.maxrep_iter(): LG.debug("s[%d:%d]=%s, I=[%d, %d], is_max = %s", si, sj, substr, i, j, is_max) bin_is_max = (res[i] == res[j - 1] == '1') assert is_max == bin_is_max
def test_ping_h1_h5_and_h2_h5_balance_of_charges(): utils.get_output("wireshark/s2-eth1-h1-ping-h5.pcap", "test/s2-ping-h1-h5.txt") utils.get_output("wireshark/s2-eth1-h2-ping-h5.pcap", "test/s2-ping-h2-h5.txt") utils.get_output("wireshark/s3-eth1-h1-ping-h5.pcap", "test/s3-ping-h1-h5.txt") utils.get_output("wireshark/s3-eth1-h2-ping-h5.pcap", "test/s3-ping-h2-h5.txt") def callback(x): return x['dst'] == '10.0.0.5' and x['protocol'] == 'ICMP' df_1 = utils.get_df("test/s2-ping-h1-h5.txt") df_1.columns = ['src', 'dst', 'protocol'] df_1 = utils.filter_by(df_1, ['dst', 'protocol'], callback) size_1 = len(df_1.index) df_2 = utils.get_df("test/s2-ping-h2-h5.txt") df_2.columns = ['src', 'dst', 'protocol'] df_2 = utils.filter_by(df_2, ['dst', 'protocol'], callback) size_2 = len(df_2.index) df_3 = utils.get_df("test/s3-ping-h1-h5.txt") df_3.columns = ['src', 'dst', 'protocol'] df_3 = utils.filter_by(df_3, ['dst', 'protocol'], callback) size_3 = len(df_3.index) df_4 = utils.get_df("test/s3-ping-h2-h5.txt") df_4.columns = ['src', 'dst', 'protocol'] df_4 = utils.filter_by(df_4, ['dst', 'protocol'], callback) size_4 = len(df_4.index) os.system("rm test/s2-ping-h1-h5.txt") os.system("rm test/s2-ping-h2-h5.txt") os.system("rm test/s3-ping-h1-h5.txt") os.system("rm test/s3-ping-h2-h5.txt") ecmp_h1_h5 = (size_1 == 0 and size_3 != 0) or (size_1 != 0 and size_3 == 0) ecmp_h2_h5 = (size_2 == 0 and size_4 != 0) or (size_2 != 0 and size_4 == 0) assert ecmp_h1_h5 and ecmp_h2_h5
def main(opt): logging.getLogger().setLevel(logging.DEBUG if opt.v else logging.INFO) if opt.output != '/dev/stdout' and os.path.exists(opt.output): LG.error("output file (%s) exsts. Exiting ...", opt.output) return 1 command = MsInterface.command_from_dict(vars(opt)) pref = os.path.basename(opt.s_path).replace(".s", "") header_suff = ",".join(['label', opt.repeat_colname, 'b_path']) for j in range(opt.repeat): with open(opt.output, 'a') as fd: res = get_output(command) if j == 0: fd.write(res[0] + "," + header_suff + "\n") for line in res[1:]: fd.write(line.replace(" ", "") + ("," + opt.label) + ("," + str(j + 1)) + ("," + pref) + "\n")
def linear_search_plus_merge(file_path_P, file_path_T): read_accesses = 0 write_accesses = 0 to_write = [] P = get_P(file_path_P) P.sort() file_T = get_T(file_path_T) length_T = get_length_file(file_path_T) output = get_output('output_files/output_linear_search_plus_merge.txt') for iteration_index in range(0, length_T, B): start_reading_from = LINE_SIZE * iteration_index if iteration_index + B > length_T: number_of_lines = length_T - iteration_index else: number_of_lines = B lines = read_many_lines(start_reading_from, number_of_lines, file_T) read_accesses += 1 index_P = 0 index_T = 0 while index_P < len(P) and index_T < len(lines): element_P = P[index_P] element_T = lines[index_T] if element_P > element_T: index_T += 1 elif element_P < element_T: index_P += 1 else: found_element = element_T.zfill(9) + '\n' to_write.append(found_element) index_P += 1 index_T += 1 if len(to_write) * LINE_SIZE >= BLOCK_SIZE: output.write(''.join(to_write)) write_accesses += 1 to_write = [] file_T.close() if (len(to_write) != 0): output.write(''.join(to_write)) write_accesses += 1 output.close() return read_accesses, write_accesses
def main(): print("hello world") interpreter = make_interpreter("models/quantized_vae.tflite") interpreter.allocate_tensors() size = utils.input_size(interpreter) print(size) #image = cv2.imread("0.0722970757_0.223937735123.png", 0) #image = cv2.resize(image, (200, 120), interpolation=cv2.INTER_AREA) # width(x), height(y) #with open("test.jpg", "rb") as f: # b = io.BytesIO(f.read()) image = Image.open('test.jpg') image = image.convert('L') #.resize(size, Image.ANTIALIAS) image = np.expand_dims(np.array(image).astype(np.float32), axis=2) print(image[image > 0]) utils.set_input(interpreter, image) image_x = utils.get_output(interpreter) print(image_x[image_x > 0]) print(image_x.shape) image_x = Image.fromarray(image_x.reshape(120, 200)) new_p = image_x.convert("L") new_p.save('out.jpg')
def binary_search(path_p, path_t): read_accesses = 0 write_accesses = 0 to_write = [] P = get_P(path_p) ft = get_T(path_t) output = get_output('output_files/output_binary.txt') # We need to iterate over P loaded in memory for p in P: l = 0 h = get_length_file(path_t) - 1 m = 0 stop = False while (l <= h and not stop): m = (l + h) // 2 current_num = read_a_line_from_file(ft, m) read_accesses += 1 if p < current_num: h = m - 1 elif p > current_num: l = m + 1 else: num = current_num.zfill(9) + '\n' to_write.append(num) if len(to_write) * LINE_SIZE >= BLOCK_SIZE: output.write(''.join(to_write)) write_accesses += 1 to_write = [] stop = True ft.close() if (len(to_write) != 0): output.write(''.join(to_write)) write_accesses += 1 output.close() return read_accesses, write_accesses
def _prepare_solution(self, identifier, configuration, mode='task'): """ Generate a working directory, configuration files and multiprocessing Process object to be ready to just run it. :param identifier: Job or task identifier. :param configuration: A dictionary with a cinfiguration or description. :param mode: 'task' or 'job'. :raise SchedulerException: Raised if the preparation fails and task or job cannot be scheduled. """ self.logger.info("Going to prepare execution of the {} {}".format(mode, identifier)) node_status = self._manager.node_info(self._node_name) if mode == 'task': subdir = 'tasks' client_conf = self._get_task_configuration() self._manager.check_resources(configuration, job=False) else: subdir = 'jobs' client_conf = self._job_conf_prototype.copy() self._manager.check_resources(configuration, job=True) args = [self._client_bin, mode] self._create_work_dir(subdir, identifier) client_conf["Klever Bridge"] = self.conf["Klever Bridge"] client_conf["identifier"] = identifier work_dir = os.path.join(self.work_dir, subdir, identifier) file_name = os.path.join(work_dir, 'client.json') args.extend(['--file', file_name]) self._reserved[subdir][identifier] = dict() # Check disk space limitation if "keep working directory" in self.conf["scheduler"] and self.conf["scheduler"]["keep working directory"] and \ 'disk memory size' in configuration["resource limits"] and \ configuration["resource limits"]['disk memory size']: current_space = int(utils.get_output('du -bs {} | cut -f1'.format(work_dir))) if current_space > configuration["resource limits"]['disk memory size']: raise schedulers.SchedulerException( "Clean manually existing working directory of {} since its size on the disk is {}B which is " "greater than allowed limitation of {}B". format(os.path.abspath(work_dir), current_space, configuration["resource limits"]['disk memory size'])) if configuration["resource limits"].get("CPU time"): # This is emergency timer if something will hang timeout = int((configuration["resource limits"]["CPU time"] * 1.5) / 100) else: timeout = None process = multiprocessing.Process(None, self._process_starter, identifier, [timeout, args]) if mode == 'task': client_conf["Klever Bridge"] = self.conf["Klever Bridge"] client_conf["identifier"] = identifier client_conf["common"]["working directory"] = work_dir for name in ("verifier", "upload input files of static verifiers"): client_conf[name] = configuration[name] # Speculative flag if configuration.get('speculative'): client_conf["speculative"] = True # Do verification versions check if client_conf['verifier']['name'] not in client_conf['client']['verification tools']: raise schedulers.SchedulerException( 'Use another verification tool or install and then specify verifier {!r} with its versions at {!r}'. format(client_conf['verifier']['name'], self.conf["scheduler"]["task client configuration"])) if 'version' not in client_conf['verifier']: raise schedulers.SchedulerException('Cannot find any given {!r} version at at task description'. format(client_conf['verifier']['name'])) if client_conf['verifier']['version'] not in \ client_conf['client']['verification tools'][client_conf['verifier']['name']]: raise schedulers.SchedulerException( 'Use another version of {!r} or install given version {!r} and specify it at scheduler client ' 'configuration {!r}'.format(client_conf['verifier']['name'], client_conf['verifier']['version'], self.conf["scheduler"]["task client configuration"])) self._task_processes[identifier] = process else: klever_core_conf = configuration.copy() del klever_core_conf["resource limits"] klever_core_conf["Klever Bridge"] = self.conf["Klever Bridge"] klever_core_conf["working directory"] = "klever-core-work-dir" self._reserved["jobs"][identifier]["configuration"] = klever_core_conf client_conf["common"]["working directory"] = work_dir client_conf["Klever Core conf"] = self._reserved["jobs"][identifier]["configuration"] self._job_processes[identifier] = process client_conf["resource limits"] = configuration["resource limits"] # Add particular cores if "resource limits" not in client_conf: client_conf["resource limits"] = {} client_conf["resource limits"]["CPU cores"] = \ self._get_virtual_cores(int(node_status["available CPU number"]), int(node_status["reserved CPU number"]), int(configuration["resource limits"]["number of CPU cores"])) if mode != "task": if len(client_conf["resource limits"]["CPU cores"]) == 0: data = utils.extract_cpu_cores_info() client_conf["Klever Core conf"]["task resource limits"]["CPU Virtual cores"] = \ sum((len(data[a]) for a in data)) else: client_conf["Klever Core conf"]["task resource limits"]["CPU Virtual cores"] = \ len(client_conf["resource limits"]["CPU cores"]) # Save Klever Core configuration to default configuration file with open(os.path.join(work_dir, "core.json"), "w", encoding="utf8") as fh: json.dump(client_conf["Klever Core conf"], fh, ensure_ascii=False, sort_keys=True, indent=4) with open(file_name, 'w', encoding="utf8") as fp: json.dump(client_conf, fp, ensure_ascii=False, sort_keys=True, indent=4)
def evaluation(model, model_name, X, y, cohorts, all_tasks, FLAGS): ''' model: pytorch model X: (n, d) pytorch dataset y: (n,) pytorch dataset cohorts: (n,) pytorch dataset all_tasks (Numpy array/list): List of tasks ''' cohort_aucs = [] loader = create_loader(X, y) if 'mtl' in model_name: y_pred = get_output_mtl(model, X, y, cohorts).ravel() else: y_pred = get_output(model, loader).ravel() y = dataset2numpy(y).astype(int) cohorts = dataset2numpy(cohorts).astype(int) # all bootstrapped AUCs for task in all_tasks: y_pred_in_cohort = y_pred[cohorts == task] y_in_cohort = y[cohorts == task] if FLAGS.bootstrap: all_aucs = bootstrap_metric(y_in_cohort, y_pred_in_cohort, FLAGS.num_bootstrap_samples, roc_auc_score) cohort_aucs.append(np.array(all_aucs)) min_auc, max_auc, avg_auc = np.min(all_aucs), np.max(all_aucs), np.mean(all_aucs) print('{} Model AUC on {}: [min: {}, max: {}, avg: {}]'.format(model_name, task, min_auc, max_auc, avg_auc)) else: auc = roc_auc_score(y_in_cohort, y_pred_in_cohort) cohort_aucs.append(auc) print('{} Model AUC on {}: {}'.format(model_name, task, cohort_aucs[-1])) if FLAGS.bootstrap: # Macro AUC cohort_aucs = np.array(cohort_aucs) cohort_aucs = np.concatenate( (cohort_aucs, np.expand_dims(np.mean(cohort_aucs, axis=0), 0))) all_aucs = cohort_aucs[-1] min_auc, max_auc, avg_auc = np.min(all_aucs), np.max(all_aucs), np.mean(all_aucs) print('{} Model AUC Macro: [min: {}, max: {}, avg: {}]'.format(model_name, min_auc, max_auc, avg_auc)) # Micro AUC all_micro_aucs = bootstrap_metric(y, y_pred, FLAGS.num_bootstrap_samples, roc_auc_score) cohort_aucs = np.concatenate( (cohort_aucs, np.array([all_micro_aucs]))) all_aucs = cohort_aucs[-1] min_auc, max_auc, avg_auc = np.min(all_aucs), np.max(all_aucs), np.mean(all_aucs) print('{} Model AUC Micro: [min: {}, max: {}, avg: {}]'.format(model_name, min_auc, max_auc, avg_auc)) else: # Macro AUC macro_auc = np.mean(cohort_aucs) cohort_aucs.append(macro_auc) print('{} Model AUC Macro: {}'.format(model_name, cohort_aucs[-1])) # Micro AUC micro_auc = roc_auc_score(y, y_pred) cohort_aucs.append(micro_auc) print('{} Model AUC Micro: {}'.format(model_name, cohort_aucs[-1])) return cohort_aucs
def get_rev(): return utils.get_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=BASE_DIR)
SPEC = os.path.join(BASE_DIR, 'win', 'rednotebook.spec') WINE_BUILD = os.path.join(DRIVE_C, 'build') WINE_DIST = os.path.join(DRIVE_C, 'dist') LOCALE_DIR = os.path.join(WINE_DIST, 'share', 'locale') WINE_RN_EXE = os.path.join(WINE_DIST, 'rednotebook.exe') WINE_PYTHON = os.path.join(DRIVE_C, 'Python34', 'python.exe') utils.confirm_overwrite(DIST_DIR) os.environ['WINEPREFIX'] = DIST_DIR utils.ensure_path(os.path.dirname(DIST_DIR)) print('Start copying {} to {}'.format(BUILD_DIR, DIST_DIR)) utils.fast_copytree(BUILD_DIR, DIST_DIR) print('Finished copying') archive = '/tmp/rednotebook-archive.tar' stash_name = utils.get_output(['git', 'stash', 'create'], cwd=BASE_DIR) stash_name = stash_name or 'HEAD' run(['git', 'archive', stash_name, '-o', archive], cwd=BASE_DIR) utils.ensure_path(WINE_RN_DIR) run(['tar', '-xf', archive], cwd=WINE_RN_DIR) os.mkdir(os.path.join(DRIVE_C, 'Python34/share')) shutil.copytree(os.path.join(DRIVE_C, 'Python34/Lib/site-packages/gnome/share/gir-1.0/'), os.path.join(DRIVE_C, 'Python34/share/gir-1.0/')) run(['wine', WINE_PYTHON, '-m', 'PyInstaller', '--workpath', WINE_BUILD, '--distpath', DRIVE_C, SPEC]) # will be built at ...DRIVE_C/dist run(['python3', 'build-translations.py', LOCALE_DIR], cwd=DIR) if args.test: run(['wine', WINE_RN_EXE])
def training(self, image_name, image_i, epoch, fold_num): """Método de treinamento da rede""" mlp_input = None image = None try: self.cursor. execute( 'SELECT {} FROM treinamento WHERE image_name = "{}"'.format(self.descriptor, image_name)) row = self.cursor.fetchone() image = np.frombuffer(row[0]) except TypeError: print('Erro na obtenção da matriz {} da imagem {} da tabela "treinamento"'.format( self.descriptor, image_name)) print('Execução encerrada') exit() # camada de entrada: preparação mlp_input = np.array(image.reshape(1, np.size(image))) self.l0_neurons = len(mlp_input) expected_output = np.array(u.get_output(image_name, self.part_2)) # pesos: preparação dos pesos para esta execução weights_0 = self.weights_0 weights_1 = self.weights_1 # bias bias_0 = self.bias_0 bias_1 = self.bias_1 # feed forward layer_0 = mlp_input # 1x576 (HOG) layer_1 = self.activFunction(np.dot(layer_0, weights_0) + bias_0) # ->1x32 (1x576 por 576x32) layer_2 = self.activFunction(np.dot(layer_1, weights_1) + bias_1).T # ->1X3 (1x32 por 32x3) (T)-> 3x1 y_error = (expected_output - layer_2) # 3x1 - 3x1 = 3x1 # erro quadrático médio de uma imagem avg_y_error = np.sum(np.power(y_error, 2)) / 2 self.error_training_avg = self.error_training_avg + avg_y_error self.training_number = self.training_number + 1 # erros: segunda camada y_error = np.array(y_error) * self.derivative(np.array(layer_2)) # 3x1 (3x1 por 3x1) y_error = y_error.T # 1x3 y_delta = self.alpha * layer_1.T.dot(y_error) # 32x3 (32x1 por 1x3) bias_1_delta = self.alpha * y_error # 1x3 # erros: repasse para a camada escondida z_error = y_error.dot(weights_1.T) # 1x32 (1x3 por 3x32) z_error = z_error * self.derivative(np.array(layer_1)) # 1x32 (1x32 por 1x32) z_delta = self.alpha * layer_0.T.dot(z_error) # 576x32 (576x1(HOG) por 1x32) bias_0_delta = self.alpha * z_error # pesos e bias: atualização weights_1 += y_delta weights_0 += z_delta bias_1_delta += bias_1 bias_0_delta += bias_0 self.weights_1 = weights_1 self.weights_0 = weights_0 self.bias_1 = bias_1_delta self.bias_0 = bias_0_delta
lr = learning_rate(args.lr, epoch) for param_group in optimizer.param_groups: param_group['lr'] = lr # label-correction if epoch > args.correction: train_eval_loader = torch.utils.data.DataLoader(train_dataset, batch_size=256, shuffle=False, **kwargs) noisy_targets = np.eye(args.num_class)[train_dataset.targets] log(logpath, 'Updating labels.\n') args.sigma = 0 output, losses = get_output(ema_net, device, train_eval_loader) losses = (losses - min(losses)) / (max(losses) - min(losses) ) # normalize to [0,1] losses = losses.reshape([len(losses), 1]) targets = losses * noisy_targets + (1 - losses) * output train_dataset.targets = targets train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, **kwargs) _, train_acc = train_noise(args, net, device, train_loader, optimizer, epoch, ema_optimizer) _, test_acc_NoEMA = test(args, net, device, test_loader) _, val_acc = test(args, ema_net, device, val_loader)
SPEC = os.path.join(BASE_DIR, 'win', 'rednotebook.spec') WINE_BUILD = os.path.join(DRIVE_C, 'build') WINE_DIST = os.path.join(DRIVE_C, 'dist') LOCALE_DIR = os.path.join(WINE_DIST, 'share', 'locale') WINE_RN_EXE = os.path.join(WINE_DIST, 'rednotebook.exe') WINE_PYTHON = os.path.join(DRIVE_C, 'Python34', 'python.exe') utils.confirm_overwrite(DIST_DIR) os.environ['WINEPREFIX'] = DIST_DIR utils.ensure_path(os.path.dirname(DIST_DIR)) print('Start copying {} to {}'.format(BUILD_DIR, DIST_DIR)) utils.fast_copytree(BUILD_DIR, DIST_DIR) print('Finished copying') archive = '/tmp/rednotebook-archive.tar' stash_name = utils.get_output(['git', 'stash', 'create'], cwd=BASE_DIR) stash_name = stash_name or 'HEAD' run(['git', 'archive', stash_name, '-o', archive], cwd=BASE_DIR) utils.ensure_path(WINE_RN_DIR) run(['tar', '-xf', archive], cwd=WINE_RN_DIR) os.mkdir(os.path.join(DRIVE_C, 'Python34/share')) shutil.copytree( os.path.join(DRIVE_C, 'Python34/Lib/site-packages/gnome/share/gir-1.0/'), os.path.join(DRIVE_C, 'Python34/share/gir-1.0/')) run([ 'wine', WINE_PYTHON, '-m', 'PyInstaller', '--workpath', WINE_BUILD, '--distpath', DRIVE_C, SPEC ]) # will be built at ...DRIVE_C/dist run(['python3', 'build-translations.py', LOCALE_DIR], cwd=DIR)
def __extract_description(self, solution_dir): """ Get directory with BenchExec output and extract results from there saving them to JSON file according to provided path. :param solution_dir: Path with BenchExec output. :return: Identifier string of the solution. """ identifier = str(uuid.uuid4()) description = {"id": identifier, "resources": {}, "comp": {}} # Import description desc_file = os.path.join(solution_dir, "runDescription.txt") self.logger.debug( "Import description from the file {}".format(desc_file)) description["desc"] = "" if os.path.isfile(desc_file): with open(desc_file, encoding="utf8") as di: for line in di: key, value = line.strip().split("=") if key == "tool": description["desc"] += value elif key == "revision": description["desc"] += " {}".format(value) else: raise FileNotFoundError( "There is no solution file {}".format(desc_file)) # Import general information general_file = os.path.join(solution_dir, "runInformation.txt") self.logger.debug( "Import general information from the file {}".format(general_file)) termination_reason = None number = re.compile("(\d.*\d)") if os.path.isfile(general_file): with open(general_file, encoding="utf8") as gi: for line in gi: key, value = line.strip().split("=", maxsplit=1) if key == "terminationreason": termination_reason = value elif key == "command": description["comp"]["command"] = value elif key == "exitsignal": description["signal num"] = int(value) elif key == "exitcode": description["return value"] = int(value) elif key == "walltime": sec = number.match(value).group(1) if sec: description["resources"]["wall time"] = int( float(sec) * 1000) else: self.logger.warning( "Cannot properly extract wall time from {}". format(general_file)) elif key == "cputime": sec = number.match(value).group(1) if sec: description["resources"]["CPU time"] = int( float(sec) * 1000) else: self.logger.warning( "Cannot properly extract CPU time from {}". format(general_file)) elif key == "memory": mem_bytes = number.match(value).group(1) if mem_bytes: description["resources"]["memory size"] = int( mem_bytes) else: self.logger.warning( "Cannot properly extract exhausted memory from {}" .format(general_file)) elif key == "coreLimit": cores = int(value) description["resources"]["coreLimit"] = cores else: raise FileNotFoundError( "There is no solution file {}".format(general_file)) # Set final status if termination_reason: if termination_reason == "cputime": description["status"] = "TIMEOUT" elif termination_reason == "memory": description["status"] = 'OUT OF MEMORY' else: raise ValueError("Unsupported termination reason {}".format( termination_reason)) elif "signal num" in description: description["status"] = "killed by signal" elif "return value" in description: if description["return value"] == 0: if glob.glob( os.path.join(solution_dir, "output", "witness.*.graphml")): description["status"] = "false" else: # Check that soft limit has not activated status_checker = 'grep -F "Verification result: UNKNOWN" -m 1 -c {}'.\ format(os.path.join(solution_dir, "output", "benchmark.logfiles", "*.log")) number = int(utils.get_output(status_checker)) if number > 0: description["status"] = "unknown" else: description["status"] = "true" else: description["status"] = "unknown" else: raise ValueError( "Cannot determine termination reason according to the file {}". format(general_file)) # Import Host information host_file = os.path.join(solution_dir, "hostInformation.txt") self.logger.debug( "Import host information from the file {}".format(host_file)) lv_re = re.compile("Linux\s(\d.*)") if os.path.isfile(host_file): with open(host_file, encoding="utf8") as hi: for line in hi: key, value = line.strip().split("=", maxsplit=1) if key == "name": description["comp"]["node name"] = value elif key == "os": version = lv_re.match(value).group(1) if version: description["comp"][ "Linux kernel version"] = version else: self.logger.warning( "Cannot properly extract Linux kernel version from {}" .format(host_file)) elif key == "memory": description["comp"]["mem size"] = value elif key == "cpuModel": description["comp"]["CPU model"] = value elif key == "cores": description["comp"]["number of CPU cores"] = value else: raise FileNotFoundError( "There is no solution file {}".format(host_file)) return identifier, description