def get_instances(self, n_instances): i = 0 while i < n_instances: if i % 2 == 0: yield CNF.read_dimacs(self.filenames[self.index][0]) else: yield CNF.read_dimacs(self.filenames[self.index][1]) self.index += 1 #end if-else i += 1
def get_instances(self, n_instances): for i in range(n_instances): yield CNF.read_dimacs(self.filenames[self.index]) if self.index + 1 < len(self.filenames): self.index += 1 else: self.reset()
def resolve(rgoal, clause): rgoal = rgoal.copy() clause = clause.copy() if len(rgoal.get_terms()) > len(clause.get_terms()): return resolve(clause, rgoal) if not is_suitable(rgoal, clause): return False theta = Substitution() for goal_term in rgoal.get_terms(): op = goal_term.get_op() suitable_terms = clause.find_term_by_op(op) if suitable_terms == False: continue for suitable_term in suitable_terms: if goal_term.isNegative == suitable_term.isNegative: continue theta = Substitution() unify(goal_term, suitable_term, theta) if theta is not False: clause.remove(suitable_term) rgoal.remove(goal_term) break result = CNF(rgoal.get_terms() + clause.get_terms()) for term in result.terms: theta.SUBST(term) return result
def get_metrics(heuristic, n_puzzles, n_runs, example_puzzles): splits = [] backtracks = [] unit_assigns = [] if heuristic is None: print("Solving using DP") else: print("Solving using %s" % heuristic) i = 1 for puz in example_puzzles[:n_puzzles]: print("Puzzle number %d" % i) total_splits = 0 total_backtracks = 0 total_unit_assigns = 0 for t in range(1, n_runs + 1): cnf = CNF(puz) sudoku_solver = SATSolver(heuristic=heuristic) sudoku_solver.solve(cnf) total_splits += sudoku_solver.splits total_backtracks += sudoku_solver.backtracks total_unit_assigns += cnf.unit_assignments splits += [total_splits / float(t)] backtracks += [total_backtracks / float(t)] unit_assigns += [total_unit_assigns / float(t)] i += 1 avg_splits = np.mean(splits) avg_backtracks = np.mean(backtracks) avg_unit_assigns = np.mean(unit_assigns) standarddeviations = np.std(splits) / 2, np.std(backtracks) / 2, np.std( unit_assigns) / 2 return avg_splits, avg_backtracks, avg_unit_assigns / avg_splits, standarddeviations
def read_input(input_path): print("Read input from file {}".format(input_path)) comment_line_regex = re.compile(COMMENT_LINE_PATTERN) info_line_regex = re.compile(INFO_LINE_PATTERN) formula = [] num_props, num_clauses = 0, 0 with open(input_path, 'r') as input_file: for line in input_file.readlines(): line = line.strip() if line == "%" or not line: continue if not comment_line_regex.match(line): infos = info_line_regex.match(line) if infos: num_props = int(infos.group(1)) num_clauses = int(infos.group(2)) print(num_props) print(num_clauses) else: raw_props = line.rstrip('\n').split() props = [] for prop in raw_props: prop = int(prop) if prop != 0: props.append(prop) formula.append(props) return CNF(num_props, num_clauses, formula)
def load_dir(path, no_sat=False): data = [] for filename in os.listdir(path): name, ext = os.path.splitext(filename) if ext != '.cnf': continue f = CNF.from_file(os.path.join(path, filename)) sat = int(not name.startswith('uu')) if not no_sat else -1 data.append(DataSample(filename, f, adj(f), sat)) return data
def resolution(kb, goal): if goal.isNegative: return not resolution(kb, goal.get_negated()) kbCNF = [CNF.parse(fact) for fact in kb.getFacts()] kbCNF += [CNF.parse(rule) for rule in kb.getRules()] if not isinstance(goal, Fact): return rgoal = CNF.parse(goal.get_negated()) kbCNF.append(rgoal) while True: suitableFound = False for term in kbCNF: if is_suitable(term, rgoal): rgoal = resolve(term, rgoal) kbCNF.append(rgoal) suitableFound = True if len(rgoal.get_terms()) == 0: return True if not suitableFound: return False
def load_dir(path): data = [] for filename in os.listdir(path): name, ext = os.path.splitext(filename) if ext != '.cnf': continue f = CNF.from_file(os.path.join(path, filename)) if name.startswith('uu'): continue data.append(DataSample(filename, f, adj(f), None)) return data
def main(): s = CNF([ [1, 2], [-2, 3], [-3], [3], ]) print(s) print(DPLL().run(s)) rcnf = get_random_kcnf(3, 4, 20) print(rcnf) print(DPLL().run(rcnf)) print(rcnf.simplified())
def build_cnf(): diffeq = ODENet(fhidden=fhidden, fcontext=fcontext, in_shape=(fin, ), layer_type=args.layer_type) odefunc = ODEFunction(diffeq=diffeq) cnf = CNF(df=odefunc, T=args.T, train_T=args.train_T, conditional=conditional, solver=args.solver, use_adjoint=args.use_adjoint, atol=args.atol, rtol=args.rtol) return cnf
def pick_branching_variable(self): """ Pick a variable to assign a value. :return: variable, value assigned """ new_clauses = [] for clause in self.clauses.union(self.learnts): lit_vals = [self.compute_value(lit) for lit in clause] if TRUE in lit_vals: continue new_clause = [lit for lit in clause if self.compute_value(lit) == UNASSIGN] new_clauses.append(new_clause) if not new_clauses: # pick whatever, clause will be solved anyway var = next(self.all_unassigned_vars()) return var, TRUE self.number_of_runs += 1 lit = self.suggest(CNF(new_clauses)) return abs(lit), (TRUE if lit > 0 else FALSE)
def read_cnf_file(filename: str) -> CNF: with open(filename, "r") as cnfFile: lines = list(filter(lambda line: line.strip(), cnfFile.readlines())) cnf = CNF() cnf.clauses = list(list()) for line in lines: if len(line) > 20 and line[:20] == "c automorphism group": cnf.automorphism_group_size = int(line.split()[-1]) elif line[0] == "c": continue elif line[0] == "p": metadata = line.split() cnf.number_of_literals = int(metadata[2]) cnf.number_of_clauses = int(metadata[3]) else: cnf.clauses.append([int(x) for x in line.split()[:-1]]) return cnf
def main(): s = CNF_extract_file(from_file='uf125-01.cnf') #s = CNF(s) #print(s.clauses) #print(CDCLRawSolver(CNF(s.clauses)).run()) query = np.array([[ 0, -1, 1, 0, 0, 0, -1], [-1, -1, 0, 0, 1, 1, 1], [ 1, 0, 1, 0, 0, 0, -1], [ 1, 0, 1, 0, 0, 0, 1]], dtype=np.int32) table = np.array([[-1], [-1], [ 0], [ 1], [ 1], [ 0], [ 1]], dtype=np.int32) # Warm up #out_unit(table, query) timer(lambda: print(CDCLRawSolver(CNF(s.clauses)).run()))
def produce_cnf( problemType: str, pattern_filename: str, graph_filename: str, target_dir: str = suggested_dir, compute_automorph_size=True, ) -> str: cnf_file_name = "{}/cnf-{}-h-{}-g-{}.cnf".format( target_dir, problemType.strip("--"), pattern_filename.split("/")[-1].split(".")[0], graph_filename.split("/")[-1].split(".")[0], ) # if file already exists, return filename. if path.isfile(cnf_file_name): return cnf_file_name # read input files pattern = read_graph(pattern_filename) graph = read_graph(graph_filename) # create CNF: cnf = CNF(number_of_literals=pattern.number_of_verticies * graph.number_of_verticies, clauses=list(list())) def pair(a: int, v: int) -> int: return (a - 1) * graph.number_of_verticies + v # adding clauses to guarantee that each a is mapped to at least one vertex for a in range(1, pattern.number_of_verticies + 1): cnf.clauses.append( list(pair(a, v) for v in range(1, graph.number_of_verticies + 1))) for a in range(1, pattern.number_of_verticies + 1): for v in range(1, graph.number_of_verticies + 1): for w in range(v + 1, graph.number_of_verticies + 1): cnf.clauses.append([-pair(a, v), -pair(a, w)]) # dictionary, to use for finding non existant pairs: non_existant_pairs = {} for i in range(1, graph.number_of_verticies + 1): non_existant_pairs[i] = list(range(i, graph.number_of_verticies + 1)) for v, w in graph.edges: assert v < w non_existant_pairs[v].remove(w) for a, b in pattern.edges: for v, ws in non_existant_pairs.items(): for w in ws: cnf.clauses.append([-pair(a, v), -pair(b, w)]) if pair(a, v) != pair(a, w) or pair(b, w) != pair(b, v): cnf.clauses.append([-pair(a, w), -pair(b, v)]) # changed second pair to negative, not in the SS notes. if problemType == "--emb": for v in range(1, graph.number_of_verticies + 1): for a in range(1, pattern.number_of_verticies + 1): for b in range(a + 1, pattern.number_of_verticies + 1): clause = [-pair(a, v), -pair(b, v)] if clause not in cnf.clauses: cnf.clauses.append(clause) # count the automorphism group size, by counting embeddings of the pattern unto the pattern. # embed it in a comment comment line on top. auto_size = 0 if compute_automorph_size: auto_size_problem = produce_cnf("--emb", pattern_filename, pattern_filename, target_dir, False) auto_size = int(count_sharp_file(auto_size_problem)[0]) cnf.number_of_clauses = len(cnf.clauses) write_cnf_file(cnf, cnf_file_name, auto_size) return cnf_file_name
def main(args): if args.seed > -1: random.seed(args.seed) if args.dir_path: med_flips = [] avg_flips = [] max_flips = [] solved = [] med_backflips = [] avg_backflips = [] max_backflips = [] unsats = [] mean_delta = [] downwards = [] upwards = [] sideways = [] for i, filename in enumerate(os.listdir(args.dir_path)): if i >= args.samples: break formula = CNF.from_file(os.path.join(args.dir_path, filename)) walksat = WalkSAT(args.max_tries, args.max_flips, args.p) walksat.run(formula) flips = walksat.flips_to_solution backflips = walksat.backflips unsats = walksat.unsat_clauses diffs = [np.diff(u) for u in unsats] diff_a = np.array(diffs) upwards.append(np.mean([np.sum(a > 0) for a in diff_a])) downwards.append(np.mean([np.sum(a < 0) for a in diff_a])) sideways.append(np.mean([np.sum(a == 0) for a in diff_a])) mean_delta.append(np.mean([np.mean(d) for d in diffs])) med_backflips.append(np.median(backflips)) avg_backflips.append(np.mean(backflips)) max_backflips.append(np.max(backflips)) med = np.median(flips) med_flips.append(med) avg_flips.append(np.mean(flips)) max_flips.append(np.max(flips)) solved.append(int(med < args.max_flips)) # print(med_flips) # print(avg_flips) # print(max_flips) logging.info( 'Acc: {:.4f}, Med: {:.4f}, Mean: {:.4f}, Max: {:.4f}'.format( 100 * np.mean(solved), np.median(med_flips), np.mean(avg_flips), np.mean(max_flips))) logging.info( '(Backflips) Med: {:.4f}, Mean: {:.4f}, Max: {:.4f}'.format( np.median(med_backflips), np.mean(avg_backflips), np.mean(max_backflips))) logging.info('(Delta) Mean: {:.4f}'.format(np.mean(mean_delta))) logging.info( '(Movement) Up: {:.4f}, Side: {:.4f}, Down: {:.4f}'.format( np.mean(upwards), np.mean(sideways), np.mean(downwards))) elif args.file_path: formula = CNF.from_file(args.file_path) walksat = WalkSAT(args.max_tries, args.max_flips, args.p) walksat.run(formula) logging.info('Number of solutions found: {}\n' 'Avg, std of flips to solution: {:.4f}, {:.4f}'.format( len(walksat.flips_to_solution), np.mean(walksat.flips_to_solution), np.std(walksat.flips_to_solution), ))
def generate_minisat_problem(self): cnf = CNF("Generated K-Color Problem") self._at_leat_one_color(cnf) self._max_one_color(cnf) self._different_colors(cnf) return str(cnf)
def __init__(self): self.n_variables = np.random.randint(4, 6) self.j = np.random.randint(2, 4) self.__cnf = CNF.random(self.n_variables, self.j) self.__test_reward = 0
with open("test-64.log", "w") as f: f.write( "file epoch batchnumber nvertices nedges loss sat cn neurosat_cnpred pred\n" ) for e, filename in enumerate(os.listdir(test_folder)): if filename.endswith(".graph"): Ma, _, cn, diff_edge = read_graph(test_folder + "/" + filename) #first iterate without the diff edge and with cn for j in range(2, cn + 5): nv = Ma.shape[0] ne = len(np.nonzero(Ma)[0]) parse_glucose(Ma, j, "test-tmp", "temp.cnf") batch = create_batchCNF( [CNF.read_dimacs("test-tmp/temp.cnf")]) l, a, p = run_test_batch(sess, solver, batch, time_steps) if p == 1: f.write( "{filename} {epoch} {batch} {nv} {ne} {loss:.4f} {accuracy:.4f} {cn} {cnpred} {avg_pred:.4f}\n" .format( filename=filename, epoch=e, batch=0, nv=nv, ne=ne, loss=l, accuracy=a, cn=cn, cnpred=j,
def play_rounds(rounds=cf.ROUNDS, info=True, num_agents=None, num_epistemic=None, all_args=cf.ALL_ARGS_IN_AF, replacement=cf.REPLACEMENT, all_forget=cf.ALL_AGENTS_FORGET, opp_selection=cf.OPPONENT_SELECTION, forget=cf.ODD_AGENTS_FORGET, num_args_AF=None, num_practical=None, num_attacks=None, num_offers=cf.NUM_OFFERS, p_attack_AF=cf.P_ATTACK_IN_AF, classic=cf.ALL_AGENTS_CLASSIC): """ Generate agents and run a number of rounds with them. Parameters with a None default value are range-based as config default, and will be overwritten accordingly (in the MAS generator). """ # Generate a system of agents agents = generate_system(num_agents=num_agents, num_epistemic=num_epistemic, all_args=all_args, num_args_AF=num_args_AF, num_practical=num_practical, num_attacks=num_attacks, num_offers=num_offers, p_attack_AF=p_attack_AF) # Prepare the directory where the AFs and CAFs may be saved if cf.VISUALIZE: prepare_directory() n_iter = 0 n_negotiation = 0 # Play a number of rounds for rnd in range(rounds): # Track which agents have played in the current round track_played = [] # If odd number of agents, add one to the track_played, and play # rounds with the remaining even number. if len(agents) % 2: track_played.append( random.choice([ ag_idx for ag_idx in range(len(agents)) if ag_idx not in track_played ])) # Play until all agents have played once. (Or if replacement, # equally as many times.) while len(track_played) != len(agents): # Select a random agent (if sampling with replacement, then # the agent mustn't have played yet) agent1 = random.choice([ ag_idx for ag_idx in range(len(agents)) if replacement or ag_idx not in track_played ]) track_played.append(agent1) # Find list of potential opponents -- at this point, all # other agents (if sampling with replacement, then # the opponent mustn't have played yet) potential_opponents = [ (ix, ag) for ix, ag in enumerate(agents) if ix != agent1 and (replacement or ix not in track_played) ] # If opponent selection is enabled, agents will choose # opponents with a similar score if opp_selection: # Choose the opponent with the most similar score agent2 = sorted(potential_opponents, key=lambda opp: abs(opp[1].score - agents[ agent1].score))[0][0] else: # Choose a random opponent agent2 = random.choice(potential_opponents)[0] track_played.append(agent2) # then overwrite the new agent with its prior state if (all_forget or (forget and is_odd_agent(agents[agent1].name))): ag1backup = agents[agent1] if (all_forget or (forget and is_odd_agent(agents[agent2].name))): ag2backup = agents[agent2] # Play a negotiation session if classic: nf = CNF(agents[agent1], agents[agent2]) else: nf = NF(agents[agent1], agents[agent2]) ag1, ag2, result, this_iter = nf.negotiate(score_agg_mode=cf.MODE) n_negotiation += 1 n_iter += this_iter del nf # Negotiation framework no longer needed # Optionally apply forgetting to the agents: if applicable, # then overwrite the new agent with its prior state if (all_forget or (forget and is_odd_agent(ag1.name))): agents[agent1] = ag1backup del ag1backup else: agents[agent1] = ag1 if (all_forget or (forget and is_odd_agent(ag2.name))): agents[agent2] = ag2backup del ag2backup else: agents[agent2] = ag2 # Visualize if enabled. Disabled in parameter sweeps. if info and cf.VISUALIZE: visualize(ag1=agents[agent1], ag2=agents[agent2], rnd=rnd) if info: print_result(ag1=agents[agent1], ag2=agents[agent2], result=result, rnd=rnd) scores = [agent.score for agent in agents] return agents, scores, n_iter, n_negotiation
def main(): save_dir = args.save_dir + f"/{args.energy_fun}/{args.width}/{args.hidden_dim}" if not os.path.exists(save_dir): os.makedirs(save_dir) t0 = 0 t1 = 10 norm = torch.distributions.MultivariateNormal(loc=torch.tensor([0.0, 0.0]).to(device), covariance_matrix=torch.tensor([[1.0, 0.0], [0.0, 1.0]]).to(device)) odefunc = CNF(in_out_dim=2, hidden_dim=args.hidden_dim, width=args.width) if device != "cpu": odefunc = odefunc.cuda() optimizer = optim.Adam(odefunc.parameters(), lr=1e-3, weight_decay=0.) z0_test = norm.sample([args.num_samples * 20]).to(device) logp_z0_test = norm.log_prob(z0_test).unsqueeze(dim=-1).to(device) if args.energy_fun == 1: energy_fun = energy_function_1 elif args.energy_fun == 2: energy_fun = energy_function_2 elif args.energy_fun == 3: energy_fun = energy_function_3 elif args.energy_fun == 4: energy_fun = energy_function_4 else: raise Exception("Energy function not implemented.") # Train model if args.train: for itr in tqdm(range(args.epochs + 1)): optimizer.zero_grad() z0 = norm.sample([args.num_samples]).to(device) logp_z0 = norm.log_prob(z0).unsqueeze(dim=-1).to(device) loss = calc_loss(odefunc, z0, logp_z0, t0, t1, energy_fun) loss.backward() optimizer.step() best_loss = np.inf if itr % 100 == 0: z_t, logp_diff_t = odeint( odefunc, (z0_test, logp_z0_test), torch.tensor([t0, t1]).type(torch.float32).to(device), atol=1e-5, rtol=1e-5, method='dopri5', ) x, logp_x = z_t[-1], logp_diff_t[-1] loss = (logp_x.mean(0) + energy_fun(x).mean(0)).item() print(f"{itr} Test loss: {loss}") if loss < best_loss: best_loss = loss torch.save(odefunc.state_dict(), f"{save_dir}/best_model.pt") torch.save(odefunc.state_dict(), f"{save_dir}/last_model.pt") plt.figure(figsize=(4, 4), dpi=200) plt.hist2d(*x.detach().cpu().numpy().T, bins=300, density=True, range=[[-4, 4], [-4, 4]]) plt.axis('off') plt.gca().invert_yaxis() plt.margins(0, 0) plt.savefig(save_dir + f"/tgt_itr_{itr:05d}.jpg", pad_inches=0, bbox_inches='tight') plt.close() odefunc.load_state_dict(torch.load(f"{save_dir}/best_model.pt")) if not args.train: z_t, logp_diff_t = odeint( odefunc, (z0_test, logp_z0_test), torch.tensor([t0, t1]).type(torch.float32).to(device), atol=1e-5, rtol=1e-5, method='dopri5', ) x, logp_x = z_t[-1], logp_diff_t[-1] best_loss = (logp_x.mean(0) + energy_fun(x).mean(0)).item() # Generate evolution of density x = np.linspace(-6, 6, 600) y = np.linspace(-6, 6, 600) points = np.vstack(np.meshgrid(x, y)).reshape([2, -1]).T z_t0 = torch.tensor(points).type(torch.float32).to(device) logp_t0 = norm.log_prob(z_t0).unsqueeze(dim=-1).to(device) z_t, logp_t = odeint( odefunc, (z_t0, logp_t0), torch.tensor(np.linspace(t0, t1, 21)).to(device), atol=1e-5, rtol=1e-5, method='dopri5', ) for (t, z, logp) in zip(np.linspace(t0, t1, 21), z_t, logp_t): plt.figure(figsize=(4, 4), dpi=200) plt.tricontourf(*z.detach().cpu().numpy().T, np.exp(logp.view(-1).detach().cpu().numpy()), 200) plt.xlim([-4, 4]) plt.ylim([-4, 4]) plt.tight_layout() plt.axis('off') plt.gca().invert_yaxis() plt.margins(0, 0) plt.savefig(save_dir + f"/density_{best_loss:.10f}_{t:f}.jpg", pad_inches=0, bbox_inches='tight') plt.close()
def main(): s = CNF_extract_file(from_file='uf150-01.cnf') #s = CNF(s) #print(s.clauses) timer(lambda: print(CDCLRawSolver(CNF(s.clauses)).run()))
def get_instances(self, n_instances): for i in range(n_instances): yield CNF.read_dimacs(self.filenames[self.index]) self.index += 1
def genCNF(self): n = self.size expression = CNF(n) # Cell clauses for i in range(n): terms = [] for j in range(1, n + 1): terms.append(Term(j + n * i)) expression.add(Clause(terms)) # Row clauses for i in range(n): for j in range(1, n): number = n * i + j for l in range(1, n - j + 1): expression.add(Clause([-Term(number), -Term(number + l)])) # Columns clauses for j in range(1, n + 1): for i in range(n): number = n * i + j for l in range(1, n - i): expression.add( Clause([-Term(number), -Term(number + l * n)])) # Clauses diagonals left to right (upper bound) for i in range(n - 1): for j in range(i, n - 1): number = n * i + j + 1 for l in range(1, n - j): expression.add( Clause([-Term(number), -Term(number + l * (n + 1))])) # Clauses diagonals left to right (lower bound) for i in range(n - 1): for j in range(i): number = n * i + j + 1 for l in range(1, n - i): expression.add( Clause([-Term(number), -Term(number + l * (n + 1))])) # Clauses diagonals right to left (upper bound) for i in range(n): for j in range(n - i): number = n * i + j + 1 for l in range(1, j + 1): expression.add( Clause([-Term(number), -Term(number + l * (n - 1))])) # Clauses diagonals right to left (lower bound) for i in range(n): for j in range(n - i, n): number = n * i + j + 1 if (number != n * n): for l in range(1, n - i): expression.add( Clause( [-Term(number), -Term(number + l * (n - 1))])) return expression
import sys from graphviz import Graph from cnf import CNF from util import adj f = CNF.from_file(sys.argv[1]) a_pos, a_neg = adj(f) a_pos = a_pos.todense() a_neg = a_neg.todense() n, m = a_pos.shape g = Graph('G', filename=sys.argv[2], format='pdf', engine='neato') for x in range(1, n + 1): g.node(f'x{x}', label='x', color='blue', fillcolor='blue', shape='point') for c in range(1, m + 1): g.node(f'c{c}', label='y', color='red', fillcolor='red', shape='point') pw = sys.argv[3] for x in range(n): for c in range(m): var = f'x{x + 1}' cl = f'c{c + 1}' if a_pos[x, c] == 1: # g.edge(var, cl, color='#ff0000', penwidth='0.001') g.edge(var, cl, color='#ff0000', penwidth=pw)
parser.add_argument( 'inputfile', help="Input file containing SAT problem in DIMACS form") args = vars(parser.parse_args()) if args['S1'] == args['S2'] == args['S3'] is False: print("Choose one among S1/S1/S3") parser.print_help() parser.exit(1) input_file = args['inputfile'] output_file = input_file.split('.')[0] + '.out' with open(input_file) as f: dimacs_str = f.read() if args['S1'] is True: heuristic = None elif args['S2'] is True: heuristic = 'UP' elif args['S3'] is True: heuristic = 'LEFV' cnf = CNF(dimacs_str) sat_solver = SATSolver(heuristic=heuristic) if sat_solver.solve(cnf): write_sol_to_file(cnf, output_file, satisfiable=True) print("Problem is satisfiable. Solution written to %s" % output_file) else: write_sol_to_file(cnf, output_file, satisfiable=False) print("Problem is unsatisfiable")
def main(dataset_name, net_name, xp_path, data_path, load_config, load_model, device, seed, tokenizer, clean_txt, embedding_size, pretrained_model, embedding_reduction, num_dimensions, flow_type, coupling_hidden_size, coupling_hidden_layers, coupling_num_flows, coupling_num_mixtures, coupling_dropout, coupling_input_dropout, max_seq_len, use_length_prior, use_time_embed, prior_dist_type, prior_dist_mu, prior_dist_sigma, prior_dist_start_x, prior_dist_stop_x, ad_score, n_attention_heads, attention_size, lambda_p, alpha_scheduler, optimizer_name, lr, n_epochs, lr_milestone, batch_size, weight_decay, n_jobs_dataloader, n_threads, normal_class): """ :arg DATASET_NAME: Name of the dataset to load. :arg NET_NAME: Name of the neural network to use. :arg XP_PATH: Export path for logging the experiment. :arg DATA_PATH: Root path of data. """ # Get configuration cfg = Config(locals().copy()) # Set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger() logger.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') xp_path += '/text_{}_{}'.format(dataset_name, net_name) if not os.path.exists(xp_path): os.makedirs(xp_path) log_file = xp_path + '/log.txt' file_handler = logging.FileHandler(log_file) file_handler.setLevel(logging.INFO) file_handler.setFormatter(formatter) logger.addHandler(file_handler) # Print paths logger.info('Log file is %s.' % log_file) logger.info('Data path is %s.' % data_path) logger.info('Export path is %s.' % xp_path) # Print experimental setup logger.info('Dataset: %s' % dataset_name) logger.info('Normal class: %d' % normal_class) logger.info('Network: %s' % net_name) logger.info('Tokenizer: %s' % cfg.settings['tokenizer']) logger.info('Clean text in pre-processing: %s' % cfg.settings['clean_txt']) if cfg.settings['embedding_size'] is not None: logger.info('Word vector embedding size: %d' % cfg.settings['embedding_size']) logger.info('Load pre-trained model: %s' % cfg.settings['pretrained_model']) # If specified, load experiment config from JSON-file if load_config: cfg.load_config(import_json=load_config) logger.info('Loaded configuration from %s.' % load_config) # Set seed for reproducibility if cfg.settings['seed'] != -1: random.seed(cfg.settings['seed']) np.random.seed(cfg.settings['seed']) torch.manual_seed(cfg.settings['seed']) torch.cuda.manual_seed(cfg.settings['seed']) torch.backends.cudnn.deterministic = True logger.info('Set seed to %d.' % cfg.settings['seed']) # Default device to 'cpu' if cuda is not available if not torch.cuda.is_available(): device = 'cpu' logger.info('Computation device: %s' % device) logger.info('Number of dataloader workers: %d' % n_jobs_dataloader) if n_threads > 0: torch.set_num_threads(n_threads) logger.info( 'Number of threads used for parallelizing CPU operations: %d' % n_threads) # Load data dataset = load_dataset(dataset_name, data_path, normal_class, cfg.settings['tokenizer'], clean_txt=cfg.settings['clean_txt'], max_seq_len=cfg.settings['max_seq_len']) if net_name == 'CNF': # Initialize CNF model cnf = CNF() encoding_params = { "num_flows": 0, "hidden_layers": 2, "hidden_size": 128 } cnf.set_network( net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], num_dimensions=cfg.settings['num_dimensions'], encoding_params=encoding_params, coupling_hidden_size=cfg.settings['coupling_hidden_size'], coupling_hidden_layers=cfg.settings['coupling_hidden_layers'], coupling_num_flows=cfg.settings['coupling_num_flows'], coupling_num_mixtures=cfg.settings['coupling_num_mixtures'], coupling_dropout=cfg.settings['coupling_dropout'], coupling_input_dropout=cfg.settings['coupling_input_dropout'], max_seq_len=cfg.settings['max_seq_len'], use_time_embed=cfg.settings['use_time_embed']) # If specified, load model parameters from already trained model if load_model: cnf.load_model(import_path=load_model, device=device) logger.info('Loading model from %s.' % load_model) # Train model on dataset prior_dist_params = { "distribution_type": cfg.settings['prior_dist_type'], "mu": cfg.settings['prior_dist_mu'], "sigma": cfg.settings['prior_dist_sigma'], "start_x": cfg.settings['prior_dist_start_x'], "stop_x": cfg.settings['prior_dist_stop_x'] } cnf.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], use_length_prior=cfg.settings['use_length_prior'], prior_dist_params=prior_dist_params, weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model cnf.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader) elif net_name == 'EmbeddingNF': # Initialize EmbeddingNF model and set word embedding enf = EmbeddingNF() enf.set_network( net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], embedding_reduction=cfg.settings['embedding_reduction'], flow_type=cfg.settings['flow_type'], coupling_hidden_size=cfg.settings['coupling_hidden_size'], coupling_num_flows=cfg.settings['coupling_num_flows'], use_length_prior=cfg.settings['use_length_prior'], device=cfg.settings['device']) # If specified, load model parameters from already trained model if load_model: enf.load_model(import_path=load_model, device=device) logger.info('Loading model from %s.' % load_model) # Train model on dataset enf.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model enf.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader) elif net_name == 'date_Net': os.environ["TOKENIZERS_PARALLELISM"] = 'false' masks_ = pkl.load(open(data_path + '/pseudo_labels128_p50.pkl', 'rb')) subset = [dataset.subset] tensorboard_dir = 'run' exp_prefix = 'tests' now = datetime.now() date_time = now.strftime("%m%d%Y_%H%M%S") run_name = f'{subset[0]}_{date_time}' train_args = { "fp16": False, "use_multiprocessing": False, "reprocess_input_data": False, "overwrite_output_dir": True, "num_train_epochs": 20, "save_eval_checkpoints": False, "save_model_every_epoch": False, "learning_rate": 1e-5, "warmup_steps": 1000, "train_batch_size": 16, #was 32 "eval_batch_size": 16, #was 32 "gradient_accumulation_steps": 1, "block_size": 128 + 2, "max_seq_length": 128 + 2, "dataset_type": "simple", "logging_steps": 500, "evaluate_during_training": True, "evaluate_during_training_steps": 500, #was 500 "evaluate_during_training_steps_anomaly": 500, #was 500 "anomaly_batch_size": 16, "evaluate_during_training_verbose": True, "use_cached_eval_features": True, "sliding_window": True, "vocab_size": 52000, "eval_anomalies": True, "random_generator": 1, "use_rtd_loss": True, "rtd_loss_weight": 50, "rmd_loss_weight": 100, "mlm_loss_weight": 1, "dump_histogram": 0, "eval_anomaly_after": 0, "train_just_generator": 0, "replace_tokens": 0, "extract_scores": 1, "subset_name": subset[0], "extract_repr": 0, # "vanilla_electra": { # "no_masks": masks, # }, # "vanilla_electra": False, "train_document": True, "tokenizer_name": "bert-base-uncased", "tensorboard_dir": f'{tensorboard_dir}/{exp_prefix}/{run_name}', "extract_reps": 0, "weight_decay": weight_decay, "optimizer": "AdamW", "scores_export_path": f"./token_scores/{run_name}/", "generator_config": { "embedding_size": 128, "hidden_size": 16, "num_hidden_layers": 1, }, "discriminator_config": { "hidden_dropout_prob": 0.5, "attention_probs_dropout_prob": 0.5, "embedding_size": 128, "hidden_size": 256, "num_hidden_layers": 4, }, "mlm_lr_ratio": 1, } train_file = f"{data_path}/{dataset_name}/train/{subset[0]}.txt" test_file = f"{data_path}/{dataset_name}/test/{subset[0]}.txt" outlier_file = f"{data_path}/{dataset_name}/test/{subset[0]}-outliers.txt" elif net_name == 'cvdd_Net': # Print CVDD configuration logger.info('Anomaly Score: %s' % cfg.settings['ad_score']) logger.info('Number of attention heads: %d' % cfg.settings['n_attention_heads']) logger.info('Attention size: %d' % cfg.settings['attention_size']) logger.info('Orthogonality regularization hyperparameter: %.3f' % cfg.settings['lambda_p']) logger.info('Temperature alpha annealing strategy: %s' % cfg.settings['alpha_scheduler']) # Initialize CVDD model and set word embedding cvdd = CVDD(cfg.settings['ad_score']) cvdd.set_network(net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], attention_size=cfg.settings['attention_size'], n_attention_heads=cfg.settings['n_attention_heads']) # If specified, load model parameters from already trained model if load_model: cvdd.load_model(import_path=load_model, device=device) logger.info('Loading model from %s.' % load_model) # Train model on dataset cvdd.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], lambda_p=cfg.settings['lambda_p'], alpha_scheduler=cfg.settings['alpha_scheduler'], weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model cvdd.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader) elif net_name == 'cvdd_flow': # Initialize CVDD_Flow model and set word embedding cvdd_flow = CVDD_Flow(cfg.settings['ad_score']) cvdd_flow.set_network( net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], attention_size=cfg.settings['attention_size'], n_attention_heads=cfg.settings['n_attention_heads']) # Train model on dataset cvdd_flow.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], lambda_p=cfg.settings['lambda_p'], alpha_scheduler=cfg.settings['alpha_scheduler'], weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model cvdd_flow.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
def main(): save_dir = args.save_dir + f"/{args.dataset}/{args.width}/{args.hidden_dim}" if not os.path.exists(save_dir): os.makedirs(save_dir) t0 = 0 t1 = 10 p_z0 = torch.distributions.MultivariateNormal( loc=torch.tensor([0.0, 0.0]).to(device), covariance_matrix=torch.tensor([[0.3, 0.0], [0.0, 0.3]]).to(device)) odefunc = CNF(in_out_dim=2, hidden_dim=args.hidden_dim, width=args.width) if device != "cpu": odefunc = odefunc.cuda() optimizer = optim.Adam(odefunc.parameters(), lr=1e-3, weight_decay=0.) x_test, logp_diff_t1_test = get_batch(args.num_samples * 20, args.dataset) # Train model for itr in tqdm(range(args.epochs + 1)): optimizer.zero_grad() x, logp_diff_t1 = get_batch(args.num_samples, args.dataset) loss = calc_loss(odefunc, x, logp_diff_t1, t0, t1, p_z0) loss.backward() optimizer.step() best_loss = np.inf if itr % 100 == 0: z_t, logp_diff_t = odeint( odefunc, (x_test, logp_diff_t1_test), torch.tensor([t1, t0]).type(torch.float32).to(device), atol=1e-5, rtol=1e-5, method='dopri5', ) z_t0, logp_diff_t0 = z_t[-1], logp_diff_t[-1] logp_x = p_z0.log_prob(z_t0).to(device) - logp_diff_t0.view(-1) loss = -logp_x.mean(0) print(f"{itr} Test loss: {loss}") if loss < best_loss: best_loss = loss torch.save(odefunc.state_dict(), f"{save_dir}/best_model.pt") torch.save(odefunc.state_dict(), f"{save_dir}/last_model.pt") plt.figure(figsize=(4, 4), dpi=200) plt.hist2d(*z_t0.detach().cpu().numpy().T, bins=300, density=True, range=[[-2, 2], [-2, 2]]) plt.axis('off') plt.gca().invert_yaxis() plt.margins(0, 0) plt.savefig(save_dir + f"/tgt_itr_{itr:05d}.jpg", pad_inches=0, bbox_inches='tight') plt.close() odefunc.load_state_dict(torch.load(f"{save_dir}/best_model.pt")) # Generate evolution of sampled points z_t0 = p_z0.sample([30000]).to(device) logp_diff_t0 = torch.zeros(30000, 1).type(torch.float32).to(device) z_t, logp_diff_t = odeint( odefunc, (z_t0, logp_diff_t0), torch.tensor(np.linspace(t0, t1, 21)).to(device), atol=1e-5, rtol=1e-5, method='dopri5', ) for (t, z) in zip(np.linspace(t0, t1, 21), z_t.detach().cpu().numpy()): plt.figure(figsize=(4, 4), dpi=200) plt.hist2d(*z.T, bins=300, density=True, range=[[-2, 2], [-2, 2]]) plt.axis('off') plt.gca().invert_yaxis() plt.margins(0, 0) plt.savefig(save_dir + f"/samples_{t:f}.jpg", pad_inches=0, bbox_inches='tight') plt.close() # Generate evolution of density x = np.linspace(-2, 2, 100) y = np.linspace(-2, 2, 100) points = np.vstack(np.meshgrid(x, y)).reshape([2, -1]).T z_t1 = torch.tensor(points).type(torch.float32).to(device) logp_diff_t1 = torch.zeros(z_t1.shape[0], 1).type(torch.float32).to(device) z_t, logp_diff_t = odeint( odefunc, (z_t1, logp_diff_t1), torch.tensor(np.linspace(t1, t0, 21)).to(device), atol=1e-5, rtol=1e-5, method='dopri5', ) for (t, z, logp_diff) in zip(np.linspace(t0, t1, 21), z_t, logp_diff_t): logp = p_z0.log_prob(z) - logp_diff.view(-1) plt.figure(figsize=(4, 4), dpi=200) plt.tricontourf(*z_t1.detach().cpu().numpy().T, np.exp(logp.detach().cpu().numpy()), 200) plt.tight_layout() plt.axis('off') plt.gca().invert_yaxis() plt.margins(0, 0) plt.savefig(save_dir + f"/density_{t:f}.jpg", pad_inches=0, bbox_inches='tight') plt.close()