def main(): import sys sys.setrecursionlimit(32768) random.seed(flutes.__MAGIC__) # wabbits = [(-1, '?'), (0, 'R')] wabbits = [(-1, '?'), (0, 'R'), (0, 'R')] wabbits = [(-1, 'R'), (0, 'R'), (1, '?')] wabbits = [(-1, '?'), (0, 'R'), (1, 'R')] p_numerator = 1 p_denominator = 5 # with open("data/sl_wascally_wabbits.pkl", "rb") as f: # wabbits, p_numerator, p_denominator = pickle.load(f) n = 10000 types = [random.choice("?RG") for _ in range(n)] # parent = [-1] + [random.randint(0, idx) for idx in range(n - 1)] parent = [-1] + list(range(n - 1)) # parent = [-1] + list(range(n - 1)) wabbits = list(zip(parent, types)) p_numerator = 49 p_denominator = 100 # print(wabbits) # print(p_numerator, p_denominator) # with flutes.work_in_progress(): # print(solution(wabbits, p_numerator, p_denominator)) with flutes.work_in_progress(): print(solution_frac(wabbits, p_numerator, p_denominator)) with flutes.work_in_progress(): print(solution_frac_builtin(wabbits, p_numerator, p_denominator)) with flutes.work_in_progress(): print(solution_n2(wabbits, p_numerator, p_denominator)) with flutes.work_in_progress(): print(solution_brute(wabbits, p_numerator, p_denominator))
def main(): with open("data/sl_task_genie.pkl", "rb") as f: tasks, wishes = pickle.load(f) with flutes.work_in_progress(): print(solution_n2(tasks, wishes)) with flutes.work_in_progress(): print(solution_n3(tasks, wishes)) with flutes.work_in_progress(): print(solution_wrong(tasks, wishes))
def main(): random.seed(flutes.__MAGIC__) targets = [(0, 2), (2, 3), (8, 3)] with open("data/sl_explodium.pkl", "rb") as f: targets, = pickle.load(f) # targets = [(0, 8), (2, 8), (7, 4), (8, 4), (10, 2)] with flutes.work_in_progress(): print(solution(targets)) with flutes.work_in_progress(): print(solution_brute(targets))
def main(): random.seed(flutes.__MAGIC__) a = "SRR" b = "PPS" n = 1000 a = "".join(random.choice("RSP") for _ in range(n)) b = "".join(random.choice("RSP") for _ in range(n)) with flutes.work_in_progress(): print(solution(a, b)) with flutes.work_in_progress(): print(solution_ref(a, b)) with flutes.work_in_progress(): print(solution_rev(a, b))
def main(): random.seed(flutes.__MAGIC__) n = 10 rs = [('I', 3), ('I', 3), ('O', 0), ('I', 4), ('I', 2), ('O', 2), ('I', 2)] with open("data/sl_grand_hotel.pkl", "rb") as f: n, rs = pickle.load(f) n = int(1e8) rs = [] reservations = set() r_cnt = 0 for idx in range(100000): choices = [] if len(reservations) > 0: choices.append('O') if len(reservations) < 100: choices.append('I') if random.choice(choices) == 'I': l = random.randint(1, n // 100) rs.append(('I', l)) reservations.add(r_cnt) r_cnt += 1 else: idx = random.choice(list(reservations)) rs.append(('O', idx)) reservations.remove(idx) with flutes.work_in_progress(): print(solution(n, rs))
def main(): with open("data/sl_hardcore_parkour.pkl", "rb") as f: ps, = pickle.load(f) n = 1000 ps = [list(range(1, 20 + 1)) for _ in range(n)] # print(ps) with flutes.work_in_progress(): print(solution(ps))
def main(): random.seed(flutes.__MAGIC__) with open("data/sl_doubling_danny.pkl", "rb") as f: data = pickle.load(f) # print(data) # print(solution(*data)) m = 2**2203 - 1 # a = [m - random.randint(0, 10000) for _ in range(1000)] a = [random.randint(0, m - 1) for _ in range(500)] with flutes.work_in_progress(): print(solution_10(a, m)) with flutes.work_in_progress(): print(solution(a, m)) with flutes.work_in_progress(): print(solution_2(a, m))
def main(): random.seed(19260817) with flutes.work_in_progress(): n = 5000 k = 5 masses = [random.randint(0, 10000) for _ in range(n)] locations = [ tuple(random.randint(-10000, 10000) for _ in range(k)) for _ in range(n) ] # with open("/Users/kanari/Downloads/sl_spacedogs.pkl", "rb") as f: # masses, locations = pickle.load(f) # masses = [2, 5, 4] # locations = [(1, 4), (3, 1), (2, 6)] with flutes.work_in_progress(): print(solution(masses, locations)) with flutes.work_in_progress(): print(solution_truth(masses, locations))
def main(): sys.setrecursionlimit(32768) random.seed(flutes.__MAGIC__) n = 3 proposition = "A AND NOT NOT B OR C AND NOT(A OR B)" with open("data/sl_satisfaction.pkl", "rb") as f: n, proposition = pickle.load(f) n = 26 proposition = str(generate(n, 500)) proposition = proposition.replace("and", "AND").replace("or", "OR").replace( "not", "NOT") print(proposition) with flutes.work_in_progress(): print(solution(n, proposition)) with flutes.work_in_progress(): print(solution_brute(n, proposition))
def main(): random.seed(flutes.__MAGIC__) s = "297891" n = 100 s = "".join(chr(48 + random.randint(0, 9)) for _ in range(n)) # s = "000001" with flutes.work_in_progress(): print(solution(s))
def main(): random.seed(flutes.__MAGIC__) skills = {'Andy': 7, 'Novak': 5, 'Roger': 3, 'Rafael': 2} with open("data/sl_gone_to_seed.pkl", "rb") as f: skills, = pickle.load(f) n = 16 values = [random.randint(1, 20) for _ in range(n)] names = ['Andy'] + [chr(65 + x) for x in range(1, n)] skills = dict(zip(names, values)) print(skills) with flutes.work_in_progress(): print(solution(skills)) with flutes.work_in_progress(): print(solution_old(skills)) with flutes.work_in_progress(): print(solution_search(skills))
def main(): random.seed(flutes.__MAGIC__) a = [5, 3, 10, 6] b = [9, 7, 12] n = 1000 m = 1000 a = [random.randint(0, 99999) for _ in range(n)] b = [random.randint(0, 99999) for _ in range(m)] # print(a) # print(b) with flutes.work_in_progress(): print(solution(a, b)) with flutes.work_in_progress(): print(solution_reverse(a, b)) with flutes.work_in_progress(): print(solution_full(a, b)) with flutes.work_in_progress(): print(solution_brute(a, b))
def main() -> None: if len(sys.argv) < 2: print(f"Usage: python {sys.argv[0]} [file]") sys.exit(1) path = sys.argv[1] with flutes.work_in_progress("Read file"): with open(path) as f: sentences = [] for line in f: sentences.append(line) if len(sentences) >= 100000: break with flutes.work_in_progress("Parallel"): with flutes.safe_pool(processes=4, state_class=WordCounter) as pool_stateful: for _ in pool_stateful.imap_unordered(WordCounter.count_words, sentences, chunksize=1000): pass parallel_word_counter: CounterT[str] = Counter() with flutes.work_in_progress("Get states"): states = pool_stateful.get_states() for state in states: parallel_word_counter.update(state.word_cnt) with flutes.work_in_progress("Naive parallel"): naive_word_counter: CounterT[str] = Counter() data_chunks = flutes.chunk(1000, sentences) with flutes.safe_pool(processes=4) as pool: for counter in pool.imap_unordered(count_words, data_chunks): naive_word_counter.update(counter) with flutes.work_in_progress("Sequential"): seq_word_counter: CounterT[str] = Counter() for sent in sentences: seq_word_counter.update(word.lower() for word in sent.split()) assert seq_word_counter == naive_word_counter == parallel_word_counter
def main(): random.seed(flutes.__MAGIC__) with open("data/sl_snakes_and_ladders.pkl", "rb") as f: n, snakes, ladders = pickle.load(f) n, snakes, ladders = gen_data(10000, 100, 100) with flutes.work_in_progress(): print(solution(n, snakes, ladders)) with flutes.work_in_progress(): print(solution_unoptimized(n, snakes, ladders)) n = 10000 snakes = [] ladders = [] for i in range(80): snakes.append((n - 200 + i + (i // 5), i + 1 + (i // 5))) with flutes.work_in_progress(): print(solution(n, snakes, ladders)) with flutes.work_in_progress(): print(solution_unoptimized(n, snakes, ladders)) # with flutes.work_in_progress(): # print(solution_brute(n, snakes, ladders)) # with flutes.work_in_progress(): # print(solution_iter(n, snakes, ladders)) return cases = 0 while True: n, snakes, ladders = gen_data(100000, 100, 100) out = solution(n, snakes, ladders) ans = solution_unoptimized(n, snakes, ladders) if out != ans: breakpoint() else: cases += 1 print(f"Passed case {cases}")
def main(args) -> None: files = [(args.output_dir, args.input_dir / f"{i}.pkl") for i in range(0, 10000, 100)] total_map = {} with flutes.work_in_progress("Parallel"): with flutes.safe_pool(processes=args.njobs, state_class=Worker) as pool_stateful: for idx, _ in enumerate( pool_stateful.imap_unordered(Worker.merge, files, chunksize=1)): flutes.log(f"Processed {(idx + 1)} files")
def main(args) -> None: Proc = LegacyFilter if args.legacy else Filter if args.target == "paper": processing = Proc.filter_ids_complete # gathers pid, in/out cite pids elif args.target == "citation": processing = Proc.filter_ids_text # gathers pids if args.valid_citations is not None and args.valid_citations.exists(): with args.valid_citations.open("rb") as f: d = pickle.load(f) dict_valid_citations = {k: True for _, pids in d.items() for k in pids} del d else: dict_valid_citations = {} files = [ (f, dict_valid_citations, args.min_cite, args.max_cite, args.seed) for f in list(args.input_dir.glob("*")) ] with flutes.work_in_progress("Parallel"): total_results = defaultdict(list) with flutes.safe_pool(processes=args.njobs, state_class=Proc) as pool_stateful: for idx, _ in enumerate( pool_stateful.imap_unordered(processing, files, chunksize=10) ): flutes.log(f"Processed {(idx + 1)} files") with flutes.work_in_progress("Get states"): states = pool_stateful.get_states() for state in states: total_results.update(state.results) with args.output_file.open("wb") as f: # Dict[batchnum, List[obj]] pickle.dump(total_results, f)
def main(): random.seed(flutes.__MAGIC__) sys.setrecursionlimit(32768) qs = [(3, 2, 5, 6)] with open("data/sl_paper_cut.pkl", "rb") as f: qs, = pickle.load(f) with flutes.work_in_progress(): print(solution(qs)) with flutes.work_in_progress(): print(solution_brute(qs)) # import numpy as np # arr = np.zeros((100, 100)) # a, b = 5, 18 # ok = [] # for x in range(1, arr.shape[0]): # for y in range(1, arr.shape[1]): # if check(a, b, x, y): # arr[x, y] = 1 # ok.append((x, y)) # plt.imshow(arr) # plt.show() # print(ok) for _ in tqdm.trange(1000): a = random.randint(1, 20) b = random.randint(1, 20) c = random.randint(1, 100) d = random.randint(1, 100) ans = check(a, b, c, d) out = guess(a, b, c, d) if ans != out: print(f"{a, b, c, d}, expected: {ans}, received: {out}") guess(a, b, c, d)
def main(): random.seed(flutes.__MAGIC__) dice_vals = [1, 2, 3] d = 1 r = 2 with open("data/sl_dicey_situation.pkl", "rb") as f: dice_vals, d, r = pickle.load(f) dice_vals = [random.randint(1, 20) for _ in range(4)] d = 8 r = 8 # dice_vals = [2, 3, 4, 5] # d = 8 # r = 8 print(dice_vals, d, r) with flutes.work_in_progress(): print(solution(dice_vals, d, r))
def test_work_in_progress() -> None: with flutes.work_in_progress("abc"): time.sleep(0.1)
def main(args) -> None: pids_with_text = set() if args.valid_pids and args.valid_pids.exists(): pids_with_text = set() with args.valid_pids.open("rb") as f: data = pickle.load(f) buf = set() if args.mode == "citation": for k, d in tqdm(enumerate(data.values()), ncols=88, ascii=True): buf = buf.union( set([pid for i in d for pid in i[1] + i[2]])) if k % 500 == 0: pids_with_text = pids_with_text.union(buf) buf = set() elif args.mode == "paper": for k, d in tqdm(data.items(), ncols=88, ascii=True): buf = buf.union(set([i[0] for i in d])) if k % 500 == 0: pids_with_text = pids_with_text.union(buf) buf = set() # remaining one pids_with_text = pids_with_text.union(buf) flutes.log(f"# of valid pids to consider: {len(pids_with_text)}") if args.legacy: # glob takes more time than this? files = ((args.input_dir / f"{i}.jsonl.gz", pids_with_text) for i in range(10000)) Proc = LegacyFilter else: files = ((args.input_dir / f"pdf_parses_{i}.jsonl.gz", pids_with_text) for i in range(100)) Proc = Filter with flutes.work_in_progress("Parallel"): total_map = {} with flutes.safe_pool(processes=args.njobs, state_class=Proc) as pool_stateful: for idx, _ in enumerate( pool_stateful.imap_unordered(Proc.make_map, files, chunksize=10)): flutes.log(f"Processed {(idx + 1)} files") with flutes.work_in_progress("Get states"): states = pool_stateful.get_states() for state in states: # TODO: Incorporate incite number total_map.update(state.results) flutes.log(f"Total map size: {len(total_map)}") with args.output.open("w") as f: for k, v in total_map.items(): print(k, v[0], v[1], sep="\t", file=f) flutes.log(f"Dumped to {args.output}")