def main(): logging.basicConfig( level=logging.INFO, format='%(levelname).1s %(module)10.10s:%(lineno)-4d %(message)s') cars = {} logger.info('Pretending to scrape some cars...') for _ in range(10): name = random.choice('abcdefgh') + random.choice('qwerty') data = [random.randrange(10) for _ in range(random.randrange(5, 10))] logger.info(f'Found car {name!r} {data}') cars[name] = data conn = db.get_conn() cur = conn.cursor() for name, data in cars.items(): cur.execute( ''' INSERT INTO cars(name, data, invocation_id, timestamp) VALUES (%s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING id ''', [ name, json.dumps(data), db.get_this_invocation_id(conn), time.time() ]) res = cur.fetchall() if res: [[car_id]] = res logger.info(f'Adding car/{car_id}') else: logger.info(f'Car {name!r} already exists') conn.commit()
def main(): conn = db.get_conn() cur = conn.cursor() cur.execute(''' SELECT id, name, data FROM tasks ''') stats_by_id = {} for id, name, data in cur: data = zlib.decompress(data).decode() task = Task.parse(data) bb = geom.poly_bb(task.border) boosters = collections.Counter(b.code for b in task.boosters) stats = dict(width=bb.x2, height=bb.y2, boosters=dict(boosters)) logger.info(f'stats for task/{id} {name}: {stats}') stats_by_id[id] = json.dumps(stats) logger.info('saving stats for all tasks...') execute_values( cur, ''' UPDATE tasks SET stats=data.stats::json FROM (VALUES %s) AS data(id, stats) WHERE tasks.id=data.id ''', stats_by_id.items()) conn.commit()
def main(): if len(sys.argv) < 2: print('Usage:') print( ' python -m production.solver_runner <solver> [<solver args>...]' ) print(f'where <solver> is one of {ALL_SOLVERS.keys()}') sys.exit(1) conn = db.get_conn() cur = conn.cursor() solver = ALL_SOLVERS[sys.argv[1]](sys.argv[2:]) logger.info(f'Solver scent: {solver.scent()!r}') cur.execute(''' SELECT id FROM tasks ''') problem_ids = [id for [id] in cur] logger.info(f'Problems to solve: {problem_ids}') for problem_id in problem_ids: logger.info('-' * 50) cur.execute('SELECT name, data, extra FROM tasks WHERE id = %s', [problem_id]) [problem_name, task_data, extra] = cur.fetchone() logger.info( f'Solving task/{problem_id} ({problem_name}, {extra["legend"]})...' ) task_data = zlib.decompress(task_data).decode() sr = solver.solve(task_data) if sr.extra: logging.info(f'extra = {sr.extra}') if isinstance(sr.data, interface.Pass): logging.info('Solver passed') elif isinstance(sr.data, interface.Fail): logging.warning('Solver failed') else: logging.info(f'Expected score = {sr.expected_score}, checking ...') res = validate.run(task_data, sr.data) assert res.time is not None, res if sr.expected_score is None: logging.info(f'Actual score = {res.time}') else: if sr.expected_score != res.time: logging.error( f'Actual score = {res.time}, the solver was wrong!!!' ) else: logging.info('Validation ok') logger.info('All done')
def main(): if len(sys.argv) < 2: print('Usage:') print( ' python -m production.solver_runner <solver> [<solver args>...]' ) print(f'where <solver> is one of {ALL_SOLVERS.keys()}') sys.exit(1) logging.basicConfig( level=logging.INFO, format='%(levelname).1s %(module)10.10s:%(lineno)-4d %(message)s') conn = db.get_conn() cur = conn.cursor() solver = ALL_SOLVERS[sys.argv[1]](sys.argv[2:]) logger.info(f'Solver scent: {solver.scent()!r}') cur.execute(''' SELECT problems.id, problems.name FROM problems WHERE problems.name LIKE 'F%' ''') problem_ids = [] for id, name in cur: if solver.supports(solver_interface.ProblemType.from_name(name)): problem_ids.append(id) logger.info(f'Problems to solve: {problem_ids}') for problem_id in problem_ids: logger.info('-' * 50) cur.execute( 'SELECT name, src_data, tgt_data FROM problems WHERE id = %s', [problem_id]) [problem_name, src_data, tgt_data] = cur.fetchone() logger.info(f'Solving problem/{problem_id} ({problem_name})...') if src_data is not None: src_data = zlib.decompress(src_data) if tgt_data is not None: tgt_data = zlib.decompress(tgt_data) sr = solver.solve(problem_name, src_data, tgt_data) logging.info(f'extra: {sr.extra}') if isinstance(sr.trace_data, solver_interface.Pass): logging.info('Solver passed') elif isinstance(sr.trace_data, solver_interface.Fail): logging.warning('Solver failed') else: logging.info('Solver produced a trace, checking with pyjs...') er = pyjs_run_full(src_data, tgt_data, sr.trace_data) logging.info(er) logger.info('All done')
def main(): t = int(time.time()) conn = db.get_conn() cur = conn.cursor() cur.execute(''' SELECT problems.name, traces.id, traces.energy FROM problems JOIN traces ON traces.problem_id = problems.id WHERE problems.name LIKE 'F%' AND traces.status = 'DONE' ''') rows = cur.fetchall() best_by_problem = {} for problem_name, trace_id, energy in rows: assert energy is not None k = (energy, trace_id) if problem_name not in best_by_problem: best_by_problem[problem_name] = k else: if k < best_by_problem[problem_name]: best_by_problem[problem_name] = k print(best_by_problem) with open(utils.project_root() / 'outputs' / f'submission_{t}.manifest', 'w') as fout: for problem_name, (score, trace_id) in sorted(best_by_problem.items()): fout.write(f'{problem_name} {score:>15} /trace/{trace_id}\n') path = (utils.project_root() / 'outputs' / f'submission_{t}.zip') z = zipfile.ZipFile(path, 'w') for problem_name, (score, trace_id) in sorted(best_by_problem.items()): print(problem_name) cur.execute('SELECT data FROM traces WHERE id = %s', [trace_id]) [data] = cur.fetchone() data = zlib.decompress(data) z.writestr(zipfile.ZipInfo(f'{problem_name}.nbt'), data) for name in data_files.full_names(): if name not in best_by_problem: print(name, 'from defaults') z.writestr(zipfile.ZipInfo(f'{name}.nbt'), data_files.full_default_trace(name)) z.close() with open(path, 'rb') as fin: h = hashlib.sha256(fin.read()).hexdigest() print(f'File: {path}') print(f'SHA256: {h}')
def main(): logging.basicConfig( level=logging.INFO, format='%(levelname).1s %(module)10.10s:%(lineno)-4d %(message)s') conn = db.get_conn() cur = conn.cursor() legends = {} for part_number, part_name in (1, 'initial'), (2, 'teleports'), (3, 'clones'): with ZipFile(utils.project_root() / 'tasks' / f'part-{part_number}-{part_name}.zip') as z: with z.open(f'part-{part_number}-legend.txt') as fin: for line in fin: name, legend = line.decode().split(' - ', maxsplit=1) legends[name] = legend.rstrip() for filename in z.namelist(): if filename.endswith('-legend.txt'): continue m = re.match(r'(prob-\d{3}).desc$', filename) assert m, filename name = m.group(1) with z.open(filename, 'r') as fin: task = fin.read().decode() data = zlib.compress(task.encode()) extra = dict(legend=legends[name]) cur.execute( ''' INSERT INTO tasks( name, data, extra, invocation_id, time) VALUES (%s, %s, %s, %s, NOW()) ON CONFLICT DO NOTHING RETURNING id ''', [ name, data, json.dumps(extra), db.get_this_invocation_id(conn) ]) res = cur.fetchall() if res: [[task_id]] = res logger.info(f'Uploaded {name!r} as /task/{task_id}') else: logger.info(f'Task {name!r} already exists') db.record_this_invocation(conn, status=db.Stopped()) conn.commit()
def main(): t = int(time.time()) conn = db.get_conn() cur = conn.cursor() cur.execute(''' SELECT tasks.name, solutions.id, solutions.score, solutions.scent FROM tasks JOIN solutions ON solutions.task_id = tasks.id WHERE solutions.status = 'DONE' AND tasks.name LIKE 'prob-%' ''') rows = cur.fetchall() best_by_task = {} for task_name, sol_id, score, scent in rows: assert score is not None k = (score, sol_id, scent) if task_name not in best_by_task: best_by_task[task_name] = k else: if k < best_by_task[task_name]: best_by_task[task_name] = k print(best_by_task) with open(utils.project_root() / 'outputs' / f'submission_{t}.manifest', 'w') as fout: for task_name, (score, sol_id, scent) in sorted(best_by_task.items()): fout.write(f'{task_name} {score:>15} /sol/{sol_id} by {scent}\n') path = (utils.project_root() / 'outputs' / f'submission_{t}.zip') z = zipfile.ZipFile(path, 'w') for task_name, (score, sol_id, scent) in sorted(best_by_task.items()): print(task_name) cur.execute('SELECT data FROM solutions WHERE id = %s', [sol_id]) [data] = cur.fetchone() data = zlib.decompress(data).decode() z.writestr(zipfile.ZipInfo(f'{task_name}.sol'), data) z.close() with open(path, 'rb') as fin: h = hashlib.sha256(fin.read()).hexdigest() print(f'File: {path}') print(f'SHA256: {h}')
def main(): conn = db.get_conn() while True: block = lambda_chain.get_block_info() upload_current_task(conn, block) f = utils.project_root() / 'outputs' / f'block-{block.number:04d}.cond' f.write_text(block.puzzle) logging.info(f'Block puzzle saved to {f}') try: task = puzzle_solver.solve(Puzzle.parse(block.puzzle)) task = str(task) logging.info(f'Validating puzzle solution...') result = validate.puz(block.puzzle, task) logging.info(result) assert result == 'ok' except KeyboardInterrupt: raise except: traceback.print_exc() db.record_this_invocation(conn, status=db.KeepRunning(60)) conn.commit() logging.info('waiting and retrying...') time.sleep(20) continue block_submitted = False while True: new_block = lambda_chain.get_block_info() logging.info(f'block age: {int(new_block.age_in_seconds)}s') if new_block.number != block.number: logging.info('new block appeared ' + '-' * 30) break if not block_submitted: sol = find_best_solution(conn, f'block-{block.number:04d}') if sol is not None and new_block.age_in_seconds > 850: lambda_chain.submit(block.number, solution=sol, task=task) block_submitted = True logging.info('waiting...') db.record_this_invocation(conn, status=db.KeepRunning(60)) conn.commit() time.sleep(20)
def main(): logging.basicConfig( level=logging.INFO, format='%(levelname).1s %(module)10.10s:%(lineno)-4d %(message)s') conn = db.get_conn() cur = conn.cursor() for name in sorted(data_files.full_names()): if not name.startswith('FR'): continue logging.info(name) src_data, tgt_data = data_files.full_problem(name) tgt_data = None stats = {} if src_data is not None: m = Model.parse(src_data) num_full_voxels = 0 for pos in m.enum_voxels(): if m[pos]: num_full_voxels += 1 stats.update(R=m.R, src_size=num_full_voxels) src_data = zlib.compress(src_data) if tgt_data is not None: m = Model.parse(tgt_data) num_full_voxels = 0 for pos in m.enum_voxels(): if m[pos]: num_full_voxels += 1 stats.update(R=m.R, tgt_size=num_full_voxels) tgt_data = zlib.compress(tgt_data) logging.info(stats) name = name.replace('FR', 'ZD') logging.info(name) extra = {} cur.execute( ''' INSERT INTO problems( name, src_data, tgt_data, stats, extra, invocation_id, timestamp) VALUES (%s, %s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING id ''', [ name, src_data, tgt_data, json.dumps(stats), json.dumps(extra), db.get_this_invocation_id(conn), time.time() ]) res = cur.fetchall() if res: [[model_id]] = res logger.info(f'Recorded as model/{model_id}') else: logger.info(f'Model {name!r} already exists') conn.commit()
def main(): logging.basicConfig( level=logging.INFO, format='%(levelname).1s %(module)10.10s:%(lineno)-4d %(message)s') conn = db.get_conn() cur = conn.cursor() cur.execute(''' SELECT cars.name, fuels.id, fuels.score, COUNT(fuel_submissions.id), COUNT(fuel_submissions.data) FROM fuels JOIN cars ON fuels.car_id = cars.id LEFT OUTER JOIN fuel_submissions ON fuel_submissions.fuel_id = fuels.id WHERE fuels.score IS NOT NULL GROUP BY cars.name, fuels.id ''') # For each car, take all fuels that don't have failed submissions, # and submit the one with the best score # (unless it already has a successful submission). fuels_by_car_name = {} for car_name, fuel_id, fuel_score, num_submissions, num_successful_submissions in cur: if num_submissions > 0 and num_successful_submissions == 0: continue # failed submission fuels = fuels_by_car_name.setdefault(car_name, []) fuels.append(FuelInfo( id=fuel_id, score=fuel_score, has_successful_submission=bool(num_successful_submissions))) done = False def signal_handler(sig, frame): nonlocal done done = True logging.warning('Caught Ctrl-C, will finish current item and exit.') logging.warning('To abort immediately, hit Ctrl-C again.') signal.signal(signal.SIGINT, old_signal_handler) old_signal_handler = signal.signal(signal.SIGINT, signal_handler) for car_name, fuels in fuels_by_car_name.items(): if done: break fuel = max(fuels, key=lambda f: f.score) if fuel.has_successful_submission: logging.info(f'Best fuel for car {car_name!r} already submitted') continue cur.execute('SELECT data FROM fuels WHERE id = %s', [fuel.id]) [fuel_data] = cur.fetchone() submission_data, extra = submit(car_name, fuel_data) if submission_data is not None: submission_data = json.dumps(submission_data) extra = json.dumps(extra) cur.execute(''' INSERT INTO fuel_submissions( data, extra, fuel_id, invocation_id, timestamp) VALUES (%s, %s, %s, %s, %s) RETURNING id ''', [submission_data, extra, fuel.id, db.get_this_invocation_id(conn), time.time()]) [submission_id] = cur.fetchone() logging.info(f'Submission recorded as fuel_sub/{submission_id}') conn.commit()
def get_conn(): global _conn if _conn is None: _conn = db.get_conn() return _conn
def main(): logging.basicConfig( level=logging.INFO, format='%(levelname).1s %(module)10.10s:%(lineno)-4d %(message)s') conn = db.get_conn() attempts_by_car_id = get_attempts(conn) def want_solve(attempts): return not attempts car_ids = [ car_id for car_id, attempts in attempts_by_car_id.items() if want_solve(attempts)] logging.info(f'Found {len(car_ids)} cars to solve') # to reduce collisions when multiple solvers are working in parallel random.shuffle(car_ids) num_workers = multiprocessing.cpu_count() output_queue = multiprocessing.SimpleQueue() input_queues = [multiprocessing.SimpleQueue() for _ in range(num_workers)] workers = [] for i, iq in enumerate(input_queues): log_path = utils.project_root() / 'outputs' / f'solver_worker_{i:02}.log' logging.info(f'Worker logging to {log_path}') w = multiprocessing.Process(target=work, args=(i, log_path, iq, output_queue)) w.start() available_workers = set(range(num_workers)) while True: if available_workers and car_ids: car_id = car_ids.pop() attempts = get_attempts(conn, car_id=car_id)[car_id] if want_solve(attempts): cur = conn.cursor() cur.execute('SELECT data FROM cars WHERE id = %s', [car_id]) [car_data] = cur.fetchone() worker_index = available_workers.pop() input_queues[worker_index].put(InputEntry(car_id=car_id, car_data=car_data)) logging.info(f'car/{car_id} goes to worker {worker_index}') else: logging.info(f'Skipping car/{car_id}') else: if len(available_workers) == num_workers: break output_entry = output_queue.get() assert output_entry.worker_index not in available_workers available_workers.add(output_entry.worker_index) logging.info( f'Got fuel for car/{output_entry.car_id}, ' f'score={output_entry.result.score} ' f'from worker {output_entry.worker_index}') put_fuel(conn, output_entry.car_id, output_entry.result) conn.commit() logging.info('All done, joining workers...') for iq in input_queues: iq.put(None) for w in workers: join()
def main(): args = parse_args() conn = db.get_conn() cur = conn.cursor() solver = ALL_SOLVERS[args.solver](args.solver_args) logger.info(f'Solver scent: {solver.scent()!r}') seen_tasks = set() def get_tasks(): # Select tasks that don't have solutions with our scent. cur.execute( ''' SELECT tasks.id FROM tasks LEFT OUTER JOIN ( SELECT task_id AS solution_task_id FROM solutions WHERE scent = %s ) AS my_solutions ON solution_task_id = tasks.id WHERE solution_task_id IS NULL AND NOT tasks.obsolete AND tasks.name LIKE %s ''', [solver.scent(), 'block-%' if args.only_chain else '%']) task_ids = [id for [id] in cur if id not in seen_tasks] seen_tasks.update(task_ids) logging.info(f'{len(task_ids)} tasks to solve: {task_ids}') # to reduce collisions when multiple solvers are working in parallel random.shuffle(task_ids) # task_ids.sort(reverse=True) return task_ids task_ids = get_tasks() num_workers = args.jobs output_queue = multiprocessing.Queue() input_queues = [multiprocessing.Queue() for _ in range(num_workers)] workers = [] for i, iq in enumerate(input_queues): log_path = utils.project_root( ) / 'outputs' / f'solver_worker_{i:02}.log' logging.info(f'Worker logging to {log_path}') w = multiprocessing.Process(target=work, args=(i, log_path, iq, output_queue)) w.start() available_workers = set(range(num_workers)) cur = conn.cursor() while True: while available_workers and task_ids: task_id = task_ids.pop() cur.execute( 'SELECT COUNT(*) FROM solutions WHERE task_id = %s AND scent = %s', [task_id, solver.scent()]) [num_attempts] = cur.fetchone() if num_attempts == 0: cur.execute('SELECT name, data FROM tasks WHERE id = %s', [task_id]) [task_name, task_data] = cur.fetchone() task_data = zlib.decompress(task_data).decode() worker_index = available_workers.pop() input_queues[worker_index].put( InputEntry(solver=solver, task_id=task_id, task_name=task_name, task_data=task_data)) logging.info(f'task/{task_id} goes to worker {worker_index}') else: logging.info( f'task/{task_id} already done by another worker, skipping') logging.info(f'{len(task_ids)} tasks remaining') #if len(available_workers) == num_workers: # break cont = False while True: if available_workers and not task_ids: logging.info( f'No more new tasks, {num_workers - len(available_workers)} still in progress, looking for more...' ) task_ids = get_tasks() if task_ids: cont = True break db.record_this_invocation( conn, status=db.KeepRunning(40), extra=dict(remaining_tasks=len(task_ids) + num_workers - len(available_workers))) conn.commit() try: output_entry = output_queue.get(timeout=20) break except queue.Empty: logging.info('waiting...') if cont: continue assert output_entry.worker_index not in available_workers available_workers.add(output_entry.worker_index) logging.info(f'Got solution for task/{output_entry.task_id}, ' f'score={output_entry.result.score} ' f'from worker {output_entry.worker_index}') if args.dry_run: logging.info(f'Skip saving because dry-run') else: put_solution(conn, output_entry.task_id, output_entry.result) conn.commit() db.record_this_invocation(conn, status=db.Stopped()) conn.commit() logging.info('All done, joining workers...') for iq in input_queues: iq.put(None) for w in workers: join()
def main(): logging.basicConfig( level=logging.INFO, format='%(levelname).1s %(module)10.10s:%(lineno)-4d %(message)s') args = parse_args() conn = db.get_conn() cur = conn.cursor() solver = ALL_SOLVERS[args.solver](args.solver_args) logger.info(f'Solver scent: {solver.scent()!r}') cur.execute( ''' SELECT problems.id, problems.name FROM problems LEFT OUTER JOIN ( SELECT problem_id AS trace_problem_id FROM traces WHERE scent = %s ) AS my_traces ON trace_problem_id = problems.id WHERE ((problems.name LIKE 'F%%') OR (problems.name LIKE 'Z%%')) AND trace_problem_id IS NULL AND (problems.stats->>'R')::integer <= %s ''', [solver.scent(), args.max_size]) problem_ids = [] for id, name in cur: if solver.supports(solver_interface.ProblemType.from_name(name)): problem_ids.append(id) logging.info(f'{len(problem_ids)} problems to solve: {problem_ids}') # to reduce collisions when multiple solvers are working in parallel random.shuffle(problem_ids) #problem_ids.sort(reverse=True) num_workers = args.jobs # num_workers = 1 output_queue = multiprocessing.SimpleQueue() input_queues = [multiprocessing.SimpleQueue() for _ in range(num_workers)] workers = [] for i, iq in enumerate(input_queues): log_path = utils.project_root( ) / 'outputs' / f'solver_worker_{i:02}.log' logging.info(f'Worker logging to {log_path}') w = multiprocessing.Process(target=work, args=(i, log_path, iq, output_queue)) w.start() available_workers = set(range(num_workers)) cur = conn.cursor() while True: if available_workers and problem_ids: problem_id = problem_ids.pop() cur.execute( 'SELECT COUNT(*) FROM traces WHERE problem_id = %s AND scent = %s', [problem_id, solver.scent()]) [num_attempts] = cur.fetchone() if num_attempts == 0: cur.execute( 'SELECT name, src_data, tgt_data FROM problems WHERE id = %s', [problem_id]) [problem_name, src_data, tgt_data] = cur.fetchone() if src_data is not None: src_data = zlib.decompress(src_data) if tgt_data is not None: tgt_data = zlib.decompress(tgt_data) worker_index = available_workers.pop() input_queues[worker_index].put( InputEntry(solver=solver, problem_id=problem_id, problem_name=problem_name, src_data=src_data, tgt_data=tgt_data)) logging.info( f'problem/{problem_id} goes to worker {worker_index}') else: logging.info(f'Skipping problem/{problem_id}') logging.info(f'{len(problem_ids)} remaining') else: if len(available_workers) == num_workers: break output_entry = output_queue.get() assert output_entry.worker_index not in available_workers available_workers.add(output_entry.worker_index) logging.info(f'Got trace for problem/{output_entry.problem_id}, ' f'energy={output_entry.result.energy} ' f'from worker {output_entry.worker_index}') if args.dry_run: logging.info(f'Skip saving because dry-run') else: put_trace(conn, output_entry.problem_id, output_entry.result) conn.commit() logging.info('All done, joining workers...') for iq in input_queues: iq.put(None) for w in workers: join()
def interactive(task_number): task_data = utils.get_problem_raw(task_number) use_db = task_number <= 999 if use_db: conn = db.get_conn() cur = conn.cursor() cur.execute( ''' SELECT id, data FROM tasks WHERE name = %s ''', [f'prob-{task_number:03d}']) [task_id, task_data_db] = cur.fetchone() task_data_db = zlib.decompress(task_data_db).decode() assert task_data_db == task_data task = GridTask(Task.parse(task_data)) game = Game(task) score = None with contextlib.closing(Display(game)) as display: code, c = '', '' display.draw_initial(game) while not score: display.draw(game, f'lastchar = {code} {c!r}') code = display.stdscr.getch() action = None c = chr(code).upper() if c in '\x1B': break if c in Action.SIMPLE: action = Action.simple(c) # to perform complex action type it without spaces: B(1,-1) elif c in Action.PARAM: while c[-1] != ')': code = display.stdscr.getch() c = c + chr(code).upper() action = Action.parameterized(c) if display.current == 0: botcount = len(game.bots) if action: try: game.apply_action(action, display.current) except InvalidActionException as exc: display.draw_error(str(exc)) else: display.current = (display.current + 1) % botcount score = game.finished() if score is not None: print(f'Score: {score}') result = validate_replay(task_data, score, game.get_actions()) print(result) if use_db: submit_replay(conn, task_id, result) else: mock_solutions = utils.project_root( ) / 'outputs' / 'mock_solutions' mock_solutions.mkdir(parents=True, exist_ok=True) with mock_solutions / f'prob-{task_number}.sol' as fin: fin.write_text(result.solution)