def fetch_entries_into_database(dbconn, entries): """Fetch a set of entries from the remote OEIS database and store the results in the database. The 'entries' parameter contains a number of OEIS IDs. This function can handle a large number of entries, up to the entire size of the OEIS database. Entries are processed in randomized batches. We use a pool of worker threads to perform the actual fetches. This enhances fetch performance (in terms of fetche-per-second) dramatically. Typical fetch performance is about 20 fetches per second. The responses of each batch of entries are processed by the 'process_responses' function defined above. """ FETCH_BATCH_SIZE = 500 # 100 -- 1000 are reasonable NUM_WORKERS = 20 # 10 -- 20 are reasonable SLEEP_AFTER_BATCH = 2.0 # [seconds] entries = set(entries) # make a copy, and ensure it is a set. with start_timer(len(entries)) as timer, concurrent.futures.ThreadPoolExecutor(NUM_WORKERS) as executor: while len(entries) > 0: batch_size = min(FETCH_BATCH_SIZE, len(entries)) batch = random.sample(entries, batch_size) logger.info("Fetching data using {} {} for {} out of {} entries ...".format(NUM_WORKERS, "worker" if NUM_WORKERS == 1 else "workers", batch_size, len(entries))) with start_timer() as batch_timer: # Execute fetches in parallel. responses = list(executor.map(safe_fetch_remote_oeis_entry, batch)) logger.info("{} fetches took {} seconds ({:.3f} fetches/second).".format(batch_size, batch_timer.duration_string(), batch_size / batch_timer.duration())) # Process the responses by updating the database. processed_entries = process_responses(dbconn, responses) entries -= processed_entries # Calculate and show estimated-time-to-completion. logger.info("Estimated time to completion: {}.".format(timer.etc_string(work_remaining = len(entries)))) # Sleep if we need to fetch more if len(entries) > 0: logger.info("Sleeping for {:.1f} seconds ...".format(SLEEP_AFTER_BATCH)) time.sleep(SLEEP_AFTER_BATCH) logger.info("Fetched {} entries in {}.".format(timer.total_work, timer.duration_string()))
def vacuum_database(dbconn): """Perform a VACUUM command on the database.""" with start_timer() as timer: logger.info("Initiating VACUUM on database ...") dbconn.execute("VACUUM;") logger.info("VACUUM done in {}.".format(timer.duration_string()))
def database_update_cycle(database_filename): """Perform a single cycle of the database update loop.""" with start_timer() as timer: highest_oeis_id = find_highest_oeis_id( ) # Check OEIS server for highest entry ID. with close_when_done(sqlite3.connect(database_filename)) as dbconn: ensure_database_schema_created(dbconn) make_database_complete( dbconn, highest_oeis_id ) # Make sure we have all entries (full fetch on first run). update_database_entries_randomly( dbconn, highest_oeis_id // 1000) # Refresh 0.1 % of entries randomly. update_database_entries_by_priority( dbconn, highest_oeis_id // 200) # Refresh 0.5 % of entries by priority. update_database_entries_for_nonzero_time_window( dbconn ) # Make sure we have t1 != t2 for all entries (full fetch on first run). consolidate_database_monthly(database_filename, remove_stale_files_flag=False) logger.info("Full database update cycle took {}.".format( timer.duration_string()))
def open(self): # print("websocket opened") self.connection = False #启动websocket校验计时器,一定时间不发送token验证,自动断开连接 self.start_timer = start_timer() self.ws_loop = asyncio.get_event_loop() self.start_timer.start(self.close, self.ws_loop)
def play_round(self, start_score, debug=False): #mocked to show some difference to ensure things are working self.current_score_value = start_score print(self) self.start_time = timer.start_timer() while not self.round_ended(self): self.current_time_value = timer.increment_timer(self) self.check_for_score(debug) print(self) print(self) return self.current_status()
def read_catalog_files(glob_pattern): sequence_name_to_type = { "PolynomialSequence": PolynomialSequence, "RecurrentSequence": RecurrentSequence, "TreeCountSequence": TreeCountSequence, "CountDivisorsSequence": CountDivisorsSequence, "EulerPhiSequence": EulerPhiSequence, "PrimeSequence": PrimeSequence, "SumDivisorsSequence": SumDivisorsSequence, "PrimePiSequence": PrimePiSequence, "TwinPrimeSequence": TwinPrimeSequence, "SmallestDivisorSequence": SmallestDivisorSequence, } with start_timer() as timer: catalog = {} filenames = sorted(glob.glob(glob_pattern)) for filename in filenames: logger.info("Fetching catalog data from file '{}' ...".format(filename)) with open(filename) as f: oeis_catalog = json.load(f) for (oeis_id_string, sequence_name, sequence_args) in oeis_catalog: assert oeis_id_string.startswith("A") oeis_id = int(oeis_id_string[1:]) if oeis_id in catalog: logger.error( "A{:06d} in file {!r} previously defined as {!r}, skipping.".format( oeis_id, filename, catalog[oeis_id] ) ) continue sequence_type = sequence_name_to_type[sequence_name] sequence = sequence_type(*sequence_args) catalog[oeis_id] = sequence catalog = OrderedDict(sorted(catalog.items())) logger.info("Fetched {} catalog entries in {}.".format(len(catalog), timer.duration_string())) return catalog
def read_catalog_files(glob_pattern): sequence_name_to_type = { "PolynomialSequence": PolynomialSequence, "RecurrentSequence": RecurrentSequence, "TreeCountSequence": TreeCountSequence, "CountDivisorsSequence": CountDivisorsSequence, "EulerPhiSequence": EulerPhiSequence, "PrimeSequence": PrimeSequence, "SumDivisorsSequence": SumDivisorsSequence, "PrimePiSequence": PrimePiSequence, "TwinPrimeSequence": TwinPrimeSequence, "SmallestDivisorSequence": SmallestDivisorSequence } with start_timer() as timer: catalog = {} filenames = sorted(glob.glob(glob_pattern)) for filename in filenames: logger.info( "Fetching catalog data from file '{}' ...".format(filename)) with open(filename) as f: oeis_catalog = json.load(f) for (oeis_id_string, sequence_name, sequence_args) in oeis_catalog: assert oeis_id_string.startswith("A") oeis_id = int(oeis_id_string[1:]) if oeis_id in catalog: logger.error( "A{:06d} in file {!r} previously defined as {!r}, skipping." .format(oeis_id, filename, catalog[oeis_id])) continue sequence_type = sequence_name_to_type[sequence_name] sequence = sequence_type(*sequence_args) catalog[oeis_id] = sequence catalog = OrderedDict(sorted(catalog.items())) logger.info("Fetched {} catalog entries in {}.".format( len(catalog), timer.duration_string())) return catalog
def compress_file(from_filename, to_filename): """Compress a file using the 'xz' compression algorithm.""" PRESET = 9 # level 9 without 'extra' works best on our data. BLOCKSIZE = 1048576 # Process data in blocks of 1 megabyte. with start_timer() as timer: logger.info("Compressing data from '{}' to '{}' ...".format(from_filename, to_filename)) with open(from_filename, "rb") as fi, lzma.open(to_filename, "wb", format = lzma.FORMAT_XZ, check = lzma.CHECK_CRC64, preset = PRESET) as fo: while True: data = fi.read(BLOCKSIZE) if len(data) == 0: break fo.write(data) logger.info("Compressing data took {}.".format(timer.duration_string()))
def database_update_cycle(database_filename): """Perform a single cycle of the database update loop.""" with start_timer() as timer: highest_oeis_id = find_highest_oeis_id() # Check OEIS server for highest entry ID. with close_when_done(sqlite3.connect(database_filename)) as dbconn: ensure_database_schema_created(dbconn) make_database_complete(dbconn, highest_oeis_id) # Make sure we have all entries (full fetch on first run). update_database_entries_randomly(dbconn, highest_oeis_id // 1000) # Refresh 0.1 % of entries randomly. update_database_entries_by_priority(dbconn, highest_oeis_id // 200) # Refresh 0.5 % of entries by priority. update_database_entries_for_nonzero_time_window(dbconn) # Make sure we have t1 != t2 for all entries (full fetch on first run). consolidate_database_monthly(database_filename, remove_stale_files_flag = False) logger.info("Full database update cycle took {}.".format(timer.duration_string()))
def consolidate_database_monthly(database_filename, remove_stale_files_flag): """Make a consolidated version of the database, once per day. The consolidated version will have a standardized filename 'oeis_vYYYYMMDD.sqlite3.xz'. If this filename already exists, we return immediately. If not, we vacuum the database, and compress its file. This process takes ~ 2 hours on a fast desktop PC. When the compressed database is written, we remove all 'stale' consolidated files, i.e., all files that are called 'oeis_vYYYYMMDD.sqlite3.xz' except the one we just wrote. """ now = datetime.datetime.now() if now.day != 1: return xz_filename = now.strftime("oeis_v%Y%m%d.sqlite3.xz") if os.path.exists(xz_filename): return # file already exists. with start_timer() as timer: logger.info("Consolidating database to '{}' ...".format(xz_filename)) # Vacuum the database with close_when_done(sqlite3.connect(database_filename)) as dbconn: vacuum_database(dbconn) # Create the xz file. compress_file(database_filename, xz_filename) # Remove stale files. if remove_stale_files_flag: stale_files = [ filename for filename in glob.glob("oeis_v????????.sqlite3.xz") if filename != xz_filename ] for filename in stale_files: logger.info("Removing stale consolidated file '{}' ...".format( filename)) os.remove(filename) logger.info("Consolidating data took {}.".format( timer.duration_string()))
def process_database_entries(database_filename_in, dirname_out): if not os.path.exists(database_filename_in): logger.critical("Database file '{}' not found! Unable to continue.".format(database_filename_in)) return (root, ext) = os.path.splitext(database_filename_in) if os.path.exists(dirname_out): logger.info("Removing stale directory '{}' ...".format(dirname_out)) shutil.rmtree(dirname_out) # ========== fetch and process database entries, ordered by oeis_id. BATCH_SIZE = 1000 with start_timer() as timer: with close_when_done(sqlite3.connect(database_filename_in)) as dbconn_in, close_when_done(dbconn_in.cursor()) as dbcursor_in: dbcursor_in.execute("SELECT oeis_id, t1, t2, main_content, bfile_content FROM oeis_entries ORDER BY oeis_id;") while True: oeis_entries = dbcursor_in.fetchmany(BATCH_SIZE) if len(oeis_entries) == 0: break logger.log(logging.PROGRESS, "Processing OEIS entries A{:06} to A{:06} ...".format(oeis_entries[0][0], oeis_entries[-1][0])) for (oeis_id, t1, t2, main_content, bfile_content) in oeis_entries: directory = os.path.join(dirname_out, "A{:03d}xxx".format(oeis_id // 1000), "A{:06d}".format(oeis_id)) os.makedirs(directory) with open(os.path.join(directory, "metadata.json"), "w") as f: metadata = [oeis_id, t1, t2] json.dump(metadata, f) with open(os.path.join(directory, "main_content.txt"), "w") as f: f.write(main_content) with open(os.path.join(directory, "bfile_content.txt"), "w") as f: f.write(bfile_content) logger.info("Processed all database entries in {}.".format(timer.duration_string()))
def process_database_entries(database_filename_in): if not os.path.exists(database_filename_in): logger.critical("Database file '{}' not found! Unable to continue.".format(database_filename_in)) return (root, ext) = os.path.splitext(database_filename_in) database_filename_out = root + "_parsed" + ext if os.path.exists(database_filename_out): logger.info("Removing stale file '{}' ...".format(database_filename_out)) os.remove(database_filename_out) # ========== fetch and process database entries, ordered by oeis_id. BATCH_SIZE = 1000 with start_timer() as timer: with close_when_done(sqlite3.connect(database_filename_in)) as dbconn_in, close_when_done(dbconn_in.cursor()) as dbcursor_in: with close_when_done(sqlite3.connect(database_filename_out)) as dbconn_out, close_when_done(dbconn_out.cursor()) as dbcursor_out: create_database_schema(dbconn_out) with concurrent.futures.ProcessPoolExecutor() as pool: dbcursor_in.execute("SELECT oeis_id, main_content, bfile_content FROM oeis_entries ORDER BY oeis_id;") while True: oeis_entries = dbcursor_in.fetchmany(BATCH_SIZE) if len(oeis_entries) == 0: break logger.log(logging.PROGRESS, "Processing OEIS entries A{:06} to A{:06} ...".format(oeis_entries[0][0], oeis_entries[-1][0])) query = "INSERT INTO oeis_entries(oeis_id, identification, value_list, name, comments, detailed_references, links, formulas, examples, maple_programs, mathematica_programs, other_programs, cross_references, keywords, offset_a, offset_b, author, extensions_and_errors) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" dbcursor_out.executemany(query, pool.map(process_oeis_entry, oeis_entries)) dbconn_out.commit() logger.info("Processed all database entries in {}.".format(timer.duration_string()))
def consolidate_database_monthly(database_filename, remove_stale_files_flag): """Make a consolidated version of the database, once per day. The consolidated version will have a standardized filename 'oeis_vYYYYMMDD.sqlite3.xz'. If this filename already exists, we return immediately. If not, we vacuum the database, and compress its file. This process takes ~ 2 hours on a fast desktop PC. When the compressed database is written, we remove all 'stale' consolidated files, i.e., all files that are called 'oeis_vYYYYMMDD.sqlite3.xz' except the one we just wrote. """ now = datetime.datetime.now() if now.day != 1: return xz_filename = now.strftime("oeis_v%Y%m%d.sqlite3.xz") if os.path.exists(xz_filename): return # file already exists. with start_timer() as timer: logger.info("Consolidating database to '{}' ...".format(xz_filename)) # Vacuum the database with close_when_done(sqlite3.connect(database_filename)) as dbconn: vacuum_database(dbconn) # Create the xz file. compress_file(database_filename, xz_filename) # Remove stale files. if remove_stale_files_flag: stale_files = [filename for filename in glob.glob("oeis_v????????.sqlite3.xz") if filename != xz_filename] for filename in stale_files: logger.info("Removing stale consolidated file '{}' ...".format(filename)) os.remove(filename) logger.info("Consolidating data took {}.".format(timer.duration_string()))
def compress_file(from_filename, to_filename): """Compress a file using the 'xz' compression algorithm.""" PRESET = 9 # level 9 without 'extra' works best on our data. BLOCKSIZE = 1048576 # Process data in blocks of 1 megabyte. with start_timer() as timer: logger.info("Compressing data from '{}' to '{}' ...".format( from_filename, to_filename)) with open(from_filename, "rb") as fi, lzma.open(to_filename, "wb", format=lzma.FORMAT_XZ, check=lzma.CHECK_CRC64, preset=PRESET) as fo: while True: data = fi.read(BLOCKSIZE) if len(data) == 0: break fo.write(data) logger.info("Compressing data took {}.".format( timer.duration_string()))
def solve_linear_recurrences(database_filename_in, terms, exclude_entries = None): if not os.path.exists(database_filename_in): logger.critical("Database file '{}' not found! Unable to continue.".format(database_filename_in)) return if exclude_entries is None: exclude_entries = frozenset() # ========== fetch and process database entries, ordered by oeis_id. BATCH_SIZE = 1000 with start_timer() as timer: with close_when_done(sqlite3.connect(database_filename_in)) as dbconn_in, close_when_done(dbconn_in.cursor()) as dbcursor_in: with concurrent.futures.ProcessPoolExecutor() as pool: dbcursor_in.execute("SELECT oeis_id, main_content, bfile_content FROM oeis_entries ORDER BY oeis_id;") while True: oeis_entries = dbcursor_in.fetchmany(BATCH_SIZE) if len(oeis_entries) == 0: break logger.log(logging.PROGRESS, "Processing OEIS entries A{:06} to A{:06} ...".format(oeis_entries[0][0], oeis_entries[-1][0])) work = [(oeis_id, main_content, bfile_content, terms) for (oeis_id, main_content, bfile_content) in oeis_entries if "A{:06d}".format(oeis_id) not in exclude_entries] for (oeis_entry, solution) in pool.map(process_oeis_entry, work): if solution is not None: yield (str(oeis_entry), solution) logger.info("Processed all database entries in {}.".format(timer.duration_string()))
def test_started_timer(self): self.assertEqual(False, timer.start_timer())
def test_decrease_time_thats_over(self): timer.is_running = False timer.start_timer(0) result = timer.decrease_time() self.assertEqual(False, result)
def test_start_started_timer(self): result = timer.start_timer(100) self.assertEqual(False, result)
def setUp(self): timer.start_timer(100)
target_mode = ['DQN', 'DDQN', 'A3C', 'MCT'] cur_target = 'MCT' max_t = env.spec.max_episode_steps #max_t = env.time_limit agent = DQNAgent(obs_dim,act_dim,memory_mode='PER',target_mode=cur_target, policy_mode=cur_policy, restore=False, net_dir='argmax/iter_56000.ckpt') # memory_mode='PER',target_mode='DDQN' avg_return_list = deque(maxlen=100) avg_loss_list = deque(maxlen=100) avg_success_list = deque(maxlen=100) result_saver = [] timer.start_timer() print(cur_mode) print(cur_policy) print(cur_target) async_num=64 if cur_target=='MCT' or cur_target=='A3C': async_grad = [np.zeros(w.shape) for w in agent.weights] obs_set = [] for n in range(async_num): obs_set.append(env.reset()) rwd_set=[0]*async_num
def fetch_entries_into_database(dbconn, entries): """Fetch a set of entries from the remote OEIS database and store the results in the database. The 'entries' parameter contains a number of OEIS IDs. This function can handle a large number of entries, up to the entire size of the OEIS database. Entries are processed in randomized batches. We use a pool of worker threads to perform the actual fetches. This enhances fetch performance (in terms of fetche-per-second) dramatically. Typical fetch performance is about 20 fetches per second. The responses of each batch of entries are processed by the 'process_responses' function defined above. """ FETCH_BATCH_SIZE = 500 # 100 -- 1000 are reasonable NUM_WORKERS = 20 # 10 -- 20 are reasonable SLEEP_AFTER_BATCH = 2.0 # [seconds] entries = set(entries) # make a copy, and ensure it is a set. with start_timer( len(entries)) as timer, concurrent.futures.ThreadPoolExecutor( NUM_WORKERS) as executor: while len(entries) > 0: batch_size = min(FETCH_BATCH_SIZE, len(entries)) batch = random.sample(entries, batch_size) logger.info( "Fetching data using {} {} for {} out of {} entries ...". format(NUM_WORKERS, "worker" if NUM_WORKERS == 1 else "workers", batch_size, len(entries))) with start_timer() as batch_timer: # Execute fetches in parallel. responses = list( executor.map(safe_fetch_remote_oeis_entry, batch)) logger.info( "{} fetches took {} seconds ({:.3f} fetches/second).". format(batch_size, batch_timer.duration_string(), batch_size / batch_timer.duration())) # Process the responses by updating the database. processed_entries = process_responses(dbconn, responses) entries -= processed_entries # Calculate and show estimated-time-to-completion. logger.info("Estimated time to completion: {}.".format( timer.etc_string(work_remaining=len(entries)))) # Sleep if we need to fetch more if len(entries) > 0: logger.info("Sleeping for {:.1f} seconds ...".format( SLEEP_AFTER_BATCH)) time.sleep(SLEEP_AFTER_BATCH) logger.info("Fetched {} entries in {}.".format( timer.total_work, timer.duration_string()))
def set_time(self, new_time): if not self.round_ended(self): self.timer_start_value = new_time self.start_time = timer.start_timer() pass