def fetch_entries_into_database(dbconn, entries):
"""Fetch a set of entries from the remote OEIS database and store the results in the database.
The 'entries' parameter contains a number of OEIS IDs.
This function can handle a large number of entries, up to the entire size of the OEIS database.
Entries are processed in randomized batches.
We use a pool of worker threads to perform the actual fetches.
This enhances fetch performance (in terms of fetche-per-second) dramatically.
Typical fetch performance is about 20 fetches per second.
The responses of each batch of entries are processed by the 'process_responses' function defined above.
"""
FETCH_BATCH_SIZE = 500 # 100 -- 1000 are reasonable
NUM_WORKERS = 20 # 10 -- 20 are reasonable
SLEEP_AFTER_BATCH = 2.0 # [seconds]
entries = set(entries) # make a copy, and ensure it is a set.
with start_timer(len(entries)) as timer, concurrent.futures.ThreadPoolExecutor(NUM_WORKERS) as executor:
while len(entries) > 0:
batch_size = min(FETCH_BATCH_SIZE, len(entries))
batch = random.sample(entries, batch_size)
logger.info("Fetching data using {} {} for {} out of {} entries ...".format(NUM_WORKERS, "worker" if NUM_WORKERS == 1 else "workers", batch_size, len(entries)))
with start_timer() as batch_timer:
# Execute fetches in parallel.
responses = list(executor.map(safe_fetch_remote_oeis_entry, batch))
logger.info("{} fetches took {} seconds ({:.3f} fetches/second).".format(batch_size, batch_timer.duration_string(), batch_size / batch_timer.duration()))
# Process the responses by updating the database.
processed_entries = process_responses(dbconn, responses)
entries -= processed_entries
# Calculate and show estimated-time-to-completion.
logger.info("Estimated time to completion: {}.".format(timer.etc_string(work_remaining = len(entries))))
# Sleep if we need to fetch more
if len(entries) > 0:
logger.info("Sleeping for {:.1f} seconds ...".format(SLEEP_AFTER_BATCH))
time.sleep(SLEEP_AFTER_BATCH)
logger.info("Fetched {} entries in {}.".format(timer.total_work, timer.duration_string()))
def vacuum_database(dbconn):
"""Perform a VACUUM command on the database."""
with start_timer() as timer:
logger.info("Initiating VACUUM on database ...")
dbconn.execute("VACUUM;")
logger.info("VACUUM done in {}.".format(timer.duration_string()))
def vacuum_database(dbconn):
"""Perform a VACUUM command on the database."""
with start_timer() as timer:
logger.info("Initiating VACUUM on database ...")
dbconn.execute("VACUUM;")
logger.info("VACUUM done in {}.".format(timer.duration_string()))
def database_update_cycle(database_filename):
"""Perform a single cycle of the database update loop."""
with start_timer() as timer:
highest_oeis_id = find_highest_oeis_id(
) # Check OEIS server for highest entry ID.
with close_when_done(sqlite3.connect(database_filename)) as dbconn:
ensure_database_schema_created(dbconn)
make_database_complete(
dbconn, highest_oeis_id
) # Make sure we have all entries (full fetch on first run).
update_database_entries_randomly(
dbconn,
highest_oeis_id // 1000) # Refresh 0.1 % of entries randomly.
update_database_entries_by_priority(
dbconn, highest_oeis_id //
200) # Refresh 0.5 % of entries by priority.
update_database_entries_for_nonzero_time_window(
dbconn
) # Make sure we have t1 != t2 for all entries (full fetch on first run).
consolidate_database_monthly(database_filename,
remove_stale_files_flag=False)
logger.info("Full database update cycle took {}.".format(
timer.duration_string()))
def open(self):
# print("websocket opened")
self.connection = False
#启动websocket校验计时器，一定时间不发送token验证，自动断开连接
self.start_timer = start_timer()
self.ws_loop = asyncio.get_event_loop()
self.start_timer.start(self.close, self.ws_loop)
def play_round(self, start_score, debug=False):
#mocked to show some difference to ensure things are working
self.current_score_value = start_score
print(self)
self.start_time = timer.start_timer()
while not self.round_ended(self):
self.current_time_value = timer.increment_timer(self)
self.check_for_score(debug)
print(self)
print(self)
return self.current_status()
def read_catalog_files(glob_pattern):
sequence_name_to_type = {
"PolynomialSequence": PolynomialSequence,
"RecurrentSequence": RecurrentSequence,
"TreeCountSequence": TreeCountSequence,
"CountDivisorsSequence": CountDivisorsSequence,
"EulerPhiSequence": EulerPhiSequence,
"PrimeSequence": PrimeSequence,
"SumDivisorsSequence": SumDivisorsSequence,
"PrimePiSequence": PrimePiSequence,
"TwinPrimeSequence": TwinPrimeSequence,
"SmallestDivisorSequence": SmallestDivisorSequence,
}
with start_timer() as timer:
catalog = {}
filenames = sorted(glob.glob(glob_pattern))
for filename in filenames:
logger.info("Fetching catalog data from file '{}' ...".format(filename))
with open(filename) as f:
oeis_catalog = json.load(f)
for (oeis_id_string, sequence_name, sequence_args) in oeis_catalog:
assert oeis_id_string.startswith("A")
oeis_id = int(oeis_id_string[1:])
if oeis_id in catalog:
logger.error(
"A{:06d} in file {!r} previously defined as {!r}, skipping.".format(
oeis_id, filename, catalog[oeis_id]
)
)
continue
sequence_type = sequence_name_to_type[sequence_name]
sequence = sequence_type(*sequence_args)
catalog[oeis_id] = sequence
catalog = OrderedDict(sorted(catalog.items()))
logger.info("Fetched {} catalog entries in {}.".format(len(catalog), timer.duration_string()))
return catalog
def read_catalog_files(glob_pattern):
sequence_name_to_type = {
"PolynomialSequence": PolynomialSequence,
"RecurrentSequence": RecurrentSequence,
"TreeCountSequence": TreeCountSequence,
"CountDivisorsSequence": CountDivisorsSequence,
"EulerPhiSequence": EulerPhiSequence,
"PrimeSequence": PrimeSequence,
"SumDivisorsSequence": SumDivisorsSequence,
"PrimePiSequence": PrimePiSequence,
"TwinPrimeSequence": TwinPrimeSequence,
"SmallestDivisorSequence": SmallestDivisorSequence
}
with start_timer() as timer:
catalog = {}
filenames = sorted(glob.glob(glob_pattern))
for filename in filenames:
logger.info(
"Fetching catalog data from file '{}' ...".format(filename))
with open(filename) as f:
oeis_catalog = json.load(f)
for (oeis_id_string, sequence_name, sequence_args) in oeis_catalog:
assert oeis_id_string.startswith("A")
oeis_id = int(oeis_id_string[1:])
if oeis_id in catalog:
logger.error(
"A{:06d} in file {!r} previously defined as {!r}, skipping."
.format(oeis_id, filename, catalog[oeis_id]))
continue
sequence_type = sequence_name_to_type[sequence_name]
sequence = sequence_type(*sequence_args)
catalog[oeis_id] = sequence
catalog = OrderedDict(sorted(catalog.items()))
logger.info("Fetched {} catalog entries in {}.".format(
len(catalog), timer.duration_string()))
return catalog
def compress_file(from_filename, to_filename):
"""Compress a file using the 'xz' compression algorithm."""
PRESET = 9 # level 9 without 'extra' works best on our data.
BLOCKSIZE = 1048576 # Process data in blocks of 1 megabyte.
with start_timer() as timer:
logger.info("Compressing data from '{}' to '{}' ...".format(from_filename, to_filename))
with open(from_filename, "rb") as fi, lzma.open(to_filename, "wb", format = lzma.FORMAT_XZ, check = lzma.CHECK_CRC64, preset = PRESET) as fo:
while True:
data = fi.read(BLOCKSIZE)
if len(data) == 0:
break
fo.write(data)
logger.info("Compressing data took {}.".format(timer.duration_string()))
def database_update_cycle(database_filename):
"""Perform a single cycle of the database update loop."""
with start_timer() as timer:
highest_oeis_id = find_highest_oeis_id() # Check OEIS server for highest entry ID.
with close_when_done(sqlite3.connect(database_filename)) as dbconn:
ensure_database_schema_created(dbconn)
make_database_complete(dbconn, highest_oeis_id) # Make sure we have all entries (full fetch on first run).
update_database_entries_randomly(dbconn, highest_oeis_id // 1000) # Refresh 0.1 % of entries randomly.
update_database_entries_by_priority(dbconn, highest_oeis_id // 200) # Refresh 0.5 % of entries by priority.
update_database_entries_for_nonzero_time_window(dbconn) # Make sure we have t1 != t2 for all entries (full fetch on first run).
consolidate_database_monthly(database_filename, remove_stale_files_flag = False)
logger.info("Full database update cycle took {}.".format(timer.duration_string()))
def consolidate_database_monthly(database_filename, remove_stale_files_flag):
"""Make a consolidated version of the database, once per day.
The consolidated version will have a standardized filename 'oeis_vYYYYMMDD.sqlite3.xz'.
If this filename already exists, we return immediately.
If not, we vacuum the database, and compress its file. This process takes ~ 2 hours on a fast desktop PC.
When the compressed database is written, we remove all 'stale' consolidated files,
i.e., all files that are called 'oeis_vYYYYMMDD.sqlite3.xz' except the one we just wrote.
"""
now = datetime.datetime.now()
if now.day != 1:
return
xz_filename = now.strftime("oeis_v%Y%m%d.sqlite3.xz")
if os.path.exists(xz_filename):
return # file already exists.
with start_timer() as timer:
logger.info("Consolidating database to '{}' ...".format(xz_filename))
# Vacuum the database
with close_when_done(sqlite3.connect(database_filename)) as dbconn:
vacuum_database(dbconn)
# Create the xz file.
compress_file(database_filename, xz_filename)
# Remove stale files.
if remove_stale_files_flag:
stale_files = [
filename for filename in glob.glob("oeis_v????????.sqlite3.xz")
if filename != xz_filename
]
for filename in stale_files:
logger.info("Removing stale consolidated file '{}' ...".format(
filename))
os.remove(filename)
logger.info("Consolidating data took {}.".format(
timer.duration_string()))
def process_database_entries(database_filename_in, dirname_out):
if not os.path.exists(database_filename_in):
logger.critical("Database file '{}' not found! Unable to continue.".format(database_filename_in))
return
(root, ext) = os.path.splitext(database_filename_in)
if os.path.exists(dirname_out):
logger.info("Removing stale directory '{}' ...".format(dirname_out))
shutil.rmtree(dirname_out)
# ========== fetch and process database entries, ordered by oeis_id.
BATCH_SIZE = 1000
with start_timer() as timer:
with close_when_done(sqlite3.connect(database_filename_in)) as dbconn_in, close_when_done(dbconn_in.cursor()) as dbcursor_in:
dbcursor_in.execute("SELECT oeis_id, t1, t2, main_content, bfile_content FROM oeis_entries ORDER BY oeis_id;")
while True:
oeis_entries = dbcursor_in.fetchmany(BATCH_SIZE)
if len(oeis_entries) == 0:
break
logger.log(logging.PROGRESS, "Processing OEIS entries A{:06} to A{:06} ...".format(oeis_entries[0][0], oeis_entries[-1][0]))
for (oeis_id, t1, t2, main_content, bfile_content) in oeis_entries:
directory = os.path.join(dirname_out, "A{:03d}xxx".format(oeis_id // 1000), "A{:06d}".format(oeis_id))
os.makedirs(directory)
with open(os.path.join(directory, "metadata.json"), "w") as f:
metadata = [oeis_id, t1, t2]
json.dump(metadata, f)
with open(os.path.join(directory, "main_content.txt"), "w") as f:
f.write(main_content)
with open(os.path.join(directory, "bfile_content.txt"), "w") as f:
f.write(bfile_content)
logger.info("Processed all database entries in {}.".format(timer.duration_string()))
def process_database_entries(database_filename_in):
if not os.path.exists(database_filename_in):
logger.critical("Database file '{}' not found! Unable to continue.".format(database_filename_in))
return
(root, ext) = os.path.splitext(database_filename_in)
database_filename_out = root + "_parsed" + ext
if os.path.exists(database_filename_out):
logger.info("Removing stale file '{}' ...".format(database_filename_out))
os.remove(database_filename_out)
# ========== fetch and process database entries, ordered by oeis_id.
BATCH_SIZE = 1000
with start_timer() as timer:
with close_when_done(sqlite3.connect(database_filename_in)) as dbconn_in, close_when_done(dbconn_in.cursor()) as dbcursor_in:
with close_when_done(sqlite3.connect(database_filename_out)) as dbconn_out, close_when_done(dbconn_out.cursor()) as dbcursor_out:
create_database_schema(dbconn_out)
with concurrent.futures.ProcessPoolExecutor() as pool:
dbcursor_in.execute("SELECT oeis_id, main_content, bfile_content FROM oeis_entries ORDER BY oeis_id;")
while True:
oeis_entries = dbcursor_in.fetchmany(BATCH_SIZE)
if len(oeis_entries) == 0:
break
logger.log(logging.PROGRESS, "Processing OEIS entries A{:06} to A{:06} ...".format(oeis_entries[0][0], oeis_entries[-1][0]))
query = "INSERT INTO oeis_entries(oeis_id, identification, value_list, name, comments, detailed_references, links, formulas, examples, maple_programs, mathematica_programs, other_programs, cross_references, keywords, offset_a, offset_b, author, extensions_and_errors) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
dbcursor_out.executemany(query, pool.map(process_oeis_entry, oeis_entries))
dbconn_out.commit()
logger.info("Processed all database entries in {}.".format(timer.duration_string()))
def consolidate_database_monthly(database_filename, remove_stale_files_flag):
"""Make a consolidated version of the database, once per day.
The consolidated version will have a standardized filename 'oeis_vYYYYMMDD.sqlite3.xz'.
If this filename already exists, we return immediately.
If not, we vacuum the database, and compress its file. This process takes ~ 2 hours on a fast desktop PC.
When the compressed database is written, we remove all 'stale' consolidated files,
i.e., all files that are called 'oeis_vYYYYMMDD.sqlite3.xz' except the one we just wrote.
"""
now = datetime.datetime.now()
if now.day != 1:
return
xz_filename = now.strftime("oeis_v%Y%m%d.sqlite3.xz")
if os.path.exists(xz_filename):
return # file already exists.
with start_timer() as timer:
logger.info("Consolidating database to '{}' ...".format(xz_filename))
# Vacuum the database
with close_when_done(sqlite3.connect(database_filename)) as dbconn:
vacuum_database(dbconn)
# Create the xz file.
compress_file(database_filename, xz_filename)
# Remove stale files.
if remove_stale_files_flag:
stale_files = [filename for filename in glob.glob("oeis_v????????.sqlite3.xz") if filename != xz_filename]
for filename in stale_files:
logger.info("Removing stale consolidated file '{}' ...".format(filename))
os.remove(filename)
logger.info("Consolidating data took {}.".format(timer.duration_string()))
def compress_file(from_filename, to_filename):
"""Compress a file using the 'xz' compression algorithm."""
PRESET = 9 # level 9 without 'extra' works best on our data.
BLOCKSIZE = 1048576 # Process data in blocks of 1 megabyte.
with start_timer() as timer:
logger.info("Compressing data from '{}' to '{}' ...".format(
from_filename, to_filename))
with open(from_filename, "rb") as fi, lzma.open(to_filename,
"wb",
format=lzma.FORMAT_XZ,
check=lzma.CHECK_CRC64,
preset=PRESET) as fo:
while True:
data = fi.read(BLOCKSIZE)
if len(data) == 0:
break
fo.write(data)
logger.info("Compressing data took {}.".format(
timer.duration_string()))
def solve_linear_recurrences(database_filename_in, terms, exclude_entries = None):
if not os.path.exists(database_filename_in):
logger.critical("Database file '{}' not found! Unable to continue.".format(database_filename_in))
return
if exclude_entries is None:
exclude_entries = frozenset()
# ========== fetch and process database entries, ordered by oeis_id.
BATCH_SIZE = 1000
with start_timer() as timer:
with close_when_done(sqlite3.connect(database_filename_in)) as dbconn_in, close_when_done(dbconn_in.cursor()) as dbcursor_in:
with concurrent.futures.ProcessPoolExecutor() as pool:
dbcursor_in.execute("SELECT oeis_id, main_content, bfile_content FROM oeis_entries ORDER BY oeis_id;")
while True:
oeis_entries = dbcursor_in.fetchmany(BATCH_SIZE)
if len(oeis_entries) == 0:
break
logger.log(logging.PROGRESS, "Processing OEIS entries A{:06} to A{:06} ...".format(oeis_entries[0][0], oeis_entries[-1][0]))
work = [(oeis_id, main_content, bfile_content, terms) for (oeis_id, main_content, bfile_content) in oeis_entries if "A{:06d}".format(oeis_id) not in exclude_entries]
for (oeis_entry, solution) in pool.map(process_oeis_entry, work):
if solution is not None:
yield (str(oeis_entry), solution)
logger.info("Processed all database entries in {}.".format(timer.duration_string()))
def test_started_timer(self):
self.assertEqual(False, timer.start_timer())
def test_started_timer(self):
self.assertEqual(False, timer.start_timer())
def test_decrease_time_thats_over(self):
timer.is_running = False
timer.start_timer(0)
result = timer.decrease_time()
self.assertEqual(False, result)
def test_start_started_timer(self):
result = timer.start_timer(100)
self.assertEqual(False, result)
def setUp(self):
timer.start_timer(100)
target_mode = ['DQN', 'DDQN', 'A3C', 'MCT']
cur_target = 'MCT'
max_t = env.spec.max_episode_steps
#max_t = env.time_limit
agent = DQNAgent(obs_dim,act_dim,memory_mode='PER',target_mode=cur_target, policy_mode=cur_policy,
restore=False, net_dir='argmax/iter_56000.ckpt') # memory_mode='PER',target_mode='DDQN'
avg_return_list = deque(maxlen=100)
avg_loss_list = deque(maxlen=100)
avg_success_list = deque(maxlen=100)
result_saver = []
timer.start_timer()
print(cur_mode)
print(cur_policy)
print(cur_target)
async_num=64
if cur_target=='MCT' or cur_target=='A3C':
async_grad = [np.zeros(w.shape) for w in agent.weights]
obs_set = []
for n in range(async_num):
obs_set.append(env.reset())
rwd_set=[0]*async_num
def test_start_started_timer(self):
result = timer.start_timer(100)
self.assertEqual(False, result)
def test_decrease_time_thats_over(self):
timer.is_running = False
timer.start_timer(0)
result = timer.decrease_time()
self.assertEqual(False, result)
def setUp(self):
timer.start_timer(100)
def fetch_entries_into_database(dbconn, entries):
"""Fetch a set of entries from the remote OEIS database and store the results in the database.
The 'entries' parameter contains a number of OEIS IDs.
This function can handle a large number of entries, up to the entire size of the OEIS database.
Entries are processed in randomized batches.
We use a pool of worker threads to perform the actual fetches.
This enhances fetch performance (in terms of fetche-per-second) dramatically.
Typical fetch performance is about 20 fetches per second.
The responses of each batch of entries are processed by the 'process_responses' function defined above.
"""
FETCH_BATCH_SIZE = 500 # 100 -- 1000 are reasonable
NUM_WORKERS = 20 # 10 -- 20 are reasonable
SLEEP_AFTER_BATCH = 2.0 # [seconds]
entries = set(entries) # make a copy, and ensure it is a set.
with start_timer(
len(entries)) as timer, concurrent.futures.ThreadPoolExecutor(
NUM_WORKERS) as executor:
while len(entries) > 0:
batch_size = min(FETCH_BATCH_SIZE, len(entries))
batch = random.sample(entries, batch_size)
logger.info(
"Fetching data using {} {} for {} out of {} entries ...".
format(NUM_WORKERS,
"worker" if NUM_WORKERS == 1 else "workers", batch_size,
len(entries)))
with start_timer() as batch_timer:
# Execute fetches in parallel.
responses = list(
executor.map(safe_fetch_remote_oeis_entry, batch))
logger.info(
"{} fetches took {} seconds ({:.3f} fetches/second).".
format(batch_size, batch_timer.duration_string(),
batch_size / batch_timer.duration()))
# Process the responses by updating the database.
processed_entries = process_responses(dbconn, responses)
entries -= processed_entries
# Calculate and show estimated-time-to-completion.
logger.info("Estimated time to completion: {}.".format(
timer.etc_string(work_remaining=len(entries))))
# Sleep if we need to fetch more
if len(entries) > 0:
logger.info("Sleeping for {:.1f} seconds ...".format(
SLEEP_AFTER_BATCH))
time.sleep(SLEEP_AFTER_BATCH)
logger.info("Fetched {} entries in {}.".format(
timer.total_work, timer.duration_string()))
def set_time(self, new_time):
if not self.round_ended(self):
self.timer_start_value = new_time
self.start_time = timer.start_timer()
pass
