def main():
args = parse_args(sys.argv[1:])
config = load_config(args.config_file_path)
client = config.client.create_client()
with psycopg2.connect(config.database.connection_string) as conn_activity, \
psycopg2.connect(config.database.connection_string) as conn_db:
activity = Activity(conn=conn_activity,
activity_type='check_completed',
run_at=datetime.now(tz=local_tz))
db = DB(conn=conn_db,
completion_registry_thread_id=config.completion_registry_thread_id)
stats = Stats()
message = None
is_successful = True
try:
scan_finished_threads(db, client, stats)
except:
exc_text = traceback.format_exc()
logging.critical(exc_text)
message = exc_text
is_successful = False
finally:
activity.report_end(is_successful, message, stats)
if is_successful:
logging.info("成功结束")
else:
exit(1)
def main():
args = parse_args(sys.argv[1:])
config = load_config(args.config_file_path)
with psycopg2.connect(config.database.connection_string) as conn_activity, \
psycopg2.connect(config.database.connection_string) as conn_db:
activity = Activity(conn=conn_activity,
activity_type='collect',
run_at=datetime.now(tz=local_tz))
db = DB(conn=conn_db,
completion_registry_thread_id=config.completion_registry_thread_id)
stats = Stats()
fetching_since = activity.should_collect_since
is_successful = False
message = None
try:
fetch_board(db=db, activity=activity, client=config.client.create_client(),
board_id=config.board_id, fetching_since=fetching_since, stats=stats)
is_successful = True
except:
exc_text = traceback.format_exc()
logging.critical(exc_text)
message = exc_text
finally:
activity.report_end(is_successful, message, stats)
if is_successful:
logging.info("成功结束")
else:
exit(1)
def migrate_activity_table(conn_s3: sqlite3.Connection,
conn_pg: psycopg2._psycopg.connection):
n = count_rows(conn_s3, 'activity')
last_fetched_until: datetime = None
for i, [
_, # id,
run_at,
fetched_since,
ensured_fetched_until,
is_successful,
message,
uploaded_bytes,
downloaded_bytes,
newly_recorded_thread_count,
affected_thread_count,
newly_recorded_post_count,
requested_board_page_count,
requested_thread_page_count,
logged_in_thread_request_count,
] in enumerate(conn_s3.execute(r'SELECT * FROM activity')):
activity = Activity(conn=conn_pg,
activity_type='legacy',
run_at=ts2dt(run_at),
logger=None)
if last_fetched_until is None:
assert (Activity.never_collected(conn=conn_pg))
assert (activity.should_collect_since)
else:
assert (not Activity.never_collected(conn=conn_pg))
assert (activity.should_collect_since == last_fetched_until)
activity.report_collecting_range(since=ts2dt(fetched_since),
until=ts2dt(ensured_fetched_until))
if ensured_fetched_until and is_successful:
last_fetched_until = ts2dt(ensured_fetched_until)
stats = Stats()
stats.new_thread_count = newly_recorded_thread_count
stats.affected_thread_count = affected_thread_count
stats.new_post_count = newly_recorded_post_count
stats.board_request_count = requested_board_page_count
stats.thread_request_count = requested_thread_page_count
stats.logged_in_thread_request_count = logged_in_thread_request_count
stats.total_bandwidth_usage.add([uploaded_bytes, downloaded_bytes])
activity.report_end(is_successful=bool(is_successful),
message=message,
stats=stats)
if i % 100 == 0:
print(f"activity: {i+1}/{n} {ts2dt(run_at)}")
