def main(): parser = argparse.ArgumentParser(description="Retrieve freebox statistics") parser.add_argument( "-d", "--database", help="Sqlite3 database where to store the statistics (default: %s)" % config.database) parser.add_argument( "-u", "--url", help="URL where the statistics are retrieved (default: %s)" % config.statsurl) args = parser.parse_args() if args.database is not None: config.database = args.database if args.url is not None: config.statsurl = args.url cnx = db.new_connection() cur = cnx.cursor() db.create_tables(cur) p = StatsPage() store_stats(p, cur) store_logs(p, cur) store_netlinks(p, cur) cur.execute("PRAGMA optimize") cnx.commit()
def main(): if len(sys.argv) != 2: print("usage: %s datadir" % sys.argv[0], file=sys.stderr) return datadir = sys.argv[1] cnx = db.new_connection() cur = cnx.cursor() db.create_tables(cur) for f in sorted(glob.glob(datadir + "/*.csv")): import_data(cur, f) cnx.commit() cur.execute("ANALYZE") cnx.commit()
def main(): parser = argparse.ArgumentParser(description="Plot the statistics") parser.add_argument("-f", "--figdir", help="Directory where to store the output figures (default: %s)" % config.figdir) parser.add_argument("-g", "--gnuplotdir", help="Directory where the gnuplot scripts are located (default: %s)" % config.gnuplotdir) parser.add_argument("-t", "--tmpdir", help="Directory where to store the temporary data files (default to system temporary directory)") parser.add_argument("-l", "--list", action='store_true', help="List available countries and exit") parser.add_argument("-c", "--country", action='append', help="Countries to plot") parser.add_argument("-N", "--ndaysavg", default=5, type=int, help="Average the number of confirmed case over the N previous days") args = parser.parse_args() if args.figdir is not None: config.figdir = args.figdir if args.gnuplotdir is not None: congnuplot.gnuplotdir = args.gnuplotdir if args.tmpdir is not None: config.tmpdir = args.tmpdir if args.country is None: countries = ["France", "Chine"] else: countries = args.country ndaysavg = args.ndaysavg cnx = db.new_connection() cur = cnx.cursor() if args.list: list_countries(cur) else: if check_countries(cur, countries): plot_raw_data(cur, countries, ndaysavg) plot_regression(cnx, countries, ndaysavg) plot_metrics_evolution(cnx, countries, ndaysavg) cur.execute("PRAGMA optimize")
from db import new_connection from mysql.connector import ProgrammingError table_contacts = """ CREATE TABLE IF NOT EXISTS contacts( name VARCHAR(50), tel VARCHAR(40) ) """ table_emails = """ CREATE TABLE IF NOT EXISTS emails( id INT AUTO_INCREMENT PRIMARY KEY, dono VARCHAR(50) ) """ with new_connection() as connection: try: cursor = connection.cursor() cursor.execute(table_contacts) cursor.execute(table_emails) except ProgrammingError as e: print(f'Error: {e.msg}')
col="#606000") lines(x=d$time, y=d$ip24, type="h", col="#006000") legend("topright", legend=c('in transit (<24h)', 'in queue (<24h)', 'in transit (24-48h)', 'in queue (24-48h)', 'in transit (>48h)', 'in queue (>48h)'), ncol=3, col=c("#006000", "#00E000", "#606000", "#D0D000", "#800000", "#FF0000"), lty=1) '''.format(**locals()) if __name__ == '__main__': import queue, db with closing(db.new_connection(initdb=False)) as conn: last_update, queue_contracts, accepted_contracts = queue.load(conn) done_1day = db.Contract.load_completed_after(conn = conn, cutoff = last_update - datetime.timedelta(days=1)) done_7day = db.Contract.load_completed_after(conn = conn, cutoff = last_update - datetime.timedelta(days=7)) history = db.load_queue_history(conn = conn, first_update = last_update - datetime.timedelta(days=7), last_update = last_update) make_queue_graph(last_update, queue_contracts, accepted_contracts, scale=3600, filename="queue_3600.png") make_queue_graph(last_update, queue_contracts, accepted_contracts, scale=900, filename="queue_900.png") make_delivery_graph(last_update, done_1day, scale=3600, filename="delivery_1day.png", title="Red Frog delivery times - last day") make_delivery_graph(last_update, done_7day, scale=3600, filename="delivery_7day.png", title="Red Frog delivery times - last week") make_history_graph(history, filename="queue_history_7day.png", title="Red Frog queue size - last week")
def run_one_cycle(): log("Starting an update cycle") conn = db.new_connection() try: previous_update = db.get_last_update(conn) try: log("Scraping NCF page") ncf_results = scrape_ncf.scrape_ncf(write_copy = 'log/ncf-latest.html', write_error = datetime.datetime.now().strftime('log/ncf-missing-%Y%m%d%H%M%S.html')) log("NCF: Server time: {t}", t = ncf_results['server_time']) log("NCF: Last update: {t}", t = ncf_results['last_update']) log("NCF: Next update: {t}", t = ncf_results['next_update']) log("NCF: Outstanding: {n}", n = ncf_results['outstanding']) log("NCF: In progress: {n}", n = ncf_results['inprogress']) log("NCF: Refresh: {r}", r = ncf_results['refresh']) if ncf_results['missing'] > 0: log("NCF: *MISSING*: {n}", n = ncf_results['missing']) except: log("Caught an exception processing the NCF page") traceback.print_exc() return datetime.timedelta(minutes=5) last_update = ncf_results['last_update'] if previous_update is not None and last_update <= previous_update: log("NCF update ({last_update}) is not newer than database's last update ({previous_update}); skipping this update", **locals()) return ncf_results['refresh'] try: log("Scraping tracker page") tracker_results = scrape_tracker.scrape_tracker(write_copy = 'log/tracker-latest.html', write_error = datetime.datetime.now().strftime('log/tracker-missing-%Y%m%d%H%M%S.html')) log("Tracker: Server time: {t}", t = tracker_results['server_time']) log("Tracker: Last update: {t}", t = tracker_results['last_update']) log("Tracker: Next update: {t}", t = tracker_results['next_update']) log("Tracker: Outstanding: {n}", n = tracker_results['outstanding']) log("Tracker: In progress: {n}", n = tracker_results['inprogress']) log("Tracker: Refresh: {r}", r = tracker_results['refresh']) if tracker_results['missing'] > 0: log("Tracker: *MISSING*: {n}", n = tracker_results['missing']) except: log("Caught an exception processing the tracker page") traceback.print_exc() return datetime.timedelta(minutes=5) if last_update != tracker_results['last_update']: log("NCF and Tracker pages had different last update times. Retrying.") return datetime.timedelta(minutes=1) seen_contracts = set() for contract in ncf_results['contracts'] + tracker_results['contracts']: try: cid = contract.contract_id seen_contracts.add(cid) db_contract = db.Contract.load(conn = conn, contract_id = cid) if not db_contract: if contract.state == db.Contract.IN_PROGRESS and contract.created_min is None: # Missed it in queue. It must have been created between the last update we processed and the time it was accepted contract.created_min = previous_update contract.created_max = contract.accepted log("New: {contract}", **locals()) contract.insert_row(conn = conn) else: merged_contract = db.Contract.merge(db_contract, contract) if db_contract.state != merged_contract.state: log("Upd: {merged_contract}", **locals()) merged_contract.update_row(conn = conn) except: log("Exception processing contract: {contract}", **locals()) raise log("Looking for contracts that disappeared from the queue") for contract in db.Contract.load_by_state(conn = conn, state = db.Contract.IN_QUEUE): if contract.contract_id not in seen_contracts: contract.state = db.Contract.GONE log("Gone: {contract}", **locals()) contract.update_row(conn = conn) log("Looking for contracts that disappeared from the tracker") for contract in db.Contract.load_by_state(conn = conn, state = db.Contract.IN_PROGRESS): if contract.contract_id not in seen_contracts: contract.state = db.Contract.DONE log("Done: {contract}", **locals()) contract.update_row(conn = conn) log("Updating queue history") active_contracts = db.Contract.load_where(conn = conn, clause = "state IN (:inqueue,:inprogress)", inqueue = db.Contract.IN_QUEUE, inprogress = db.Contract.IN_PROGRESS) history.update(conn = conn, active_contracts = active_contracts, update_time = last_update) db.add_update_info(conn, update_time = last_update, queue_count = len(ncf_results['contracts']), in_progress_count = len(tracker_results['contracts'])) db.set_last_update(conn, ncf_results['last_update']) conn.commit() try: regenerate_graphs(conn) except: log("Exception generating graphs (ignored)") traceback.print_exc() except: log("Exception in update cycle") traceback.print_exc() return datetime.timedelta(minutes=5) finally: conn.close() refresh = max(ncf_results['refresh'], tracker_results['refresh']) log("Update cycle is done, next refresh in {refresh}", **locals()) return refresh