def main(config, interval=10, restart_in_progress=False, restart_errors=False, restart_all=False, no_update_state=False, log_level='DEBUG', one_pass=False, ignore_not_ready=False): with open(config) as f: config = json.load(f) backdoor(6666) class Stop(BaseException): pass logging.getLogger().setLevel(log_level) jobs = gevent.pool.Pool(MAX_JOBS) backoff = Backoff(1, 60) try: while True: try: sheet = open_sheet(config['sheet_id'], config['worksheet_title'], config['creds']) backoff.reset() while True: start_jobs(jobs, sheet, restart_in_progress=restart_in_progress, restart_errors=restart_errors, restart_all=restart_all, no_update_state=no_update_state, ignore_not_ready=ignore_not_ready) if one_pass: raise Stop restart_in_progress = False # restart in progress on first pass only (if at all) restart_all = False gevent.sleep(interval) except Exception: logging.exception("Main loop failure") gevent.sleep(backoff.get()) except KeyboardInterrupt: logging.warning("Interrupt recieved") jobs.kill(block=True) except Stop: pass logging.info("Waiting for {} jobs".format(len(jobs.greenlets))) jobs.join()
def do_manual(row_id, *args): row_id = int(row_id) extra = [arg.split('=', 1) for arg in args] logging.basicConfig(level=logging.DEBUG) sheet = open_sheet(CONFIG['sheet_id'], CONFIG['worksheet_title'], CONFIG['creds']) row = get_rows(sheet)[row_id - 2] assert row['id'] == row_id for k, v in extra: if k not in row: raise Exception("bad override: {!r}".format(k)) row[k] = v process(sheet, row, no_update_state=True)
def do_manual(config, row_id, *args): with open(config) as f: config = json.load(f) row_id = int(row_id) extra = [arg.split('=', 1) for arg in args] logging.basicConfig(level=logging.DEBUG) sheet = open_sheet(config['sheet_id'], config['worksheet_title'], config['creds']) row = get_rows(sheet)[row_id-2] assert row['id'] == row_id for k, v in extra: if k not in row: raise Exception("bad override: {!r}".format(k)) row[k] = v process(sheet, row, no_update_state=True)
def main(interval=10, restart_in_progress=False, restart_errors=False, restart_all=False, no_update_state=False, log_level='DEBUG', one_pass=False): backdoor(6666) class Stop(BaseException): pass logging.basicConfig(level=log_level) jobs = gevent.pool.Pool(MAX_JOBS) backoff = Backoff(1, 60) try: while True: try: sheet = open_sheet(CONFIG['sheet_id'], CONFIG['worksheet_title'], CONFIG['creds']) backoff.reset() while True: start_jobs(jobs, sheet, restart_in_progress=restart_in_progress, restart_errors=restart_errors, restart_all=restart_all, no_update_state=no_update_state) if one_pass: raise Stop restart_in_progress = False # restart in progress on first pass only (if at all) restart_all = False gevent.sleep(interval) except Exception: logging.exception("Main loop failure") gevent.sleep(backoff.get()) except KeyboardInterrupt: logging.warning("Interrupt recieved") jobs.kill(block=True) except Stop: pass logging.info("Waiting for {} jobs".format(len(jobs.greenlets))) jobs.join()