def main(config, interval=10, restart_in_progress=False, restart_errors=False, restart_all=False, no_update_state=False, log_level='DEBUG', one_pass=False, ignore_not_ready=False):
with open(config) as f:
config = json.load(f)
backdoor(6666)
class Stop(BaseException): pass
logging.getLogger().setLevel(log_level)
jobs = gevent.pool.Pool(MAX_JOBS)
backoff = Backoff(1, 60)
try:
while True:
try:
sheet = open_sheet(config['sheet_id'], config['worksheet_title'], config['creds'])
backoff.reset()
while True:
start_jobs(jobs, sheet, restart_in_progress=restart_in_progress, restart_errors=restart_errors, restart_all=restart_all, no_update_state=no_update_state, ignore_not_ready=ignore_not_ready)
if one_pass:
raise Stop
restart_in_progress = False # restart in progress on first pass only (if at all)
restart_all = False
gevent.sleep(interval)
except Exception:
logging.exception("Main loop failure")
gevent.sleep(backoff.get())
except KeyboardInterrupt:
logging.warning("Interrupt recieved")
jobs.kill(block=True)
except Stop:
pass
logging.info("Waiting for {} jobs".format(len(jobs.greenlets)))
jobs.join()
def do_manual(row_id, *args):
row_id = int(row_id)
extra = [arg.split('=', 1) for arg in args]
logging.basicConfig(level=logging.DEBUG)
sheet = open_sheet(CONFIG['sheet_id'], CONFIG['worksheet_title'],
CONFIG['creds'])
row = get_rows(sheet)[row_id - 2]
assert row['id'] == row_id
for k, v in extra:
if k not in row:
raise Exception("bad override: {!r}".format(k))
row[k] = v
process(sheet, row, no_update_state=True)
def do_manual(config, row_id, *args):
with open(config) as f:
config = json.load(f)
row_id = int(row_id)
extra = [arg.split('=', 1) for arg in args]
logging.basicConfig(level=logging.DEBUG)
sheet = open_sheet(config['sheet_id'], config['worksheet_title'], config['creds'])
row = get_rows(sheet)[row_id-2]
assert row['id'] == row_id
for k, v in extra:
if k not in row:
raise Exception("bad override: {!r}".format(k))
row[k] = v
process(sheet, row, no_update_state=True)
def main(interval=10,
restart_in_progress=False,
restart_errors=False,
restart_all=False,
no_update_state=False,
log_level='DEBUG',
one_pass=False):
backdoor(6666)
class Stop(BaseException):
pass
logging.basicConfig(level=log_level)
jobs = gevent.pool.Pool(MAX_JOBS)
backoff = Backoff(1, 60)
try:
while True:
try:
sheet = open_sheet(CONFIG['sheet_id'],
CONFIG['worksheet_title'], CONFIG['creds'])
backoff.reset()
while True:
start_jobs(jobs,
sheet,
restart_in_progress=restart_in_progress,
restart_errors=restart_errors,
restart_all=restart_all,
no_update_state=no_update_state)
if one_pass:
raise Stop
restart_in_progress = False # restart in progress on first pass only (if at all)
restart_all = False
gevent.sleep(interval)
except Exception:
logging.exception("Main loop failure")
gevent.sleep(backoff.get())
except KeyboardInterrupt:
logging.warning("Interrupt recieved")
jobs.kill(block=True)
except Stop:
pass
logging.info("Waiting for {} jobs".format(len(jobs.greenlets)))
jobs.join()
