def OAuthManager(once=False, loop_rate=300, max_rows=100):
"""
Main loop to delete all expired tokens, refresh tokens eligible
for refresh and delete all expired OAuth session parameters.
It was decided to have only 1 daemon for all 3 of these cleanup activities.
:param once: If True, the loop is run just once, otherwise the daemon continues looping until stopped.
:param loop_rate: The number of seconds the daemon will wait before running next loop of operations.
:param max_rows: Max number of DB rows to deal with per operation.
:returns: None
"""
executable = 'oauth-manager'
total_workers = 0
worker_number = 0
sanity_check(executable=executable, hostname=socket.gethostname())
# make an initial heartbeat
live(executable=executable, hostname=socket.gethostname(), pid=os.getpid(), thread=threading.current_thread())
# wait a moment in case all workers started at the same time
GRACEFUL_STOP.wait(1)
while not GRACEFUL_STOP.is_set():
start = time.time()
# issuing the heartbeat for a second time to make all workers aware of each other
heartbeat = live(executable=executable, hostname=socket.gethostname(), pid=os.getpid(), thread=threading.current_thread())
total_workers = heartbeat['nr_threads']
worker_number = heartbeat['assign_thread'] + 1
ndeleted = 0
ndeletedreq = 0
nrefreshed = 0
try:
# ACCESS TOKEN REFRESH - better to run first (in case some of the refreshed tokens needed deletion after this step)
logging.info('oauth_manager[%i/%i]: ----- START ----- ACCESS TOKEN REFRESH ----- ', worker_number, total_workers)
logging.info('oauth_manager[%i/%i]: starting to query tokens for automatic refresh', worker_number, total_workers)
nrefreshed = refresh_jwt_tokens(total_workers, worker_number, refreshrate=int(loop_rate), limit=max_rows)
logging.info('oauth_manager[%i/%i]: successfully refreshed %i tokens', worker_number, total_workers, nrefreshed)
logging.info('oauth_manager[%i/%i]: ----- END ----- ACCESS TOKEN REFRESH ----- ', worker_number, total_workers)
record_counter(counters='oauth_manager.tokens.refreshed', delta=nrefreshed)
except (DatabaseException, DatabaseError) as err:
if match('.*QueuePool.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
elif match('.*ORA-03135.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
else:
logging.critical(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
except Exception as err:
logging.critical(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
try:
# waiting 1 sec as DBs does not store milisecond and tokens
# eligible for deletion after refresh might not get dleeted otherwise
GRACEFUL_STOP.wait(1)
# EXPIRED TOKEN DELETION
logging.info('oauth_manager[%i/%i]: ----- START ----- DELETION OF EXPIRED TOKENS ----- ', worker_number, total_workers)
logging.info('oauth_manager[%i/%i]: starting to delete expired tokens', worker_number, total_workers)
ndeleted += delete_expired_tokens(total_workers, worker_number, limit=max_rows)
logging.info('oauth_manager[%i/%i]: deleted %i expired tokens', worker_number, total_workers, ndeleted)
logging.info('oauth_manager[%i/%i]: ----- END ----- DELETION OF EXPIRED TOKENS ----- ', worker_number, total_workers)
record_counter(counters='oauth_manager.tokens.deleted', delta=ndeleted)
except (DatabaseException, DatabaseError) as err:
if match('.*QueuePool.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
elif match('.*ORA-03135.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
else:
logging.critical(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
except Exception as err:
logging.critical(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
try:
# DELETING EXPIRED OAUTH SESSION PARAMETERS
logging.info('oauth_manager[%i/%i]: ----- START ----- DELETION OF EXPIRED OAUTH SESSION REQUESTS ----- ', worker_number, total_workers)
logging.info('oauth_manager[%i/%i]: starting deletion of expired OAuth session requests', worker_number, total_workers)
ndeletedreq += delete_expired_oauthrequests(total_workers, worker_number, limit=max_rows)
logging.info('oauth_manager[%i/%i]: expired parameters of %i authentication requests were deleted', worker_number, total_workers, ndeletedreq)
logging.info('oauth_manager[%i/%i]: ----- END ----- DELETION OF EXPIRED OAUTH SESSION REQUESTS ----- ', worker_number, total_workers)
record_counter(counters='oauth_manager.oauthreq.deleted', delta=ndeletedreq)
except (DatabaseException, DatabaseError) as err:
if match('.*QueuePool.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
elif match('.*ORA-03135.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
else:
logging.critical(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
except Exception as err:
logging.critical(traceback.format_exc())
record_counter('oauth_manager.exceptions.' + err.__class__.__name__)
tottime = time.time() - start
logging.info('oauth_manager[%i/%i]: took %f seconds to delete %i tokens, %i session parameters and refreshed %i tokens', worker_number, total_workers, tottime, ndeleted, ndeletedreq, nrefreshed)
record_timer(stat='oauth_manager.duration', time=1000 * tottime)
if once:
break
else:
logging.info('oauth_manager[%i/%i]: Sleeping for %i seconds.', worker_number, total_workers, loop_rate)
GRACEFUL_STOP.wait(loop_rate)
die(executable=executable, hostname=socket.gethostname(), pid=os.getpid(), thread=threading.current_thread())
logging.info('oauth_manager[%i/%i]: graceful stop done', worker_number, total_workers)
def OAuthManager(once=False, loop_rate=300, max_rows=100, sleep_time=300):
"""
Main loop to delete all expired tokens, refresh tokens eligible
for refresh and delete all expired OAuth session parameters.
It was decided to have only 1 daemon for all 3 of these cleanup activities.
:param once: If True, the loop is run just once, otherwise the daemon continues looping until stopped.
:param loop_rate: obsolete, please use sleep_time instead. The number of seconds the daemon will wait before running next loop of operations.
:param max_rows: Max number of DB rows to deal with per operation.
:param sleep_time: The number of seconds the daemon will wait before running next loop of operations.
:returns: None
"""
if sleep_time == OAuthManager.__defaults__[3] and loop_rate != OAuthManager.__defaults__[1]:
sleep_time = loop_rate
executable = 'oauth-manager'
sanity_check(executable=executable, hostname=socket.gethostname())
# make an initial heartbeat
heartbeat = live(executable=executable, hostname=socket.gethostname(), pid=os.getpid(), thread=threading.current_thread())
prepend_str = 'oauth_manager [%i/%i] : ' % (heartbeat['assign_thread'], heartbeat['nr_threads'])
logger = formatted_logger(logging.log, prepend_str + '%s')
# wait a moment in case all workers started at the same time
GRACEFUL_STOP.wait(1)
while not GRACEFUL_STOP.is_set():
start = time.time()
# issuing the heartbeat for a second time to make all workers aware of each other
heartbeat = live(executable=executable, hostname=socket.gethostname(), pid=os.getpid(), thread=threading.current_thread())
prepend_str = 'oauth_manager [%i/%i] : ' % (heartbeat['assign_thread'], heartbeat['nr_threads'])
logger = formatted_logger(logging.log, prepend_str + '%s')
ndeleted = 0
ndeletedreq = 0
nrefreshed = 0
try:
# ACCESS TOKEN REFRESH - better to run first (in case some of the refreshed tokens needed deletion after this step)
logger(logging.INFO, '----- START ----- ACCESS TOKEN REFRESH ----- ')
logger(logging.INFO, 'starting to query tokens for automatic refresh')
nrefreshed = refresh_jwt_tokens(heartbeat['nr_threads'], heartbeat['assign_thread'] + 1, refreshrate=int(sleep_time), limit=max_rows)
logger(logging.INFO, 'successfully refreshed %i tokens', nrefreshed)
logger(logging.INFO, '----- END ----- ACCESS TOKEN REFRESH ----- ')
record_counter(name='oauth_manager.tokens.refreshed', delta=nrefreshed)
except (DatabaseException, DatabaseError) as err:
if match('.*QueuePool.*', str(err.args[0])):
logger(logging.WARNING, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
elif match('.*ORA-03135.*', str(err.args[0])):
logger(logging.WARNING, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
else:
logger(logging.CRITICAL, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
except Exception as err:
logger(logging.CRITICAL, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
try:
# waiting 1 sec as DBs does not store milisecond and tokens
# eligible for deletion after refresh might not get dleeted otherwise
GRACEFUL_STOP.wait(1)
# EXPIRED TOKEN DELETION
logger(logging.INFO, '----- START ----- DELETION OF EXPIRED TOKENS ----- ')
logger(logging.INFO, 'starting to delete expired tokens')
ndeleted += delete_expired_tokens(heartbeat['nr_threads'], heartbeat['assign_thread'] + 1, limit=max_rows)
logger(logging.INFO, 'deleted %i expired tokens', ndeleted)
logger(logging.INFO, '----- END ----- DELETION OF EXPIRED TOKENS ----- ')
record_counter(name='oauth_manager.tokens.deleted', delta=ndeleted)
except (DatabaseException, DatabaseError) as err:
if match('.*QueuePool.*', str(err.args[0])):
logger(logging.WARNING, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
elif match('.*ORA-03135.*', str(err.args[0])):
logger(logging.WARNING, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
else:
logger(logging.CRITICAL, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
except Exception as err:
logger(logging.CRITICAL, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
try:
# DELETING EXPIRED OAUTH SESSION PARAMETERS
logger(logging.INFO, '----- START ----- DELETION OF EXPIRED OAUTH SESSION REQUESTS ----- ')
logger(logging.INFO, 'starting deletion of expired OAuth session requests')
ndeletedreq += delete_expired_oauthrequests(heartbeat['nr_threads'], heartbeat['assign_thread'] + 1, limit=max_rows)
logger(logging.INFO, 'expired parameters of %i authentication requests were deleted', ndeletedreq)
logger(logging.INFO, '----- END ----- DELETION OF EXPIRED OAUTH SESSION REQUESTS ----- ')
record_counter(name='oauth_manager.oauthreq.deleted', delta=ndeletedreq)
except (DatabaseException, DatabaseError) as err:
if match('.*QueuePool.*', str(err.args[0])):
logger(logging.WARNING, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
elif match('.*ORA-03135.*', str(err.args[0])):
logger(logging.WARNING, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
else:
logger(logging.CRITICAL, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
except Exception as err:
logger(logging.CRITICAL, traceback.format_exc())
record_counter('oauth_manager.exceptions.{exception}', labels={'exception': err.__class__.__name__})
tottime = time.time() - start
logger(logging.INFO, 'took %f seconds to delete %i tokens, %i session parameters and refreshed %i tokens', tottime, ndeleted, ndeletedreq, nrefreshed)
record_timer(name='oauth_manager.duration', time=1000 * tottime)
if once:
break
else:
daemon_sleep(start_time=start, sleep_time=sleep_time, graceful_stop=GRACEFUL_STOP)
die(executable=executable, hostname=socket.gethostname(), pid=os.getpid(), thread=threading.current_thread())
logger(logging.INFO, 'graceful stop done')