def Mongo_Wrapper(alias=None):
try:
if alias is None:
alias = str(uuid.uuid4())
db_alias_client = connect_db(alias)
yield alias, db_alias_client
finally:
disconnect(alias)
def run(self):
self.log(logging.INFO, "Running with a polling interval of {0} seconds".format(POLL_INTERVAL))
self.log(logging.INFO, "Credential waiting timeout is {0} seconds".format(WAITING_TIMEOUT))
self.log(logging.INFO, "Credential using timeout is {0} seconds".format(USING_TIMEOUT))
connection = mongo.connect_db()
db = connection.vinz_clortho
signal.signal(signal.SIGTERM, self.graceful_term)
self.SHOULD_BE_RUNNING = True
while self.SHOULD_BE_RUNNING:
self.IS_RUNNING = True
new_requests = CMRequest.find(db, filter={'status': CMRequest.SUBMITTED})
self.PROCESS_STEP = "New Requests"
self.PROCESS_COUNT = len(new_requests)
self.PROCESS_INDEX = 0
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Processing {0} new requests".format(self.PROCESS_COUNT))
self.log(logging.DEBUG, "Ids: {0}".format(', '.join([str(r.id) for r in new_requests])))
for new_request in new_requests:
self.PROCESS_INDEX += 1
credentials = Credential.find(db, filter={'key': new_request.key})
# If we didn't find any credentials, then error out this request
if len(credentials) == 0:
self.log(logging.ERROR, "No such key found: {0}".format(new_request.key), new_request)
new_request.update(status=CMRequest.NO_SUCH_KEY)
continue
self.log(logging.INFO, "Putting into queue", new_request)
new_request.update(status=CMRequest.QUEUING)
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Done processing new requests")
# This is mainly a formality, but just mark all the ones that are a
# status of CANCEL to CANCELED instead. This indicates that the
# server acknowledged the order to cancel
cancel_requests = CMRequest.find(db, filter={'status': CMRequest.CANCEL})
self.PROCESS_STEP = "Cancel Requests"
self.PROCESS_COUNT = len(cancel_requests)
self.PROCESS_INDEX = 0
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Processing {0} cancel requests".format(self.PROCESS_COUNT))
self.log(logging.DEBUG, "Ids: {0}".format(', '.join([str(r.id) for r in cancel_requests])))
for cancel_request in cancel_requests:
self.PROCESS_INDEX += 1
self.log(logging.INFO, "Canceled by client", cancel_request)
cancel_request.update(status=CMRequest.CANCELED)
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Done processing cancel requests")
returned_credentials = CMRequest.find(db, filter={'status': CMRequest.RETURNED})
self.PROCESS_STEP = "Returned Credentials"
self.PROCESS_COUNT = len(returned_credentials)
self.PROCESS_INDEX = 0
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Processing {0} returned credentials".format(self.PROCESS_COUNT))
self.log(logging.DEBUG, "Ids: {0}".format(', '.join([str(r.id) for r in returned_credentials])))
for returned_credential in returned_credentials:
self.PROCESS_INDEX += 1
credential = Credential.find_one(db, filter={'_id': returned_credential.credential})
returned_credential.status = CMRequest.COMPLETED
returned_credential.checkin_timestamp = datetime.now()
self.log(logging.INFO, "CredentialId {0} returned by client (Elapsed: {1:.1f}s)".format(
credential.id,
(returned_credential.checkin_timestamp - returned_credential.checkout_timestamp).total_seconds()
), returned_credential)
returned_credential.release()
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Done processing returned credentials")
pending_credentials = CMRequest.find(db, filter={'status': CMRequest.GIVEN_OUT})
self.PROCESS_STEP = "Pending Credentials"
self.PROCESS_COUNT = len(pending_credentials)
self.PROCESS_INDEX = 0
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Testing {0} given out credentials for time-out".format(self.PROCESS_COUNT))
self.log(logging.DEBUG, "Ids: {0}".format(', '.join([str(r.id) for r in pending_credentials])))
for pending_credential in pending_credentials:
self.PROCESS_INDEX += 1
if (datetime.now() - pending_credential.checkout_timestamp).total_seconds() > WAITING_TIMEOUT:
self.log(logging.WARNING, "Timed out waiting for client to receive credentials", pending_credential)
pending_credential.update(status=CMRequest.TIMED_OUT_WAITING)
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Done testing given-out credentials for time-out")
in_use_credentials = CMRequest.find(db, filter={
'status': {'$in': (CMRequest.IN_USE, CMRequest.CANCEL)},
'checkout_timestamp': {'$exists': True}
})
self.PROCESS_STEP = "In-use Credentials"
self.PROCESS_COUNT = len(in_use_credentials)
self.PROCESS_INDEX = 0
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Testing {0} in-use credentials for time-out".format(self.PROCESS_COUNT))
self.log(logging.DEBUG, "Ids: {0}".format(', '.join([str(r.id) for r in in_use_credentials])))
for in_use_credential in in_use_credentials:
self.PROCESS_INDEX += 1
if (datetime.now() - in_use_credential.checkup_timestamp).total_seconds() > USING_TIMEOUT:
self.log(logging.WARNING, "Timed out waiting for client to return credentials", in_use_credential)
in_use_credential.update(status=CMRequest.TIMED_OUT_USING)
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Done testing in-use credentials for time-out")
# This is the meat of the big loop. This section is the one that
# will be doling out the credentials on a first-come, first-serve
# basis, with priority overriding that.
queued_requests = CMRequest.find(db, filter={'status': CMRequest.QUEUING}, sort=
[('priority', mongo.DESCENDING), ('submission_timestamp', mongo.ASCENDING), ('_id', mongo.ASCENDING)]
)
self.PROCESS_STEP = "Queued Requests"
self.PROCESS_COUNT = len(queued_requests)
self.PROCESS_INDEX = 0
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Checking credentials to give out for {0} queued requests".format(
self.PROCESS_COUNT)
)
self.log(logging.DEBUG, "Ids: {0}".format(', '.join([str(r.id) for r in queued_requests])))
for queued_request in queued_requests:
self.PROCESS_INDEX += 1
available_credentials = Credential.find(db, filter={'key': queued_request.key})
if len(available_credentials) > 0:
self.log(logging.DEBUG, "Testing {0} available credentials for queued request".format(
len(available_credentials)
), queued_request)
self.log(logging.DEBUG, "Ids: {0}".format(', '.join([str(c.id) for c in available_credentials])))
for available_credential in available_credentials:
# Only give out credentials if there are any available to give out
in_use_credentials = available_credential.in_use
# Also, throttle how frequently we are allowed to give out this credential
credential_frequency = len(CMRequest.find(db, filter={
'credential': available_credential.id,
'status': {
'$in': (
CMRequest.GIVEN_OUT,
CMRequest.TIMED_OUT_WAITING,
CMRequest.IN_USE,
CMRequest.TIMED_OUT_USING,
CMRequest.RETURNED,
CMRequest.COMPLETED
)
},
'checkout_timestamp': {
'$gte': datetime.now() - available_credential.throttle_timespan
}
}))
if (available_credential.max_checkouts == 0 or
in_use_credentials < available_credential.max_checkouts) and \
(available_credential.throttle_seconds == 0 or credential_frequency == 0):
# We found a credential available to be used! Yay!
queued_request.check_out(available_credential)
self.log(logging.INFO, "Assigning CredentialId: {0} to client (waited {1:.1f}s)".format(
available_credential.id,
(queued_request.checkout_timestamp - queued_request.submission_timestamp).total_seconds()
), queued_request)
break
if self.PROCESS_COUNT > 0:
self.log(logging.DEBUG, "Done checking credentials to give out for queued requests")
self.IS_RUNNING = False
self.log(logging.DEBUG, "Going to sleep for {0} seconds".format(POLL_INTERVAL))
time.sleep(POLL_INTERVAL)
def connect_db():
return mongo.connect_db()
