def release_node_lock(lockserver_handle, nodeID):
""" Release the node lock and Destroy the lockserver handle"""
#release the node lock and destroy lock handle
lockserver.unlock_node(lockserver_handle, nodeID)
log("released lock for node: "+nodeID)
lockserver.destroy_lockserver_handle(lockserver_handle)
log("Destroyed lockserver_handle for node: "+nodeID)
def regenerate_api_key(geniuser):
"""
<Purpose>
Regenerates the user's API key.
<Arguments>
geniuser
A GeniUser object of the user whose api key is to be regenerated.
<Exceptions>
None
<Side Effects>
The API key for the user is updated in the database.
<Returns>
The new API key.
"""
assert_geniuser(geniuser)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
return maindb.regenerate_api_key(geniuser)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def regenerate_api_key(geniuser):
"""
<Purpose>
Regenerates the user's API key.
<Arguments>
geniuser
A GeniUser object of the user whose api key is to be regenerated.
<Exceptions>
None
<Side Effects>
The API key for the user is updated in the database.
<Returns>
The new API key.
"""
assert_geniuser(geniuser)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
return maindb.regenerate_api_key(geniuser)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def release_node_lock(lockserver_handle, nodeID):
""" Release the node lock and Destroy the lockserver handle"""
#release the node lock and destroy lock handle
lockserver.unlock_node(lockserver_handle, nodeID)
log("released lock for node: "+nodeID)
lockserver.destroy_lockserver_handle(lockserver_handle)
log("Destroyed lockserver_handle for node: "+nodeID)
def _cleanup_single_vessel(vessel):
"""
This function is passed by cleanup_vessels() as the function argument to
run_parallelized().
"""
# This does seem wasteful of lockserver communication to require four
# round-trips with the lockserver (get handle, lock, unlock, release handle),
# but if we really want to fix that then I think the best thing to do would
# be to allow obtaining a lockhandle and releasing a lockhandle to be done
# in the same calls as lock acquisition and release.
node_id = maindb.get_node_identifier_from_vessel(vessel)
lockserver_handle = lockserver.create_lockserver_handle()
# Lock the node that the vessels is on.
lockserver.lock_node(lockserver_handle, node_id)
try:
# Get a new vessel object from the db in case it was modified in the db
# before the lock was obtained.
vessel = maindb.get_vessel(node_id, vessel.name)
# Now that we have a lock on the node that this vessel is on, find out
# if we should still clean up this vessel (e.g. maybe a node state
# transition script moved the node to a new state and this vessel was
# removed).
needscleanup, reasonwhynot = maindb.does_vessel_need_cleanup(vessel)
if not needscleanup:
log.info("[_cleanup_single_vessel] Vessel " + str(vessel) +
" no longer needs cleanup: " + reasonwhynot)
return
nodeid = maindb.get_node_identifier_from_vessel(vessel)
nodehandle = _get_node_handle_from_nodeid(nodeid)
try:
log.info("[_cleanup_single_vessel] About to ChangeUsers on vessel " + str(vessel))
nodemanager.change_users(nodehandle, vessel.name, [''])
log.info("[_cleanup_single_vessel] About to ResetVessel on vessel " + str(vessel))
nodemanager.reset_vessel(nodehandle, vessel.name)
except NodemanagerCommunicationError:
# We don't pass this exception up. Maybe the node is offline now. At some
# point, it will be marked in the database as offline (should we be doing
# that here?). At that time, the dirty vessels on that node will not be
# in the cleanup list anymore.
log.info("[_cleanup_single_vessel] Failed to cleanup vessel " +
str(vessel) + ". " + traceback.format_exc())
return
# We only mark it as clean if no exception was raised when trying to
# perform the above nodemanager operations.
maindb.mark_vessel_as_clean(vessel)
log.info("[_cleanup_single_vessel] Successfully cleaned up vessel " + str(vessel))
finally:
# Unlock the node.
lockserver.unlock_node(lockserver_handle, node_id)
lockserver.destroy_lockserver_handle(lockserver_handle)
def ban_user_and_remove_vessels(username):
try:
geniuser = maindb.get_user(username, allow_inactive=True)
except DoesNotExistError:
print "No such user: %s." % username
sys.exit(1)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
if geniuser.is_active:
geniuser.is_active = False
geniuser.save()
print "This account has been set to inactive (banned)."
else:
print "This account is already inactive (banned)."
acquired_vessels = maindb.get_acquired_vessels(geniuser)
if not acquired_vessels:
print "No acquired vessels to stop/remove access to."
else:
print "Vessels acquired by this user: %s" % acquired_vessels
print "Indicating to the backend to release %s vessels." % len(
acquired_vessels)
vessels.release_vessels(lockserver_handle, geniuser,
acquired_vessels)
print "Release indicated. Monitoring db to see if the backend cleaned them up."
while True:
for vessel in acquired_vessels[:]:
updated_vessel = maindb.get_vessel(
vessel.node.node_identifier, vessel.name)
if vessel.node.is_broken or not vessel.node.is_active:
print "Node %s is broken or inactive, so backend won't contact it." % vessel.node
acquired_vessels.remove(vessel)
continue
if updated_vessel.is_dirty:
print "Vessel %s has not been cleaned up yet." % updated_vessel
else:
print "Vessel %s has been cleaned up." % updated_vessel
acquired_vessels.remove(vessel)
if not acquired_vessels:
print "All vessels have been cleaned up."
break
else:
print "%s vessels remain to be cleaned up." % len(
acquired_vessels)
print "Sleeping 10 seconds."
time.sleep(10)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def change_user_keys(geniuser, pubkey=None):
"""
<Purpose>
Sets a new public/private key for the user and initiates the change
of user keys on all vessels the user has access to. If pubkey is
provided, that is used as the user's new pubkey. If pubkey is not
provided, a new public/private keypair is generated for the user.
<Arguments>
geniuser
A GeniUser object of the user whose keys are to be updated.
<Exceptions>
ValidationError
If the pubkey is provided and is invalid.
<Side Effects>
The public and private keys of the user are replaced in the database with
new keys (if pubkey was provided, the private key in the database will
be empty, otherwise it will be the generated private key). All vessels the
user has access to are marked as needing to have their user keys sync'd.
<Returns>
None
"""
assert_geniuser(geniuser)
if pubkey is not None:
validations.validate_pubkey_string(pubkey)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Get a key pair from the keygen api if the user didn't supply their own pubkey.
if pubkey is None:
(pubkey, privkey) = keygen.generate_keypair()
else:
privkey = None
maindb.set_user_keys(geniuser, pubkey, privkey)
# Now we need to find all of the vessels the user has access to and set
# them to have their user keys updated by the backend.
vessel_list = maindb.get_vessels_accessible_by_user(geniuser)
if vessel_list:
vessels.flag_vessels_for_user_keys_sync(lockserver_handle,
vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def renew_vessels(geniuser, vessel_list):
"""
<Purpose>
Extend the expiration dates of vessels acquired by a user.
<Arguments>
geniuser
The GeniUser whose vessels are to be renewed.
vessel_list
A list of vessels to be renewed.
<Exceptions>
InvalidRequestError
If any of the vessels in the vessel_list are not currently acquired by
geniuser or if the list of vessels is empty.
InsufficientUserResourcesError
If the user is currently over their limit of acquired resources.
<Side Effects>
The vessels are renewed to the maximum time vessels can be acquired for,
regardless of their previous individual expiration times.
<Returns>
None
"""
assert_geniuser(geniuser)
assert_list(vessel_list)
for vessel in vessel_list:
assert_vessel(vessel)
if not vessel_list:
raise InvalidRequestError("The list of vessels cannot be empty.")
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Ensure the user is not over their limit of acquired vessels due to
# donations of theirs having gone offline. This call will raise an
# InsufficientUserResourcesError if the user is currently over their
# limit.
vesselcount = 0
maindb.require_user_can_acquire_resources(geniuser, vesselcount)
# The vessels.renew_vessels function is responsible for ensuring that the
# vessels belong to this user. We let the other function do the check
# because we want to hold locks on the vessels' nodes before checking.
vessels.renew_vessels(lockserver_handle, geniuser, vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def renew_vessels(geniuser, vessel_list):
"""
<Purpose>
Extend the expiration dates of vessels acquired by a user.
<Arguments>
geniuser
The GeniUser whose vessels are to be renewed.
vessel_list
A list of vessels to be renewed.
<Exceptions>
InvalidRequestError
If any of the vessels in the vessel_list are not currently acquired by
geniuser or if the list of vessels is empty.
InsufficientUserResourcesError
If the user is currently over their limit of acquired resources.
<Side Effects>
The vessels are renewed to the maximum time vessels can be acquired for,
regardless of their previous individual expiration times.
<Returns>
None
"""
assert_geniuser(geniuser)
assert_list(vessel_list)
for vessel in vessel_list:
assert_vessel(vessel)
if not vessel_list:
raise InvalidRequestError("The list of vessels cannot be empty.")
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Ensure the user is not over their limit of acquired vessels due to
# donations of theirs having gone offline. This call will raise an
# InsufficientUserResourcesError if the user is currently over their
# limit.
vesselcount = 0
maindb.require_user_can_acquire_resources(geniuser, vesselcount)
# The vessels.renew_vessels function is responsible for ensuring that the
# vessels belong to this user. We let the other function do the check
# because we want to hold locks on the vessels' nodes before checking.
vessels.renew_vessels(lockserver_handle, geniuser, vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def change_user_keys(geniuser, pubkey=None):
"""
<Purpose>
Sets a new public/private key for the user and initiates the change
of user keys on all vessels the user has access to. If pubkey is
provided, that is used as the user's new pubkey. If pubkey is not
provided, a new public/private keypair is generated for the user.
<Arguments>
geniuser
A GeniUser object of the user whose keys are to be updated.
<Exceptions>
ValidationError
If the pubkey is provided and is invalid.
<Side Effects>
The public and private keys of the user are replaced in the database with
new keys (if pubkey was provided, the private key in the database will
be empty, otherwise it will be the generated private key). All vessels the
user has access to are marked as needing to have their user keys sync'd.
<Returns>
None
"""
assert_geniuser(geniuser)
if pubkey is not None:
validations.validate_pubkey_string(pubkey)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Get a key pair from the keygen api if the user didn't supply their own pubkey.
if pubkey is None:
(pubkey, privkey) = keygen.generate_keypair()
else:
privkey = None
maindb.set_user_keys(geniuser, pubkey, privkey)
# Now we need to find all of the vessels the user has access to and set
# them to have their user keys updated by the backend.
vessel_list = maindb.get_vessels_accessible_by_user(geniuser)
if vessel_list:
vessels.flag_vessels_for_user_keys_sync(lockserver_handle, vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def ban_user_and_remove_vessels(username):
try:
geniuser = maindb.get_user(username, allow_inactive=True)
except DoesNotExistError:
print "No such user: %s." % username
sys.exit(1)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
if geniuser.is_active:
geniuser.is_active = False
geniuser.save()
print "This account has been set to inactive (banned)."
else:
print "This account is already inactive (banned)."
acquired_vessels = maindb.get_acquired_vessels(geniuser)
if not acquired_vessels:
print "No acquired vessels to stop/remove access to."
else:
print "Vessels acquired by this user: %s" % acquired_vessels
print "Indicating to the backend to release %s vessels." % len(acquired_vessels)
vessels.release_vessels(lockserver_handle, geniuser, acquired_vessels)
print "Release indicated. Monitoring db to see if the backend cleaned them up."
while True:
for vessel in acquired_vessels[:]:
updated_vessel = maindb.get_vessel(vessel.node.node_identifier, vessel.name)
if vessel.node.is_broken or not vessel.node.is_active:
print "Node %s is broken or inactive, so backend won't contact it." % vessel.node
acquired_vessels.remove(vessel)
continue
if updated_vessel.is_dirty:
print "Vessel %s has not been cleaned up yet." % updated_vessel
else:
print "Vessel %s has been cleaned up." % updated_vessel
acquired_vessels.remove(vessel)
if not acquired_vessels:
print "All vessels have been cleaned up."
break
else:
print "%s vessels remain to be cleaned up." % len(acquired_vessels)
print "Sleeping 10 seconds."
time.sleep(10)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def acquire_specific_vessels(geniuser, vessel_list):
"""
<Purpose>
Attempt to acquire specific vessels for a user.
<Arguments>
geniuser
The GeniUser which will be assigned the vessels.
vessel_list
A list of vessels to be acquired for the user.
<Exceptions>
InsufficientUserResourcesError
The user does not have enough vessel credits to acquire the number of
vessels requested.
InvalidRequestError
If the list of vessels is empty.
<Side Effects>
Zero or more of the vessels in vessel_list have been acquired by the user.
<Returns>
A list of the vessels acquired as a result of this function call. The
length of this list may be less than the length of vessel_list if one or
more of the vessels in vessel_list could not be acquired.
"""
assert_geniuser(geniuser)
assert_list(vessel_list)
for vessel in vessel_list:
assert_vessel(vessel)
if not vessel_list:
raise InvalidRequestError("The list of vessels cannot be empty.")
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Ensure the user is allowed to acquire these resources. This call will
# raise an InsufficientUserResourcesError if the additional vessels would
# cause the user to be over their limit.
maindb.require_user_can_acquire_resources(geniuser, len(vessel_list))
return vessels.acquire_specific_vessels_best_effort(
lockserver_handle, geniuser, vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def acquire_specific_vessels(geniuser, vessel_list):
"""
<Purpose>
Attempt to acquire specific vessels for a user.
<Arguments>
geniuser
The GeniUser which will be assigned the vessels.
vessel_list
A list of vessels to be acquired for the user.
<Exceptions>
InsufficientUserResourcesError
The user does not have enough vessel credits to acquire the number of
vessels requested.
InvalidRequestError
If the list of vessels is empty.
<Side Effects>
Zero or more of the vessels in vessel_list have been acquired by the user.
<Returns>
A list of the vessels acquired as a result of this function call. The
length of this list may be less than the length of vessel_list if one or
more of the vessels in vessel_list could not be acquired.
"""
assert_geniuser(geniuser)
assert_list(vessel_list)
for vessel in vessel_list:
assert_vessel(vessel)
if not vessel_list:
raise InvalidRequestError("The list of vessels cannot be empty.")
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Ensure the user is allowed to acquire these resources. This call will
# raise an InsufficientUserResourcesError if the additional vessels would
# cause the user to be over their limit.
maindb.require_user_can_acquire_resources(geniuser, len(vessel_list))
return vessels.acquire_specific_vessels_best_effort(lockserver_handle, geniuser, vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def stop_all_vessels_on_node(node_id):
try:
node = maindb.get_node(node_id)
except DoesNotExistError:
print "No such node"
sys.exit(1)
if not node.is_active:
print "This node is marked as inactive, thus the backend will not try to clean up vessels."
sys.exit(0)
if node.is_broken:
print "This node is marked as broken, thus the backend will not try to clean up vessels."
sys.exit(0)
vessels_on_node = maindb.get_vessels_on_node(node)
if not vessels_on_node:
print "No vessels on node."
sys.exit(0)
lockserver_handle = lockserver.create_lockserver_handle()
try:
print "Indicating to the backend to release/reset all %s vessels." % len(
vessels_on_node)
lockserver.lock_node(lockserver_handle, node_id)
try:
for vessel in vessels_on_node:
maindb.record_released_vessel(vessel)
finally:
lockserver.unlock_node(lockserver_handle, node_id)
print "Releases indicated. Monitoring db to see if the backend cleaned them up."
while True:
for vessel in vessels_on_node[:]:
updated_vessel = maindb.get_vessel(node_id, vessel.name)
if updated_vessel.is_dirty:
print "Vessel %s has not been cleaned up yet." % updated_vessel
else:
print "Vessel %s has been cleaned up." % updated_vessel
vessels_on_node.remove(vessel)
if not vessels_on_node:
print "All vessels have been cleaned up."
break
else:
print "%s vessels remain to be cleaned up." % len(
vessels_on_node)
print "Sleeping 10 seconds."
time.sleep(10)
finally:
lockserver.destroy_lockserver_handle(lockserver_handle)
def stop_all_vessels_on_node(node_id):
try:
node = maindb.get_node(node_id)
except DoesNotExistError:
print "No such node"
sys.exit(1)
if not node.is_active:
print "This node is marked as inactive, thus the backend will not try to clean up vessels."
sys.exit(0)
if node.is_broken:
print "This node is marked as broken, thus the backend will not try to clean up vessels."
sys.exit(0)
vessels_on_node = maindb.get_vessels_on_node(node)
if not vessels_on_node:
print "No vessels on node."
sys.exit(0)
lockserver_handle = lockserver.create_lockserver_handle()
try:
print "Indicating to the backend to release/reset all %s vessels." % len(vessels_on_node)
lockserver.lock_node(lockserver_handle, node_id)
try:
for vessel in vessels_on_node:
maindb.record_released_vessel(vessel)
finally:
lockserver.unlock_node(lockserver_handle, node_id)
print "Releases indicated. Monitoring db to see if the backend cleaned them up."
while True:
for vessel in vessels_on_node[:]:
updated_vessel = maindb.get_vessel(node_id, vessel.name)
if updated_vessel.is_dirty:
print "Vessel %s has not been cleaned up yet." % updated_vessel
else:
print "Vessel %s has been cleaned up." % updated_vessel
vessels_on_node.remove(vessel)
if not vessels_on_node:
print "All vessels have been cleaned up."
break
else:
print "%s vessels remain to be cleaned up." % len(vessels_on_node)
print "Sleeping 10 seconds."
time.sleep(10)
finally:
lockserver.destroy_lockserver_handle(lockserver_handle)
def release_vessels(geniuser, vessel_list):
"""
<Purpose>
Remove a user from a vessel that is assigned to the user.
<Arguments>
geniuser
The GeniUser who is to be removed from the vessel.
vessel_list
A list of vessels the user is to be removed from.
<Exceptions>
InvalidRequestError
If any of the vessels in the vessel_list are not currently acquired by
geniuser or if the list of vessels is empty.
<Side Effects>
The vessel is no longer assigned to the user. If this was the last user
assigned to the vessel, the vessel is freed.
<Returns>
None
"""
assert_geniuser(geniuser)
assert_list(vessel_list)
for vessel in vessel_list:
assert_vessel(vessel)
if not vessel_list:
raise InvalidRequestError("The list of vessels cannot be empty.")
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
vessels.release_vessels(lockserver_handle, geniuser, vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def release_vessels(geniuser, vessel_list):
"""
<Purpose>
Remove a user from a vessel that is assigned to the user.
<Arguments>
geniuser
The GeniUser who is to be removed from the vessel.
vessel_list
A list of vessels the user is to be removed from.
<Exceptions>
InvalidRequestError
If any of the vessels in the vessel_list are not currently acquired by
geniuser or if the list of vessels is empty.
<Side Effects>
The vessel is no longer assigned to the user. If this was the last user
assigned to the vessel, the vessel is freed.
<Returns>
None
"""
assert_geniuser(geniuser)
assert_list(vessel_list)
for vessel in vessel_list:
assert_vessel(vessel)
if not vessel_list:
raise InvalidRequestError("The list of vessels cannot be empty.")
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
vessels.release_vessels(lockserver_handle, geniuser, vessel_list)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def change_user_affiliation(geniuser, new_affiliation):
"""
<Purpose>
Sets a new affiliation for the user
<Arguments>
geniuser
A GeniUser object of the user whose affiliation is to be updated.
new_affiliation
the new affiliation value
<Exceptions>
ValidationError
If the affiliation is provided and is invalid.
<Side Effects>
The geniuser affiliation gets changed to the new value(in the db).
<Returns>
None
"""
assert_geniuser(geniuser)
#Determines if the new affiliation is valid. The frontend should already
#checks for this but we validate again here just in case.
validations.validate_affiliation(new_affiliation)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
maindb.set_user_affiliation(geniuser, new_affiliation)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def change_user_password(geniuser, new_password):
"""
<Purpose>
Sets a new password for the geniuser.
<Arguments>
geniuser
A GeniUser object of the user whose password is to be changed.
new_password
the user specificed new password value.
<Exceptions>
ValidationError
If the password is provided and is invalid.
<Side Effects>
The geniuser password gets changed to the new value (in the db).
<Returns>
None
"""
assert_geniuser(geniuser)
# Determines if the new password is strong enough. The frontend should already
# check for this but we validate again here just in case.
validations.validate_password(new_password)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
maindb.set_user_password(geniuser, new_password)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def delete_private_key(geniuser):
"""
<Purpose>
Deletes the private key of the specified user.
<Arguments>
geniuser
A GeniUser object of the user whose private key is to be deleted.
<Exceptions>
None
<Side Effects>
The private key belonging to the user is deleted if it exists, otherwise
the user account is not modified.
<Returns>
None
"""
assert_geniuser(geniuser)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect the deletion of the key.
# That is, we want the object passed in to have the user_privkey be None
# when this function returns.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
maindb.delete_user_private_key(geniuser)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def delete_private_key(geniuser):
"""
<Purpose>
Deletes the private key of the specified user.
<Arguments>
geniuser
A GeniUser object of the user whose private key is to be deleted.
<Exceptions>
None
<Side Effects>
The private key belonging to the user is deleted if it exists, otherwise
the user account is not modified.
<Returns>
None
"""
assert_geniuser(geniuser)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect the deletion of the key.
# That is, we want the object passed in to have the user_privkey be None
# when this function returns.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
maindb.delete_user_private_key(geniuser)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def change_user_port(geniuser, new_port):
"""
<Purpose>
Sets a new port for the user
<Arguments>
geniuser
A GeniUser object of the user whose port is to be changed.
new_port
the new port value
<Exceptions>
ValidationError
If the port is provided and it is not in the allowed range.
<Side Effects>
the geniuser port gets changed to the new value(in the db).
<Returns>
None
"""
assert_geniuser(geniuser)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
maindb.set_user_port(geniuser, new_port)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def change_user_port(geniuser, new_port):
"""
<Purpose>
Sets a new port for the user
<Arguments>
geniuser
A GeniUser object of the user whose port is to be changed.
new_port
the new port value
<Exceptions>
ValidationError
If the port is provided and it is not in the allowed range.
<Side Effects>
the geniuser port gets changed to the new value(in the db).
<Returns>
None
"""
assert_geniuser(geniuser)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
# We don't use the user object we retrieve because we want the
# object passed in to the function to reflect changes we make to the object.
try:
maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
maindb.set_user_port(geniuser, new_port)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def _cleanup_single_vessel(vessel):
"""
This function is passed by cleanup_vessels() as the function argument to
run_parallelized().
"""
# This does seem wasteful of lockserver communication to require four
# round-trips with the lockserver (get handle, lock, unlock, release handle),
# but if we really want to fix that then I think the best thing to do would
# be to allow obtaining a lockhandle and releasing a lockhandle to be done
# in the same calls as lock acquisition and release.
node_id = maindb.get_node_identifier_from_vessel(vessel)
lockserver_handle = lockserver.create_lockserver_handle()
# Lock the node that the vessels is on.
lockserver.lock_node(lockserver_handle, node_id)
try:
# Get a new vessel object from the db in case it was modified in the db
# before the lock was obtained.
vessel = maindb.get_vessel(node_id, vessel.name)
# Now that we have a lock on the node that this vessel is on, find out
# if we should still clean up this vessel (e.g. maybe a node state
# transition script moved the node to a new state and this vessel was
# removed).
needscleanup, reasonwhynot = maindb.does_vessel_need_cleanup(vessel)
if not needscleanup:
log.info("[_cleanup_single_vessel] Vessel " + str(vessel) +
" no longer needs cleanup: " + reasonwhynot)
return
nodeid = maindb.get_node_identifier_from_vessel(vessel)
nodehandle = _get_node_handle_from_nodeid(nodeid)
try:
log.info(
"[_cleanup_single_vessel] About to ChangeUsers on vessel " +
str(vessel))
nodemanager.change_users(nodehandle, vessel.name, [''])
log.info(
"[_cleanup_single_vessel] About to ResetVessel on vessel " +
str(vessel))
nodemanager.reset_vessel(nodehandle, vessel.name)
except NodemanagerCommunicationError:
# We don't pass this exception up. Maybe the node is offline now. At some
# point, it will be marked in the database as offline (should we be doing
# that here?). At that time, the dirty vessels on that node will not be
# in the cleanup list anymore.
log.info("[_cleanup_single_vessel] Failed to cleanup vessel " +
str(vessel) + ". " + traceback.format_exc())
return
# We only mark it as clean if no exception was raised when trying to
# perform the above nodemanager operations.
maindb.mark_vessel_as_clean(vessel)
log.info("[_cleanup_single_vessel] Successfully cleaned up vessel " +
str(vessel))
finally:
# Unlock the node.
lockserver.unlock_node(lockserver_handle, node_id)
lockserver.destroy_lockserver_handle(lockserver_handle)
def acquire_vessels(geniuser, vesselcount, vesseltype):
"""
<Purpose>
Acquire unused vessels of a given type for a user. For information on how
the specified vesseltype affects which vessels will be considered
to satisfy the request, see the type-specific functions that are called
by this function.
<Arguments>
geniuser
The GeniUser which will be assigned the vessels.
vesselcount
The number of vessels to acquire (a positive integer).
vesseltype
The type of vessels to acquire. One of either 'lan', 'wan', 'nat', or 'rand'.
<Exceptions>
UnableToAcquireResourcesError
If not able to acquire the requested vessels (in this case, no vessels
will be acquired).
InsufficientUserResourcesError
The user does not have enough vessel credits to acquire the number of
vessels requested.
<Side Effects>
A total of 'vesselcount' previously-unassigned vessels of the specified
vesseltype have been acquired by the user.
<Returns>
A list of the vessels as a result of this function call.
"""
assert_geniuser(geniuser)
assert_positive_int(vesselcount)
assert_str(vesseltype)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Ensure the user is allowed to acquire these resources. This call will
# raise an InsufficientUserResourcesError if the additional vessels would
# cause the user to be over their limit.
maindb.require_user_can_acquire_resources(geniuser, vesselcount)
if vesseltype == 'wan':
acquired_list = vessels.acquire_wan_vessels(lockserver_handle, geniuser, vesselcount)
elif vesseltype == 'lan':
acquired_list = vessels.acquire_lan_vessels(lockserver_handle, geniuser, vesselcount)
elif vesseltype == 'nat':
acquired_list = vessels.acquire_nat_vessels(lockserver_handle, geniuser, vesselcount)
elif vesseltype == 'rand':
acquired_list = vessels.acquire_rand_vessels(lockserver_handle, geniuser, vesselcount)
else:
raise ProgrammerError("Vessel type '%s' is not a valid type" % vesseltype)
return acquired_list
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def register_user(username, password, email, affiliation, pubkey=None):
"""
<Purpose>
Creates a user record with the specified information and sets any additional
information necessary for the user record to be complete.
<Arguments>
username
password
email
affiliation
pubkey
Optional. A string. If not provided, a key pair will be generated for this user.
<Exceptions>
UsernameAlreadyExistsError
If there is already a user with the specified username.
ValidationError
If any of the arguments contains invalid values or if the username is the
same as the password.
<Side Effects>
The user record in the django db is created as well as a user record in the
corresponding user profile table that stores our custom information. A port
will be assigned to the user and the user's donation keys will be set.
<Returns>
GeniUser instance (our GeniUser model, not the django User) corresponding to the
newly registered user.
"""
# If the frontend code that called this function wants to know which field
# is invalid, it must call the validation functions itself before making the
# call to register_user().
# These will raise a ValidationError if any of the fields are invalid.
# These ensure that the data is of the correct type (e.g. a string) as well as
# that we like the content of the variable.
validations.validate_username(username)
validations.validate_password(password)
validations.validate_username_and_password_different(username, password)
validations.validate_email(email)
validations.validate_affiliation(affiliation)
if pubkey is not None:
validations.validate_pubkey_string(pubkey)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, username)
try:
# Ensure there is not already a user with this username.
try:
# Raises a DoesNotExistError if the user doesn't exist.
maindb.get_user(username)
raise UsernameAlreadyExistsError
except DoesNotExistError:
# This is what we wanted: the username isn't already taken.
pass
# Get a key pair from the keygen api if the user didn't supply their own pubkey.
if pubkey is None:
(pubkey, privkey) = keygen.generate_keypair()
else:
privkey = None
# Generate a donor key for this user. This is done through the backend
# as the private key must be stored in the keydb, which the website cannot
# directly access.
keydescription = "donor:" + username
donor_pubkey = backend.generate_key(keydescription)
# Create the user record.
geniuser = maindb.create_user(username, password, email, affiliation, pubkey, privkey, donor_pubkey)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, username)
lockserver.destroy_lockserver_handle(lockserver_handle)
return geniuser
def acquire_vessels(geniuser, vesselcount, vesseltype):
"""
<Purpose>
Acquire unused vessels of a given type for a user. For information on how
the specified vesseltype affects which vessels will be considered
to satisfy the request, see the type-specific functions that are called
by this function.
<Arguments>
geniuser
The GeniUser which will be assigned the vessels.
vesselcount
The number of vessels to acquire (a positive integer).
vesseltype
The type of vessels to acquire. One of either 'lan', 'wan', 'nat', or 'rand'.
<Exceptions>
UnableToAcquireResourcesError
If not able to acquire the requested vessels (in this case, no vessels
will be acquired).
InsufficientUserResourcesError
The user does not have enough vessel credits to acquire the number of
vessels requested.
<Side Effects>
A total of 'vesselcount' previously-unassigned vessels of the specified
vesseltype have been acquired by the user.
<Returns>
A list of the vessels as a result of this function call.
"""
assert_geniuser(geniuser)
assert_positive_int(vesselcount)
assert_str(vesseltype)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, geniuser.username)
try:
# Make sure the user still exists now that we hold the lock. Also makes
# sure that we see any changes made to the user before we obtained the lock.
try:
geniuser = maindb.get_user(geniuser.username)
except DoesNotExistError:
raise InternalError(traceback.format_exc())
# Ensure the user is allowed to acquire these resources. This call will
# raise an InsufficientUserResourcesError if the additional vessels would
# cause the user to be over their limit.
maindb.require_user_can_acquire_resources(geniuser, vesselcount)
if vesseltype == "wan":
acquired_list = vessels.acquire_wan_vessels(lockserver_handle, geniuser, vesselcount)
elif vesseltype == "lan":
acquired_list = vessels.acquire_lan_vessels(lockserver_handle, geniuser, vesselcount)
elif vesseltype == "nat":
acquired_list = vessels.acquire_nat_vessels(lockserver_handle, geniuser, vesselcount)
elif vesseltype == "rand":
acquired_list = vessels.acquire_rand_vessels(lockserver_handle, geniuser, vesselcount)
else:
raise ProgrammerError("Vessel type '%s' is not a valid type" % vesseltype)
return acquired_list
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, geniuser.username)
lockserver.destroy_lockserver_handle(lockserver_handle)
def register_user(username, password, email, affiliation, pubkey=None):
"""
<Purpose>
Creates a user record with the specified information and sets any additional
information necessary for the user record to be complete.
<Arguments>
username
password
email
affiliation
pubkey
Optional. A string. If not provided, a key pair will be generated for this user.
<Exceptions>
UsernameAlreadyExistsError
If there is already a user with the specified username.
ValidationError
If any of the arguments contains invalid values or if the username is the
same as the password.
<Side Effects>
The user record in the django db is created as well as a user record in the
corresponding user profile table that stores our custom information. A port
will be assigned to the user and the user's donation keys will be set.
<Returns>
GeniUser instance (our GeniUser model, not the django User) corresponding to the
newly registered user.
"""
# If the frontend code that called this function wants to know which field
# is invalid, it must call the validation functions itself before making the
# call to register_user().
# These will raise a ValidationError if any of the fields are invalid.
# These ensure that the data is of the correct type (e.g. a string) as well as
# that we like the content of the variable.
validations.validate_username(username)
validations.validate_password(password)
validations.validate_username_and_password_different(username, password)
validations.validate_email(email)
validations.validate_affiliation(affiliation)
if pubkey is not None:
validations.validate_pubkey_string(pubkey)
# Lock the user.
lockserver_handle = lockserver.create_lockserver_handle()
lockserver.lock_user(lockserver_handle, username)
try:
# Ensure there is not already a user with this username.
try:
# Raises a DoesNotExistError if the user doesn't exist.
maindb.get_user(username)
raise UsernameAlreadyExistsError
except DoesNotExistError:
# This is what we wanted: the username isn't already taken.
pass
# Get a key pair from the keygen api if the user didn't supply their own pubkey.
if pubkey is None:
(pubkey, privkey) = keygen.generate_keypair()
else:
privkey = None
# Generate a donor key for this user. This is done through the backend
# as the private key must be stored in the keydb, which the website cannot
# directly access.
keydescription = "donor:" + username
donor_pubkey = backend.generate_key(keydescription)
# Create the user record.
geniuser = maindb.create_user(username, password, email, affiliation, pubkey, privkey, donor_pubkey)
finally:
# Unlock the user.
lockserver.unlock_user(lockserver_handle, username)
lockserver.destroy_lockserver_handle(lockserver_handle)
return geniuser
def _sync_user_keys_of_single_vessel(vessel):
"""
This function is passed by sync_user_keys_of_vessels() as the function
argument to run_parallelized().
"""
# This does seem wasteful of lockserver communication to require four
# round-trips with the lockserver (get handle, lock, unlock, release handle),
# but if we really want to fix that then I think the best thing to do would
# be to allow obtaining a lockhandle and releasing a lockhandle to be done
# in the same calls as lock acquisition and release.
node_id = maindb.get_node_identifier_from_vessel(vessel)
lockserver_handle = lockserver.create_lockserver_handle()
# Lock the node that the vessels is on.
lockserver.lock_node(lockserver_handle, node_id)
try:
# Get a new vessel object from the db in case it was modified in the db
# before the lock was obtained.
vessel = maindb.get_vessel(node_id, vessel.name)
# Now that we have a lock on the node that this vessel is on, find out
# if we should still sync user keys on this vessel (e.g. maybe a node state
# transition script moved the node to a new state and this vessel was
# removed).
needssync, reasonwhynot = maindb.does_vessel_need_user_key_sync(vessel)
if not needssync:
log.info("[_sync_user_keys_of_single_vessel] Vessel " + str(vessel) +
" no longer needs user key sync: " + reasonwhynot)
return
nodeid = maindb.get_node_identifier_from_vessel(vessel)
nodehandle = _get_node_handle_from_nodeid(nodeid)
# The list returned from get_users_with_access_to_vessel includes the key of
# the user who has acquired the vessel along with any other users they have
# given access to.
user_list = maindb.get_users_with_access_to_vessel(vessel)
key_list = []
for user in user_list:
key_list.append(user.user_pubkey)
if len(key_list) == 0:
raise InternalError("InternalError: Empty user key list for vessel " + str(vessel))
try:
log.info("[_sync_user_keys_of_single_vessel] About to ChangeUsers on vessel " + str(vessel))
nodemanager.change_users(nodehandle, vessel.name, key_list)
except NodemanagerCommunicationError:
# We don't pass this exception up. Maybe the node is offline now. At some
# point, it will be marked in the database as offline and won't show up in
# our list of vessels to sync user keys of anymore.
log.info("[_sync_user_keys_of_single_vessel] Failed to sync user keys of vessel " +
str(vessel) + ". " + traceback.format_exc())
return
# We only mark it as sync'd if no exception was raised when trying to perform
# the above nodemanager operations.
maindb.mark_vessel_as_not_needing_user_key_sync(vessel)
log.info("[_sync_user_keys_of_single_vessel] Successfully sync'd user keys of vessel " + str(vessel))
finally:
# Unlock the node.
lockserver.unlock_node(lockserver_handle, node_id)
lockserver.destroy_lockserver_handle(lockserver_handle)
def check_node(node, readonly=True, lockserver_handle=None):
"""
<Purpose>
Check a node for problems. This will try to contact the node and will
compare the information retrieved from the node to the information we
have in our database. It will log and collect the information about
the problems. The problem information can be retrieved program
<Arguments>
node
The Node object of the node to be checked.
readonly
False if the function should mark the node in the database as inactive
or broken (and vessels released) when appropriate, True if it should
never change anything in the database. Default is True.
lockserver_handle
If an existing lockserver handle should be used for lock acquisitions,
it should be provided here. Otherwise, a new lockserver handle will
be used the during of this function call.
Note: no locking is done if readonly is True. That is, if there is
no reason to lock a node, there is no reason to provide a
lockserver_handle.
<Exceptions>
None
<Side Effects>
If readonly is False, the database may be updated appropriately based on
what the function sees. No changes are ever directly made to the nodes
through nodemanager communication regardless of the setting of readonly.
However, other scripts might take action based on database changes (e.g.
released vessel will quickly be cleaned up by the backend daemon).
<Returns>
None
"""
if not readonly:
must_destroy_lockserver_handle = False
if lockserver_handle is None:
must_destroy_lockserver_handle = True
lockserver_handle = lockserver.create_lockserver_handle()
if not readonly:
lockserver.lock_node(lockserver_handle, node.node_identifier)
# Be sure to release the node lock, if we are locking the node.
try:
# Get a fresh node record from the database. It might have changed before
# we obtained the lock.
node = maindb.get_node(node.node_identifier)
# The code beyond this point would be a good candidate for splitting out
# into a few smaller functions for readability.
donation_list = maindb.get_donations_from_node(node)
if len(donation_list) == 0:
_report_node_problem(
node, "The node has no corresponding donation records. " +
"Not marking node broken, though.")
try:
nodeinfo = nodemanager.get_node_info(node.last_known_ip,
node.last_known_port)
except NodemanagerCommunicationError:
_record_node_communication_failure(readonly, node)
_report_node_problem(node, "Can't communicate with node.")
return
try:
nodekey_str = rsa.rsa_publickey_to_string(nodeinfo["nodekey"])
except ValueError:
_mark_node_broken(readonly, node)
_report_node_problem(
node, "Invalid nodekey: " + str(nodeinfo["nodekey"]))
return
# Check that the nodeid matches. If it doesn't, it probably means seattle
# was reinstalled or there is a different system at that address now.
if node.node_identifier != nodekey_str:
_mark_node_inactive(readonly, node)
_report_node_problem(
node, "Wrong node identifier, the node reports: " +
str(nodeinfo["nodekey"]))
# Not much more worth checking in this case.
return
# Check that the database thinks it knows the extra vessel name.
if node.extra_vessel_name == "":
_mark_node_broken(readonly, node)
_report_node_problem(node, "No extra_vessel_name in the database.")
# Not much more worth checking in this case.
return
# Check that a vessel by the name of extra_vessel_name exists on the node.
if node.extra_vessel_name not in nodeinfo["vessels"]:
_mark_node_broken(readonly, node)
_report_node_problem(
node,
"The extra_vessel_name in the database is a vessel name that doesn't exist on the node."
)
# Not much more worth checking in this case.
return
extravesselinfo = nodeinfo["vessels"][node.extra_vessel_name]
vessels_in_db = maindb.get_vessels_on_node(node)
if len(extravesselinfo["userkeys"]) != 1:
_mark_node_broken(readonly, node)
_report_node_problem(
node, "The extra vessel '" + node.extra_vessel_name +
"' doesn't have 1 user key, it has " +
str(len(extravesselinfo["userkeys"])))
else:
# Figure out which state the node is in according to the state key.
recognized_state_name = ""
for statename in statekeys:
if statekeys[statename] == extravesselinfo["userkeys"][0]:
recognized_state_name = statename
if not recognized_state_name:
_mark_node_broken(readonly, node)
_report_node_problem(
node, "The extra vessel '" + node.extra_vessel_name +
"' doesn't have a recognized user/state key")
if len(vessels_in_db) == 0:
if recognized_state_name == "onepercentmanyevents" or recognized_state_name == "twopercent":
# We don't mark it as broken because it may be in transition by a
# transition script away from onepercentmanyevents. If the vessels
# in the db have been deleted first but the state key hasn't been
# changed yet, we might hit this. Also, it's not so bad to have it
# not be marked as broken when it's like this, as it has no vessels
# we know about, anyways, so we're not going to be giving questionable
# resources to users because of it.
_report_node_problem(
node, "The node is in the " + recognized_state_name +
" state " +
"but we don't have any vessels for it in the database."
)
else:
if recognized_state_name != "onepercentmanyevents" and recognized_state_name != "twopercent":
# We don't mark it as broken because it may be in transition by a
# transition script. Also, we may have other states in the future
# besides onepercentmanyevents that have vessels. We don't want
# to make all of those nodes inactive if it's just an issue of
# someone forgot to update this script.
_report_node_problem(
node, "The node is in the '" + recognized_state_name +
"' state but we have vessels for it in the database.")
known_vessel_names = []
for vessel in vessels_in_db:
known_vessel_names.append(vessel.name)
# Look for vessels on the node with our node ownerkey which aren't in our database.
for actualvesselname in nodeinfo["vessels"]:
vessel_ownerkey = nodeinfo["vessels"][actualvesselname]["ownerkey"]
try:
vessel_ownerkey_str = rsa.rsa_publickey_to_string(
vessel_ownerkey)
except ValueError:
# At this point we aren't sure it's our node, but let's assume that if
# there's an invalid key then the node is broken, period.
_mark_node_broken(readonly, node)
_report_node_problem(
node, "Invalid vessel ownerkey: " + str(vessel_ownerkey))
return
if vessel_ownerkey_str == node.owner_pubkey:
if actualvesselname not in known_vessel_names and actualvesselname != node.extra_vessel_name:
_mark_node_broken(readonly, node)
_report_node_problem(
node, "The vessel '" + actualvesselname +
"' exists on the node " +
"with the ownerkey for the node, but it's not in our vessels table."
)
# Do some checking on each vessel we have in our database.
for vessel in vessels_in_db:
# Check that the vessel in our database actually exists on the node.
if vessel.name not in nodeinfo["vessels"]:
_mark_node_broken(readonly, node)
_report_node_problem(
node, "The vessel '" + vessel.name +
"' in our db doesn't exist on the node.")
continue
vesselinfo = nodeinfo["vessels"][vessel.name]
try:
vessel_ownerkey_str = rsa.rsa_publickey_to_string(
vesselinfo["ownerkey"])
except ValueError:
_mark_node_broken(readonly, node)
_report_node_problem(
node, "Invalid vessel ownerkey on a vessel in our db: " +
str(vessel_ownerkey))
return
# Check that the owner key for the vessel is what we have for the node's owner key in our database.
if node.owner_pubkey != vessel_ownerkey_str:
_mark_node_broken(readonly, node)
_report_node_problem(
node, "The vessel '" + vessel.name +
"' doesn't have the ownerkey we use for the node.")
if not vesselinfo["advertise"]:
_mark_node_broken(readonly, node)
_report_node_problem(
node,
"The vessel '" + vessel.name + "' isn't advertising.")
# We're only concerned with non-dirty vessels as the backend daemon
# should be working on cleaning up dirty vessels.
if not vessel.is_dirty:
# Check that the user keys that have access are the ones that should have access.
users_with_access = maindb.get_users_with_access_to_vessel(
vessel)
if len(users_with_access) != len(vesselinfo["userkeys"]):
_release_vessel(readonly, vessel)
_report_node_problem(
node, "The vessel '" + vessel.name + "' reports " +
str(len(vesselinfo["userkeys"])) +
" user keys, but we expected " +
str(len(users_with_access)))
for user in users_with_access:
if rsa.rsa_string_to_publickey(
user.user_pubkey) not in vesselinfo["userkeys"]:
_release_vessel(readonly, vessel)
_report_node_problem(
node, "The vessel '" + vessel.name +
"' doesn't have the userkey for user " +
user.username + ".")
finally:
# We didn't do any locking if this readonly was True.
if not readonly:
# Release the lock
lockserver.unlock_node(lockserver_handle, node.node_identifier)
# Destroy the lockserver handle if we created it ourselves.
if must_destroy_lockserver_handle:
lockserver.destroy_lockserver_handle(lockserver_handle)
def check_node(node, readonly=True, lockserver_handle=None):
"""
<Purpose>
Check a node for problems. This will try to contact the node and will
compare the information retrieved from the node to the information we
have in our database. It will log and collect the information about
the problems. The problem information can be retrieved program
<Arguments>
node
The Node object of the node to be checked.
readonly
False if the function should mark the node in the database as inactive
or broken (and vessels released) when appropriate, True if it should
never change anything in the database. Default is True.
lockserver_handle
If an existing lockserver handle should be used for lock acquisitions,
it should be provided here. Otherwise, a new lockserver handle will
be used the during of this function call.
Note: no locking is done if readonly is True. That is, if there is
no reason to lock a node, there is no reason to provide a
lockserver_handle.
<Exceptions>
None
<Side Effects>
If readonly is False, the database may be updated appropriately based on
what the function sees. No changes are ever directly made to the nodes
through nodemanager communication regardless of the setting of readonly.
However, other scripts might take action based on database changes (e.g.
released vessel will quickly be cleaned up by the backend daemon).
<Returns>
None
"""
if not readonly:
must_destroy_lockserver_handle = False
if lockserver_handle is None:
must_destroy_lockserver_handle = True
lockserver_handle = lockserver.create_lockserver_handle()
if not readonly:
lockserver.lock_node(lockserver_handle, node.node_identifier)
# Be sure to release the node lock, if we are locking the node.
try:
# Get a fresh node record from the database. It might have changed before
# we obtained the lock.
node = maindb.get_node(node.node_identifier)
# The code beyond this point would be a good candidate for splitting out
# into a few smaller functions for readability.
donation_list = maindb.get_donations_from_node(node)
if len(donation_list) == 0:
_report_node_problem(node, "The node has no corresponding donation records. " +
"Not marking node broken, though.")
try:
nodeinfo = nodemanager.get_node_info(node.last_known_ip, node.last_known_port)
except NodemanagerCommunicationError:
_record_node_communication_failure(readonly, node)
_report_node_problem(node, "Can't communicate with node.")
return
try:
nodekey_str = rsa.rsa_publickey_to_string(nodeinfo["nodekey"])
except ValueError:
_mark_node_broken(readonly, node)
_report_node_problem(node, "Invalid nodekey: " + str(nodeinfo["nodekey"]))
return
# Check that the nodeid matches. If it doesn't, it probably means seattle
# was reinstalled or there is a different system at that address now.
if node.node_identifier != nodekey_str:
_mark_node_inactive(readonly, node)
_report_node_problem(node, "Wrong node identifier, the node reports: " + str(nodeinfo["nodekey"]))
# Not much more worth checking in this case.
return
# Check that the database thinks it knows the extra vessel name.
if node.extra_vessel_name == "":
_mark_node_broken(readonly, node)
_report_node_problem(node, "No extra_vessel_name in the database.")
# Not much more worth checking in this case.
return
# Check that a vessel by the name of extra_vessel_name exists on the node.
if node.extra_vessel_name not in nodeinfo["vessels"]:
_mark_node_broken(readonly, node)
_report_node_problem(node, "The extra_vessel_name in the database is a vessel name that doesn't exist on the node.")
# Not much more worth checking in this case.
return
extravesselinfo = nodeinfo["vessels"][node.extra_vessel_name]
vessels_in_db = maindb.get_vessels_on_node(node)
if len(extravesselinfo["userkeys"]) != 1:
_mark_node_broken(readonly, node)
_report_node_problem(node, "The extra vessel '" + node.extra_vessel_name +
"' doesn't have 1 user key, it has " +
str(len(extravesselinfo["userkeys"])))
else:
# Figure out which state the node is in according to the state key.
recognized_state_name = ""
for statename in statekeys:
if statekeys[statename] == extravesselinfo["userkeys"][0]:
recognized_state_name = statename
if not recognized_state_name:
_mark_node_broken(readonly, node)
_report_node_problem(node, "The extra vessel '" + node.extra_vessel_name +
"' doesn't have a recognized user/state key")
if len(vessels_in_db) == 0:
if recognized_state_name == "onepercentmanyevents" or recognized_state_name == "twopercent":
# We don't mark it as broken because it may be in transition by a
# transition script away from onepercentmanyevents. If the vessels
# in the db have been deleted first but the state key hasn't been
# changed yet, we might hit this. Also, it's not so bad to have it
# not be marked as broken when it's like this, as it has no vessels
# we know about, anyways, so we're not going to be giving questionable
# resources to users because of it.
_report_node_problem(node, "The node is in the " + recognized_state_name + " state " +
"but we don't have any vessels for it in the database.")
else:
if recognized_state_name != "onepercentmanyevents" and recognized_state_name != "twopercent":
# We don't mark it as broken because it may be in transition by a
# transition script. Also, we may have other states in the future
# besides onepercentmanyevents that have vessels. We don't want
# to make all of those nodes inactive if it's just an issue of
# someone forgot to update this script.
_report_node_problem(node, "The node is in the '" + recognized_state_name +
"' state but we have vessels for it in the database.")
known_vessel_names = []
for vessel in vessels_in_db:
known_vessel_names.append(vessel.name)
# Look for vessels on the node with our node ownerkey which aren't in our database.
for actualvesselname in nodeinfo["vessels"]:
vessel_ownerkey = nodeinfo["vessels"][actualvesselname]["ownerkey"]
try:
vessel_ownerkey_str = rsa.rsa_publickey_to_string(vessel_ownerkey)
except ValueError:
# At this point we aren't sure it's our node, but let's assume that if
# there's an invalid key then the node is broken, period.
_mark_node_broken(readonly, node)
_report_node_problem(node, "Invalid vessel ownerkey: " + str(vessel_ownerkey))
return
if vessel_ownerkey_str == node.owner_pubkey:
if actualvesselname not in known_vessel_names and actualvesselname != node.extra_vessel_name:
_mark_node_broken(readonly, node)
_report_node_problem(node, "The vessel '" + actualvesselname + "' exists on the node " +
"with the ownerkey for the node, but it's not in our vessels table.")
# Do some checking on each vessel we have in our database.
for vessel in vessels_in_db:
# Check that the vessel in our database actually exists on the node.
if vessel.name not in nodeinfo["vessels"]:
_mark_node_broken(readonly, node)
_report_node_problem(node, "The vessel '" + vessel.name + "' in our db doesn't exist on the node.")
continue
vesselinfo = nodeinfo["vessels"][vessel.name]
try:
vessel_ownerkey_str = rsa.rsa_publickey_to_string(vesselinfo["ownerkey"])
except ValueError:
_mark_node_broken(readonly, node)
_report_node_problem(node, "Invalid vessel ownerkey on a vessel in our db: " + str(vessel_ownerkey))
return
# Check that the owner key for the vessel is what we have for the node's owner key in our database.
if node.owner_pubkey != vessel_ownerkey_str:
_mark_node_broken(readonly, node)
_report_node_problem(node, "The vessel '" + vessel.name + "' doesn't have the ownerkey we use for the node.")
if not vesselinfo["advertise"]:
_mark_node_broken(readonly, node)
_report_node_problem(node, "The vessel '" + vessel.name + "' isn't advertising.")
# We're only concerned with non-dirty vessels as the backend daemon
# should be working on cleaning up dirty vessels.
if not vessel.is_dirty:
# Check that the user keys that have access are the ones that should have access.
users_with_access = maindb.get_users_with_access_to_vessel(vessel)
if len(users_with_access) != len(vesselinfo["userkeys"]):
_release_vessel(readonly, vessel)
_report_node_problem(node, "The vessel '" + vessel.name + "' reports " +
str(len(vesselinfo["userkeys"])) + " user keys, but we expected " + str(len(users_with_access)))
for user in users_with_access:
if rsa.rsa_string_to_publickey(user.user_pubkey) not in vesselinfo["userkeys"]:
_release_vessel(readonly, vessel)
_report_node_problem(node, "The vessel '" + vessel.name + "' doesn't have the userkey for user " + user.username + ".")
finally:
# We didn't do any locking if this readonly was True.
if not readonly:
# Release the lock
lockserver.unlock_node(lockserver_handle, node.node_identifier)
# Destroy the lockserver handle if we created it ourselves.
if must_destroy_lockserver_handle:
lockserver.destroy_lockserver_handle(lockserver_handle)
def main():
"""
This will run an infinite loop of checks over all of the active nodes in the
database.
"""
lockserver_handle = lockserver.create_lockserver_handle()
django.setup()
# Always try to release the lockserver handle, though it's probably not
# very useful in this case.
try:
while True:
# Catch unexpected exceptions to log/send mail.
try:
# We shouldn't be running in production with settings.DEBUG = True.
# Just in case, though, tell django to reset its list of saved queries
# each time through the loop.
if settings.DEBUG:
django.db.reset_queries()
# Note: although we include broken but active nodes, we don't change
# the status of broken nodes to be not broken yet if we don't detect
# any problems. For now, most of the reason we include broken nodes
# is so that we can tell which broken nodes are still online. This is
# because it's not as big of a concern to have a broken node that is
# quickly offline (e.g. broken nodes in development), but having one be
# online for an extended period of time is a stronger signal of
# potentially unknown bugs in the clearinghouse or seattle code.
active_nodes = maindb.get_active_nodes_include_broken()
log.info("Starting check of " + str(len(active_nodes)) + " active nodes.")
checked_node_count = 0
for node in active_nodes:
checked_node_count += 1
log.info("Checking node " + str(checked_node_count) + ": " + str(node))
nodestatus.check_node(node, readonly=READONLY, lockserver_handle=lockserver_handle)
# Print summary info.
log.info("Nodes checked: " + str(checked_node_count))
nodes_with_problems = nodestatus.get_node_problem_info()
nodes_with_problems_count = len(nodes_with_problems.keys())
log.info("Nodes without problems: " + str(checked_node_count - nodes_with_problems_count))
log.info("Nodes with problems: " + str(nodes_with_problems_count))
# Print information about the database changes made.
log.info("Number of database actions taken:")
actionstaken = nodestatus.get_actions_taken()
for actionname in actionstaken:
log.info("\t" + actionname + ": " + str(len(actionstaken[actionname])) +
" " + str(actionstaken[actionname]))
nodestatus.reset_collected_data()
log.info("Sleeping for " + str(SLEEP_SECONDS_BETWEEN_RUNS) + " seconds.")
time.sleep(SLEEP_SECONDS_BETWEEN_RUNS)
except KeyboardInterrupt:
raise
except:
message = "Unexpected exception in check_active_db_nodes.py: " + traceback.format_exc()
log.critical(message)
# Send an email to the addresses listed in settings.ADMINS
if not settings.DEBUG:
subject = "Critical SeattleGeni check_active_db_nodes.py error"
django.core.mail.mail_admins(subject, message)
# Sleep for ten minutes to make sure we don't flood the admins with error
# report emails.
time.sleep(600)
finally:
lockserver.destroy_lockserver_handle(lockserver_handle)
def _sync_user_keys_of_single_vessel(vessel):
"""
This function is passed by sync_user_keys_of_vessels() as the function
argument to run_parallelized().
"""
# This does seem wasteful of lockserver communication to require four
# round-trips with the lockserver (get handle, lock, unlock, release handle),
# but if we really want to fix that then I think the best thing to do would
# be to allow obtaining a lockhandle and releasing a lockhandle to be done
# in the same calls as lock acquisition and release.
node_id = maindb.get_node_identifier_from_vessel(vessel)
lockserver_handle = lockserver.create_lockserver_handle()
# Lock the node that the vessels is on.
lockserver.lock_node(lockserver_handle, node_id)
try:
# Get a new vessel object from the db in case it was modified in the db
# before the lock was obtained.
vessel = maindb.get_vessel(node_id, vessel.name)
# Now that we have a lock on the node that this vessel is on, find out
# if we should still sync user keys on this vessel (e.g. maybe a node state
# transition script moved the node to a new state and this vessel was
# removed).
needssync, reasonwhynot = maindb.does_vessel_need_user_key_sync(vessel)
if not needssync:
log.info("[_sync_user_keys_of_single_vessel] Vessel " +
str(vessel) + " no longer needs user key sync: " +
reasonwhynot)
return
nodeid = maindb.get_node_identifier_from_vessel(vessel)
nodehandle = _get_node_handle_from_nodeid(nodeid)
# The list returned from get_users_with_access_to_vessel includes the key of
# the user who has acquired the vessel along with any other users they have
# given access to.
user_list = maindb.get_users_with_access_to_vessel(vessel)
key_list = []
for user in user_list:
key_list.append(user.user_pubkey)
if len(key_list) == 0:
raise InternalError(
"InternalError: Empty user key list for vessel " + str(vessel))
try:
log.info(
"[_sync_user_keys_of_single_vessel] About to ChangeUsers on vessel "
+ str(vessel))
nodemanager.change_users(nodehandle, vessel.name, key_list)
except NodemanagerCommunicationError:
# We don't pass this exception up. Maybe the node is offline now. At some
# point, it will be marked in the database as offline and won't show up in
# our list of vessels to sync user keys of anymore.
log.info(
"[_sync_user_keys_of_single_vessel] Failed to sync user keys of vessel "
+ str(vessel) + ". " + traceback.format_exc())
return
# We only mark it as sync'd if no exception was raised when trying to perform
# the above nodemanager operations.
maindb.mark_vessel_as_not_needing_user_key_sync(vessel)
log.info(
"[_sync_user_keys_of_single_vessel] Successfully sync'd user keys of vessel "
+ str(vessel))
finally:
# Unlock the node.
lockserver.unlock_node(lockserver_handle, node_id)
lockserver.destroy_lockserver_handle(lockserver_handle)
