def _node_type_parser(cursor, invert, parameters):
'''
<Purpose>
Vessel-Level Rule. Ensures that all vessels in the group are of the
specified type.
This is a rule callback. See the Usage section of the module
docstring for more information.
<Arguments>
'node_type':
The node type to filter.
This should be a value in selexorhelper.VALID_NODETYPES.
'''
good_handles = set()
node_type = parameters['node_type']
if not invert:
query = (
"SELECT node_id, vessel_name FROM vessels WHERE node_id IN "
"(SELECT node_id FROM nodes WHERE node_type='"+node_type+"')"
)
else:
query = (
"SELECT node_id, vessel_name FROM vessels WHERE node_id IN "
"(SELECT node_id FROM nodes WHERE node_type!='"+node_type+"')"
)
selexorhelper.autoretry_mysql_command(cursor, query)
return cursor.fetchall()
def _different_location_type_parser(cursor, invert, parameters, acquired_vessels):
'''
<Purpose>
Group-Level Rule. Performs location type-based parsing for handles.
This is a rule callback. See the Usage section of the module docstring for more
information.
<Arguments>
'location_count':
The maximum number of unique locations to have. While this number is not
reached, each vessel in the group will be from a unique location.
Expected Range: [1, Infinity)
'location_type':
The kind of location that is differentiated. 'cities' or 'countries'.
'''
locations= set()
# Compile list of locations
for vesseldict in acquired_vessels:
nodekey = vesseldict['handle'].split(':')[0]
query = """
SELECT """+parameters['location_type']+""" FROM
(SELECT ip_addr FROM nodes WHERE node_key='"""+nodekey+"""') AS node_row
LEFT JOIN location USING (ip_addr)"""
selexorhelper.autoretry_mysql_command(cursor, query)
locations.add(cursor.fetchone()[0])
query = parameters['location_type'] + " "
# If we have enough locations, we want vessels to only be from the
# already acquired locations.
# If we don't have enough locations, we want vesels to not be from
# the already acquired locations.
# Truth table:
# | Invert | Dont invert
# Not enough locations | IN | NOT IN
# Enough Locations | NOT IN | IN
if ((not invert and len(locations) < parameters['location_count']) or
(invert and len(locations) == parameters['location_count'])):
query += 'NOT '
query += 'IN ("'+'", "'.join(locations)+'")'
query = """
SELECT node_id, vessel_name FROM
(SELECT node_id FROM
(SELECT ip_addr FROM location WHERE city NOT IN ("""+query+""")
) as matching_locations LEFT JOIN nodes using (ip_addr)
) as matching_nodes LEFT JOIN vessels USING (node_id)"""
logger.debug(query)
cursor.execute(query)
return cursor.fetchall()
def update_userkeys_table(cursor, node_id, vessel_dict):
query = 'SELECT vessel_name FROM userkeys WHERE node_id='+str(node_id)
num_vessels = selexorhelper.autoretry_mysql_command(cursor, query)
vessel_rows = cursor.fetchall()
if vessel_rows:
# Remove vessels that were lost, if any
vessels_to_remove = []
for [vessel_name] in vessel_rows:
if not vessel_name in vessel_dict:
# We pass this to MySQL later; MySQL expects strings to be wrapped in quotes.
vessels_to_remove += ['"'+vessel_name+'"']
if vessels_to_remove:
query = 'DELETE FROM userkeys WHERE node_id='+str(node_id)+' AND vessel_name in ('+', '.join(vessels_to_remove)+')'
selexorhelper.autoretry_mysql_command(cursor, query)
# Update the userkeys
for vessel_name in vessel_dict:
# v2 can never be used... No sense in tracking it in the userkey database.
if vessel_name == 'v2':
continue
query = 'SELECT userkey FROM userkeys WHERE node_id='+str(node_id)+" AND vessel_name='"+vessel_name+"'"
selexorhelper.autoretry_mysql_command(cursor, query)
userkey_rows = cursor.fetchall()
userkeys_to_remove = []
# Remove the userkeys that were lost, if any
for [userkey] in userkey_rows:
if not rsa_string_to_publickey(userkey) in vessel_dict[vessel_name]['userkeys']:
# We pass this to MySQL later; MySQL expects strings to be wrapped in quotes.
userkeys_to_remove += ['"'+userkey+'"']
if userkeys_to_remove:
query = 'DELETE FROM userkeys WHERE (node_id, vessel_name)=('+str(node_id)+', "'+vessel_name+'") AND userkey in ('+', '.join(userkeys_to_remove)+')'
selexorhelper.autoretry_mysql_command(cursor, query)
# Vessels may not have userkeys on them. Don't bother adding them in that case.
if vessel_dict[vessel_name]['userkeys']:
# IGNORE keyword is to tell MySQL to ignore userkeys that already exist.
query = 'INSERT IGNORE INTO userkeys (node_id, vessel_name, userkey) VALUES'
for userkey in vessel_dict[vessel_name]['userkeys']:
query += " ('"+str(node_id)+"', '"+vessel_name+"', '"+rsa_publickey_to_string(userkey)+"'),"
# Get rid of the trailing comma after the last tuple
query = query.strip(',')
selexorhelper.autoretry_mysql_command(cursor, query)
def update_ports_table(cursor, node_id, ports):
query = 'SELECT vessel_name FROM vesselports WHERE node_id='+str(node_id)
num_vessels = selexorhelper.autoretry_mysql_command(cursor, query)
vessel_rows = cursor.fetchall()
if vessel_rows:
# Remove vessels that were lost, if any
vessels_to_remove = []
for [vessel_name] in vessel_rows:
if not vessel_name in ports:
# We pass this to MySQL later; MySQL expects strings to be wrapped in quotes.
vessels_to_remove += ['"'+vessel_name+'"']
if vessels_to_remove:
query = 'DELETE FROM vesselports WHERE node_id='+str(node_id)+' AND vessel_name in ('+', '.join(vessels_to_remove)+')'
selexorhelper.autoretry_mysql_command(cursor, query)
# Update the ports
for vessel_name in ports:
# v2 can never be used... No sense in tracking it in the userkey database.
if vessel_name == 'v2':
continue
query = 'SELECT port FROM vesselports WHERE node_id='+str(node_id)+" AND vessel_name='"+vessel_name+"'"
selexorhelper.autoretry_mysql_command(cursor, query)
ports_rows = cursor.fetchall()
ports_to_remove = []
# Remove the userkeys that were lost, if any
for [port] in ports_to_remove:
if not port in ports[vessel_name]:
# We pass this to MySQL later; MySQL expects strings to be wrapped in quotes.
ports_to_remove += ['"'+port+'"']
if ports_to_remove:
query = 'DELETE FROM vesselports WHERE (node_id, vessel_name)=('+str(node_id)+', "'+vessel_name+'") AND port in ('+', '.join(ports_to_remove)+')'
selexorhelper.autoretry_mysql_command(cursor, query)
# Vessels may not have ports on them. Don't bother adding them in that case.
if ports[vessel_name]:
# IGNORE keyword is to tell MySQL to ignore userkeys that already exist.
query = 'INSERT IGNORE INTO vesselports (node_id, vessel_name, port) VALUES'
for port in ports[vessel_name]:
query += " ('"+str(node_id)+"', '"+vessel_name+"', "+str(port)+"),"
# Get rid of the trailing comma after the last tuple
query = query.strip(',')
selexorhelper.autoretry_mysql_command(cursor, query)
def update_location_table(cursor, ip_addr, geoinfo):
# City is not always defined
if 'city' in geoinfo:
city = geoinfo['city']
else:
city = ""
# == Update Userkeys ==
country_code = geoinfo['country_code']
longitude = str(geoinfo['longitude'])
latitude = str(geoinfo['latitude'])
query = 'INSERT INTO location (ip_addr, city, country_code, longitude, latitude) VALUES '
# Specifies the location tuple
query += "('%s', '%s', '%s', %s, %s) " % (ip_addr, city, country_code, longitude, latitude)
query += "ON DUPLICATE KEY UPDATE "
# Specifies the location tuple for the update clause
query += "city='%s', country_code='%s', longitude=%s, latitude=%s" % (city, country_code, longitude, latitude)
selexorhelper.autoretry_mysql_command(cursor, query)
def _port_parser(cursor, invert, parameters):
'''
<Purpose>
Vessel-Level Rule. Ensures that all vessels in the group have the specified port number.
This is a rule callback. See the Usage section of the module docstring for more
information.
<Arguments>
'port': The port number that all vessels in the set must have available.
'''
good_handles = set()
port = parameters['port']
if not invert:
query = "SELECT node_id, vessel_name FROM vesselports WHERE port="+str(port)
else:
query = "SELECT node_id, vessel_name FROM vesselports WHERE port !="+str(port)
logger.debug(query)
selexorhelper.autoretry_mysql_command(cursor, query)
return cursor.fetchall()
def update_vessels_table(cursor, node_id, vessel_dict):
query = 'SELECT vessel_name FROM vessels WHERE node_id='+str(node_id)
num_vessels = selexorhelper.autoretry_mysql_command(cursor, query)
vessel_rows = cursor.fetchall()
if vessel_rows:
# Remove vessels that were lost, if any
vessels_to_remove = []
for [vessel_name] in vessel_rows:
if not vessel_name in vessel_dict:
# We pass this to MySQL later; MySQL expects strings to be wrapped in quotes.
vessels_to_remove += ['"'+vessel_name+'"']
if vessels_to_remove:
logger.info('\n'.join([
"Node #"+str(node_id),
"Lost vessels: "+str(vessels_to_remove),
]))
query = 'DELETE FROM vessels WHERE node_id='+str(node_id)+' AND vessel_name in ('+', '.join(vessels_to_remove)+')'
selexorhelper.autoretry_mysql_command(cursor, query)
# Update the list of vessels
logger.info('\n'.join([
"Node #"+str(node_id),
"Current vessels: "+str(vessel_dict.keys()),
"Number of Vessels in database (excluding v2): " + str(num_vessels)
]))
if vessel_dict:
# IGNORE keyword is to tell MySQL to ignore vessels that already exist.
query = 'INSERT IGNORE INTO vessels (node_id, vessel_name, acquirable) VALUES'
for vessel_name in vessel_dict:
# v2 can never be used... No sense in tracking it in the vessel database.
if vessel_name == 'v2':
continue
acquirable = vessel_dict[vessel_name]['acquirable']
query += " ('"+str(node_id)+"', '"+vessel_name+"', "+str(acquirable)+"),"
# Get rid of the trailing comma after the last tuple
query = query.strip(',')
selexorhelper.autoretry_mysql_command(cursor, query)
def resolve_node(self, identity, client, node, db, cursor):
# We should never run into these...
if node['status'] == STATUS_RESOLVED:
logger.error(str(identity) + ": Group already resolved: " + str(node))
return node
elif node['status'] == STATUS_FAILED:
logger.error(str(identity) + ": Exceeded pass limit: " + str(node))
return node
logger.info(str(identity) + ": Group " + node['id'] + " on Pass " + str(node['pass']))
vessels_to_acquire = []
remaining = node['allocate'] - len(node['acquired'])
selexorhelper.autoretry_mysql_command(cursor, "SELECT node_id, vessel_name FROM vessels WHERE acquirable")
all_vessels = cursor.fetchall()
# Get vessels that match the vessel rules
handles_vesselrulematch = selexorruleparser.apply_vessel_rules(node['rules'], cursor, all_vessels)
logger.info(str(identity) + ": Vessel-level matches: " + str(len(handles_vesselrulematch)))
# The number of times we tried to resolve this group in the current attempt
in_group_retry_count = 0
MAX_IN_GROUP_RETRIES = 3
candidate_vessels = []
while len(candidate_vessels) < remaining and \
in_group_retry_count < MAX_IN_GROUP_RETRIES:
if not self._running:
# Stop if we receive a quit message
break
if node['pass'] >= MAX_PASSES_PER_NODE:
raise selexorexceptions.SelexorInternalError("Performing more passes than max pass!")
handles_grouprulematch = selexorruleparser.apply_group_rules(
cursor = cursor,
acquired_vessels = candidate_vessels,
rules = node['rules'],
vesselset = handles_vesselrulematch)
# Pick any vessel.
vessellist = list(handles_grouprulematch)
if vessellist:
logger.info(str(identity) + ": Candidates for next vessel: " + str(len(vessellist)))
# If we run out of handles, we simply get another random one, instead of
# programming a special case.
node_id, vesselname = random.choice(vessellist)
vessellist.remove((node_id, vesselname))
selexorhelper.autoretry_mysql_command(cursor, 'SELECT node_key FROM nodes WHERE node_id='+str(node_id))
nodekey = cursor.fetchone()[0]
handle = nodekey + ':' + vesselname
logger.info(str(identity)+":\n"+"Considering: "+str(handle))
# node_id and vessel_name are used extensively by rule parsers
# We should include them here to prevent each rule from looking the up
vessel_dict = {
'handle': handle,
'node_id': node_id,
'node_key': nodekey,
'vessel_name': vesselname,
}
candidate_vessels.append(vessel_dict)
# We ran out of vessels to check
else:
logger.info(str(identity) + ": Can't find any suitable vessels!")
in_group_retry_count += 1
# Have we exceeded the maximum in-group retry count?
if in_group_retry_count >= MAX_IN_GROUP_RETRIES:
break
# We retry if there could be another combination that MIGHT satisfy
# the group rules. If there are no group rules, there is no point
# to retry.
if not selexorruleparser.has_group_rules(node['rules']):
logger.info(str(identity) + ": There are no group rules applied; no point in retrying.")
break
if candidate_vessels:
# Get the vessel that causes the largest drop in the
# size of the available vessel pool
worst_vessel = selexorruleparser.get_worst_vessel(
candidate_vessels,
handles_grouprulematch,
cursor,
node['rules'])
# Release the worst vessel so that we can try to get a better one
# in the next iteration
logger.info(str(identity) + ": Releasing: " + str(worst_vessel))
candidate_vessels.remove(worst_vessel)
client.release_resources([worst_vessel['handle']])
# We may get vessels that are unusable (i.e. extra vessels containing
# leftover resources). If so, drop them and try again
while candidate_vessels:
vessels_to_acquire = []
for vesseldict in candidate_vessels:
vessels_to_acquire.append(vesseldict['handle'])
try:
acquired_vesseldicts = client.acquire_specific_vessels(vessels_to_acquire)
logger.info(str(identity)+": Requested "+str(len(vessels_to_acquire))+" vessels, acquired "+str(len(acquired_vesseldicts))+":\n"+'\n'.join(i['handle'][-10:] +':'+ i['vessel_id'] for i in acquired_vesseldicts))
# We must be careful to count only the vessels that we have
# actually acquired, as some vessels may not have been given to
# the user due to the database having slightly outdated
# information.
acquired_vesseldicts = _get_acquired_vesseldicts(
clearinghouse_vesseldicts=acquired_vesseldicts,
selexor_vesseldicts=candidate_vessels)
node['acquired'] += acquired_vesseldicts
break
except seattleclearinghouse_xmlrpc.NotEnoughCreditsError, e:
logger.error(str(identity) + ": Not enough vessel credits")
raise
except seattleclearinghouse_xmlrpc.InvalidRequestError, e:
error_string = str(e)
# This may be an extra vessel.
if 'There is no vessel with the node identifier' in error_string:
logger.error(str(identity) + ": " + str(e))
extra_vessels = []
for vessel in candidate_vessels:
if (vessel['node_key'] in error_string and
vessel['vessel_name'] in error_string):
extra_vessels.append(vessel)
for vessel in extra_vessels:
candidate_vessels.remove(vessel)
logger.info("Removing: ..." + vessel['node_key'][-10:] + ':' + vessel['vessel_name'])
# Store into the db so that future lookups do not need to
# spend time acquiring the vessel to discover that it is not
# acquirable, as there are no definitive ways of determining
# if a vessel is non-acquirable, aside from the management
# vessel (v2)
update_command = ("UPDATE vessels SET acquirable=false \
WHERE (node_id, vessel_name) IN (" +
", ".join( "(%s, '%s')" % (handle['node_id'], handle['vessel_name'])
for handle in extra_vessels
) +
")")
selexorhelper.autoretry_mysql_command(cursor, update_command)
db.commit()
else:
logger.error(str(identity) + ": " + str(e))
raise
def commit_data_to_database(db, cursor, node_ip, node_port, node_dict, ports, geoinfo):
# Just in case we attempted to make any changes in a previous run and failed
db.rollback()
# == Update Nodes Table ==
nodekeystr = rsa_publickey_to_string(node_dict['nodekey'])
new_node = selexorhelper.autoretry_mysql_command(cursor, "SELECT node_id FROM nodes WHERE node_key='"+nodekeystr+"'") == 0L
if new_node:
node_type = selexorhelper.get_node_type(node_ip)
# Node isn't recognized, add it to the db
cmd = ("INSERT INTO nodes " +
"(node_key, ip_addr, node_port, node_type, last_ip_change, last_seen) " +
"VALUES ('%s', '%s', %i, '%s', NOW(), NOW())" % (nodekeystr, node_ip, node_port, node_type)
)
selexorhelper.autoretry_mysql_command(cursor, cmd)
# Now retrieve the internal node_id.
selexorhelper.autoretry_mysql_command(cursor, "SELECT node_id FROM nodes WHERE node_key='"+nodekeystr+"'")
node_id = cursor.fetchone()[0]
logger.info('\n'.join([
"New node found: #" + str(node_id),
"Nodekey:",
nodekeystr
]))
else:
# Handle already exists
node_id = cursor.fetchone()[0]
logger.info('\n'.join([
"Updating node: #" + str(node_id),
"Nodekey:",
nodekeystr
]))
# Did the IP address change? If so mark that down.
selexorhelper.autoretry_mysql_command(cursor, "SELECT (ip_addr) FROM nodes WHERE node_key='"+nodekeystr+"'")
old_node_ip = cursor.fetchone()[0]
node_ip_changed = node_ip != old_node_ip
if node_ip_changed:
cmd = (
"UPDATE nodes SET ip_addr='%s', last_ip_change=NOW() WHERE node_id=%i" %
(node_ip, node_id)
)
selexorhelper.autoretry_mysql_command(cursor, cmd)
# Update the node type if necessary
if node_ip_changed or settings.force_refresh_node_type:
node_type = selexorhelper.get_node_type(node_ip)
cmd = "UPDATE nodes SET node_type='%s' WHERE node_id=%i" % (node_type, node_id)
selexorhelper.autoretry_mysql_command(cursor, cmd)
# Update the last time we saw the node.
cmd = "UPDATE nodes SET last_seen=NOW() WHERE node_id=%i" % (node_id)
selexorhelper.autoretry_mysql_command(cursor, cmd)
# == Update Vessels Table ==
update_vessels_table(cursor, node_id, node_dict['vessels'])
# == Update Userkeys ==
update_userkeys_table(cursor, node_id, node_dict['vessels'])
# == Update Ports ==
update_ports_table(cursor, node_id, ports)
# == Update Location Table ==
if not geoinfo is None and geoinfo:
update_location_table(cursor, node_ip, geoinfo)
db.commit()
def contact_vessels_and_update_database(nodes_to_check):
'''
Of all nodes that need checking, take one node.
Obtain its:
IP address
name of each vessel on it
Location
For each vessel, obtain:
vessel name
ports available
Finally, update the information about these nodes.
'''
db, cursor = selexorhelper.connect_to_db()
while nodes_to_check:
nodelocation = nodes_to_check.pop()
nodeinfo = selexorhelper.get_node_ip_port_from_nodelocation(nodelocation)
# We can't use NAT addresses, nor ipv6
if not selexorhelper.is_ipv4_address(nodeinfo['id']):
# if nodeinfo['id'].startswith('NAT'):
# self._nat_nodes.append(nodeinfo['id'])
continue
# Used to communicate with the node
node_nmhandle = None
try:
node_nmhandle = nmclient_createhandle(nodeinfo['id'], nodeinfo['port'])
node_dict = nmclient_getvesseldict(node_nmhandle)
ports = {}
for vesselname in node_dict['vessels']:
resources_string = nmclient_rawsay(node_nmhandle, "GetVesselResources", vesselname)
ports[vesselname] = selexorhelper.get_ports_from_resource_string(resources_string)
node_dict['vessels'][vesselname]['acquirable'] = \
selexorhelper.is_resource_acquirable(resources_string)
# Retrieve the geographic information
try:
# We need to some initial value so that it is not undefined when we check it later.
geoinfo = None
# Only retrieve the geographic information if we don't have it already
# The geoip server's data doesn't change, so we don't need to constantly update it.
geoinfo_exists = selexorhelper.autoretry_mysql_command(cursor, "SELECT ip_addr FROM location WHERE ip_addr='"+nodeinfo['id']+"'") == 1L
if not geoinfo_exists:
logger.info("Location data not in database, looking up on geoip: "+nodelocation)
geoinfo = geoip_record_by_addr(nodeinfo['id'])
except Exception, e:
if not "Unable to contact the geoip server" in str(e):
raise
# The geoip lookup sometimes returns None.
if geoinfo is None:
geoinfo = {}
format_geoinfo(geoinfo)
commit_data_to_database(db, cursor, nodeinfo['id'], nodeinfo['port'], node_dict, ports, geoinfo)
except NMClientException, e:
if not node_nmhandle:
nmclient_destroyhandle(node_nmhandle)
# self._bad_node_locations.append(nodelocation)
errstr = str(e)
if ("timed out" in errstr or
'No connection could be made because the target machine actively refused it' in errstr or
"Connection refused" in errstr or
"Socket closed" in errstr):
continue
logger.error("Unknown error contacting " + nodelocation + traceback.format_exc())