def _node_type_parser(cursor, invert, parameters): ''' <Purpose> Vessel-Level Rule. Ensures that all vessels in the group are of the specified type. This is a rule callback. See the Usage section of the module docstring for more information. <Arguments> 'node_type': The node type to filter. This should be a value in selexorhelper.VALID_NODETYPES. ''' good_handles = set() node_type = parameters['node_type'] if not invert: query = ( "SELECT node_id, vessel_name FROM vessels WHERE node_id IN " "(SELECT node_id FROM nodes WHERE node_type='"+node_type+"')" ) else: query = ( "SELECT node_id, vessel_name FROM vessels WHERE node_id IN " "(SELECT node_id FROM nodes WHERE node_type!='"+node_type+"')" ) selexorhelper.autoretry_mysql_command(cursor, query) return cursor.fetchall()
def _different_location_type_parser(cursor, invert, parameters, acquired_vessels): ''' <Purpose> Group-Level Rule. Performs location type-based parsing for handles. This is a rule callback. See the Usage section of the module docstring for more information. <Arguments> 'location_count': The maximum number of unique locations to have. While this number is not reached, each vessel in the group will be from a unique location. Expected Range: [1, Infinity) 'location_type': The kind of location that is differentiated. 'cities' or 'countries'. ''' locations= set() # Compile list of locations for vesseldict in acquired_vessels: nodekey = vesseldict['handle'].split(':')[0] query = """ SELECT """+parameters['location_type']+""" FROM (SELECT ip_addr FROM nodes WHERE node_key='"""+nodekey+"""') AS node_row LEFT JOIN location USING (ip_addr)""" selexorhelper.autoretry_mysql_command(cursor, query) locations.add(cursor.fetchone()[0]) query = parameters['location_type'] + " " # If we have enough locations, we want vessels to only be from the # already acquired locations. # If we don't have enough locations, we want vesels to not be from # the already acquired locations. # Truth table: # | Invert | Dont invert # Not enough locations | IN | NOT IN # Enough Locations | NOT IN | IN if ((not invert and len(locations) < parameters['location_count']) or (invert and len(locations) == parameters['location_count'])): query += 'NOT ' query += 'IN ("'+'", "'.join(locations)+'")' query = """ SELECT node_id, vessel_name FROM (SELECT node_id FROM (SELECT ip_addr FROM location WHERE city NOT IN ("""+query+""") ) as matching_locations LEFT JOIN nodes using (ip_addr) ) as matching_nodes LEFT JOIN vessels USING (node_id)""" logger.debug(query) cursor.execute(query) return cursor.fetchall()
def update_userkeys_table(cursor, node_id, vessel_dict): query = 'SELECT vessel_name FROM userkeys WHERE node_id='+str(node_id) num_vessels = selexorhelper.autoretry_mysql_command(cursor, query) vessel_rows = cursor.fetchall() if vessel_rows: # Remove vessels that were lost, if any vessels_to_remove = [] for [vessel_name] in vessel_rows: if not vessel_name in vessel_dict: # We pass this to MySQL later; MySQL expects strings to be wrapped in quotes. vessels_to_remove += ['"'+vessel_name+'"'] if vessels_to_remove: query = 'DELETE FROM userkeys WHERE node_id='+str(node_id)+' AND vessel_name in ('+', '.join(vessels_to_remove)+')' selexorhelper.autoretry_mysql_command(cursor, query) # Update the userkeys for vessel_name in vessel_dict: # v2 can never be used... No sense in tracking it in the userkey database. if vessel_name == 'v2': continue query = 'SELECT userkey FROM userkeys WHERE node_id='+str(node_id)+" AND vessel_name='"+vessel_name+"'" selexorhelper.autoretry_mysql_command(cursor, query) userkey_rows = cursor.fetchall() userkeys_to_remove = [] # Remove the userkeys that were lost, if any for [userkey] in userkey_rows: if not rsa_string_to_publickey(userkey) in vessel_dict[vessel_name]['userkeys']: # We pass this to MySQL later; MySQL expects strings to be wrapped in quotes. userkeys_to_remove += ['"'+userkey+'"'] if userkeys_to_remove: query = 'DELETE FROM userkeys WHERE (node_id, vessel_name)=('+str(node_id)+', "'+vessel_name+'") AND userkey in ('+', '.join(userkeys_to_remove)+')' selexorhelper.autoretry_mysql_command(cursor, query) # Vessels may not have userkeys on them. Don't bother adding them in that case. if vessel_dict[vessel_name]['userkeys']: # IGNORE keyword is to tell MySQL to ignore userkeys that already exist. query = 'INSERT IGNORE INTO userkeys (node_id, vessel_name, userkey) VALUES' for userkey in vessel_dict[vessel_name]['userkeys']: query += " ('"+str(node_id)+"', '"+vessel_name+"', '"+rsa_publickey_to_string(userkey)+"')," # Get rid of the trailing comma after the last tuple query = query.strip(',') selexorhelper.autoretry_mysql_command(cursor, query)
def update_ports_table(cursor, node_id, ports): query = 'SELECT vessel_name FROM vesselports WHERE node_id='+str(node_id) num_vessels = selexorhelper.autoretry_mysql_command(cursor, query) vessel_rows = cursor.fetchall() if vessel_rows: # Remove vessels that were lost, if any vessels_to_remove = [] for [vessel_name] in vessel_rows: if not vessel_name in ports: # We pass this to MySQL later; MySQL expects strings to be wrapped in quotes. vessels_to_remove += ['"'+vessel_name+'"'] if vessels_to_remove: query = 'DELETE FROM vesselports WHERE node_id='+str(node_id)+' AND vessel_name in ('+', '.join(vessels_to_remove)+')' selexorhelper.autoretry_mysql_command(cursor, query) # Update the ports for vessel_name in ports: # v2 can never be used... No sense in tracking it in the userkey database. if vessel_name == 'v2': continue query = 'SELECT port FROM vesselports WHERE node_id='+str(node_id)+" AND vessel_name='"+vessel_name+"'" selexorhelper.autoretry_mysql_command(cursor, query) ports_rows = cursor.fetchall() ports_to_remove = [] # Remove the userkeys that were lost, if any for [port] in ports_to_remove: if not port in ports[vessel_name]: # We pass this to MySQL later; MySQL expects strings to be wrapped in quotes. ports_to_remove += ['"'+port+'"'] if ports_to_remove: query = 'DELETE FROM vesselports WHERE (node_id, vessel_name)=('+str(node_id)+', "'+vessel_name+'") AND port in ('+', '.join(ports_to_remove)+')' selexorhelper.autoretry_mysql_command(cursor, query) # Vessels may not have ports on them. Don't bother adding them in that case. if ports[vessel_name]: # IGNORE keyword is to tell MySQL to ignore userkeys that already exist. query = 'INSERT IGNORE INTO vesselports (node_id, vessel_name, port) VALUES' for port in ports[vessel_name]: query += " ('"+str(node_id)+"', '"+vessel_name+"', "+str(port)+")," # Get rid of the trailing comma after the last tuple query = query.strip(',') selexorhelper.autoretry_mysql_command(cursor, query)
def update_location_table(cursor, ip_addr, geoinfo): # City is not always defined if 'city' in geoinfo: city = geoinfo['city'] else: city = "" # == Update Userkeys == country_code = geoinfo['country_code'] longitude = str(geoinfo['longitude']) latitude = str(geoinfo['latitude']) query = 'INSERT INTO location (ip_addr, city, country_code, longitude, latitude) VALUES ' # Specifies the location tuple query += "('%s', '%s', '%s', %s, %s) " % (ip_addr, city, country_code, longitude, latitude) query += "ON DUPLICATE KEY UPDATE " # Specifies the location tuple for the update clause query += "city='%s', country_code='%s', longitude=%s, latitude=%s" % (city, country_code, longitude, latitude) selexorhelper.autoretry_mysql_command(cursor, query)
def _port_parser(cursor, invert, parameters): ''' <Purpose> Vessel-Level Rule. Ensures that all vessels in the group have the specified port number. This is a rule callback. See the Usage section of the module docstring for more information. <Arguments> 'port': The port number that all vessels in the set must have available. ''' good_handles = set() port = parameters['port'] if not invert: query = "SELECT node_id, vessel_name FROM vesselports WHERE port="+str(port) else: query = "SELECT node_id, vessel_name FROM vesselports WHERE port !="+str(port) logger.debug(query) selexorhelper.autoretry_mysql_command(cursor, query) return cursor.fetchall()
def update_vessels_table(cursor, node_id, vessel_dict): query = 'SELECT vessel_name FROM vessels WHERE node_id='+str(node_id) num_vessels = selexorhelper.autoretry_mysql_command(cursor, query) vessel_rows = cursor.fetchall() if vessel_rows: # Remove vessels that were lost, if any vessels_to_remove = [] for [vessel_name] in vessel_rows: if not vessel_name in vessel_dict: # We pass this to MySQL later; MySQL expects strings to be wrapped in quotes. vessels_to_remove += ['"'+vessel_name+'"'] if vessels_to_remove: logger.info('\n'.join([ "Node #"+str(node_id), "Lost vessels: "+str(vessels_to_remove), ])) query = 'DELETE FROM vessels WHERE node_id='+str(node_id)+' AND vessel_name in ('+', '.join(vessels_to_remove)+')' selexorhelper.autoretry_mysql_command(cursor, query) # Update the list of vessels logger.info('\n'.join([ "Node #"+str(node_id), "Current vessels: "+str(vessel_dict.keys()), "Number of Vessels in database (excluding v2): " + str(num_vessels) ])) if vessel_dict: # IGNORE keyword is to tell MySQL to ignore vessels that already exist. query = 'INSERT IGNORE INTO vessels (node_id, vessel_name, acquirable) VALUES' for vessel_name in vessel_dict: # v2 can never be used... No sense in tracking it in the vessel database. if vessel_name == 'v2': continue acquirable = vessel_dict[vessel_name]['acquirable'] query += " ('"+str(node_id)+"', '"+vessel_name+"', "+str(acquirable)+")," # Get rid of the trailing comma after the last tuple query = query.strip(',') selexorhelper.autoretry_mysql_command(cursor, query)
def resolve_node(self, identity, client, node, db, cursor): # We should never run into these... if node['status'] == STATUS_RESOLVED: logger.error(str(identity) + ": Group already resolved: " + str(node)) return node elif node['status'] == STATUS_FAILED: logger.error(str(identity) + ": Exceeded pass limit: " + str(node)) return node logger.info(str(identity) + ": Group " + node['id'] + " on Pass " + str(node['pass'])) vessels_to_acquire = [] remaining = node['allocate'] - len(node['acquired']) selexorhelper.autoretry_mysql_command(cursor, "SELECT node_id, vessel_name FROM vessels WHERE acquirable") all_vessels = cursor.fetchall() # Get vessels that match the vessel rules handles_vesselrulematch = selexorruleparser.apply_vessel_rules(node['rules'], cursor, all_vessels) logger.info(str(identity) + ": Vessel-level matches: " + str(len(handles_vesselrulematch))) # The number of times we tried to resolve this group in the current attempt in_group_retry_count = 0 MAX_IN_GROUP_RETRIES = 3 candidate_vessels = [] while len(candidate_vessels) < remaining and \ in_group_retry_count < MAX_IN_GROUP_RETRIES: if not self._running: # Stop if we receive a quit message break if node['pass'] >= MAX_PASSES_PER_NODE: raise selexorexceptions.SelexorInternalError("Performing more passes than max pass!") handles_grouprulematch = selexorruleparser.apply_group_rules( cursor = cursor, acquired_vessels = candidate_vessels, rules = node['rules'], vesselset = handles_vesselrulematch) # Pick any vessel. vessellist = list(handles_grouprulematch) if vessellist: logger.info(str(identity) + ": Candidates for next vessel: " + str(len(vessellist))) # If we run out of handles, we simply get another random one, instead of # programming a special case. node_id, vesselname = random.choice(vessellist) vessellist.remove((node_id, vesselname)) selexorhelper.autoretry_mysql_command(cursor, 'SELECT node_key FROM nodes WHERE node_id='+str(node_id)) nodekey = cursor.fetchone()[0] handle = nodekey + ':' + vesselname logger.info(str(identity)+":\n"+"Considering: "+str(handle)) # node_id and vessel_name are used extensively by rule parsers # We should include them here to prevent each rule from looking the up vessel_dict = { 'handle': handle, 'node_id': node_id, 'node_key': nodekey, 'vessel_name': vesselname, } candidate_vessels.append(vessel_dict) # We ran out of vessels to check else: logger.info(str(identity) + ": Can't find any suitable vessels!") in_group_retry_count += 1 # Have we exceeded the maximum in-group retry count? if in_group_retry_count >= MAX_IN_GROUP_RETRIES: break # We retry if there could be another combination that MIGHT satisfy # the group rules. If there are no group rules, there is no point # to retry. if not selexorruleparser.has_group_rules(node['rules']): logger.info(str(identity) + ": There are no group rules applied; no point in retrying.") break if candidate_vessels: # Get the vessel that causes the largest drop in the # size of the available vessel pool worst_vessel = selexorruleparser.get_worst_vessel( candidate_vessels, handles_grouprulematch, cursor, node['rules']) # Release the worst vessel so that we can try to get a better one # in the next iteration logger.info(str(identity) + ": Releasing: " + str(worst_vessel)) candidate_vessels.remove(worst_vessel) client.release_resources([worst_vessel['handle']]) # We may get vessels that are unusable (i.e. extra vessels containing # leftover resources). If so, drop them and try again while candidate_vessels: vessels_to_acquire = [] for vesseldict in candidate_vessels: vessels_to_acquire.append(vesseldict['handle']) try: acquired_vesseldicts = client.acquire_specific_vessels(vessels_to_acquire) logger.info(str(identity)+": Requested "+str(len(vessels_to_acquire))+" vessels, acquired "+str(len(acquired_vesseldicts))+":\n"+'\n'.join(i['handle'][-10:] +':'+ i['vessel_id'] for i in acquired_vesseldicts)) # We must be careful to count only the vessels that we have # actually acquired, as some vessels may not have been given to # the user due to the database having slightly outdated # information. acquired_vesseldicts = _get_acquired_vesseldicts( clearinghouse_vesseldicts=acquired_vesseldicts, selexor_vesseldicts=candidate_vessels) node['acquired'] += acquired_vesseldicts break except seattleclearinghouse_xmlrpc.NotEnoughCreditsError, e: logger.error(str(identity) + ": Not enough vessel credits") raise except seattleclearinghouse_xmlrpc.InvalidRequestError, e: error_string = str(e) # This may be an extra vessel. if 'There is no vessel with the node identifier' in error_string: logger.error(str(identity) + ": " + str(e)) extra_vessels = [] for vessel in candidate_vessels: if (vessel['node_key'] in error_string and vessel['vessel_name'] in error_string): extra_vessels.append(vessel) for vessel in extra_vessels: candidate_vessels.remove(vessel) logger.info("Removing: ..." + vessel['node_key'][-10:] + ':' + vessel['vessel_name']) # Store into the db so that future lookups do not need to # spend time acquiring the vessel to discover that it is not # acquirable, as there are no definitive ways of determining # if a vessel is non-acquirable, aside from the management # vessel (v2) update_command = ("UPDATE vessels SET acquirable=false \ WHERE (node_id, vessel_name) IN (" + ", ".join( "(%s, '%s')" % (handle['node_id'], handle['vessel_name']) for handle in extra_vessels ) + ")") selexorhelper.autoretry_mysql_command(cursor, update_command) db.commit() else: logger.error(str(identity) + ": " + str(e)) raise
def commit_data_to_database(db, cursor, node_ip, node_port, node_dict, ports, geoinfo): # Just in case we attempted to make any changes in a previous run and failed db.rollback() # == Update Nodes Table == nodekeystr = rsa_publickey_to_string(node_dict['nodekey']) new_node = selexorhelper.autoretry_mysql_command(cursor, "SELECT node_id FROM nodes WHERE node_key='"+nodekeystr+"'") == 0L if new_node: node_type = selexorhelper.get_node_type(node_ip) # Node isn't recognized, add it to the db cmd = ("INSERT INTO nodes " + "(node_key, ip_addr, node_port, node_type, last_ip_change, last_seen) " + "VALUES ('%s', '%s', %i, '%s', NOW(), NOW())" % (nodekeystr, node_ip, node_port, node_type) ) selexorhelper.autoretry_mysql_command(cursor, cmd) # Now retrieve the internal node_id. selexorhelper.autoretry_mysql_command(cursor, "SELECT node_id FROM nodes WHERE node_key='"+nodekeystr+"'") node_id = cursor.fetchone()[0] logger.info('\n'.join([ "New node found: #" + str(node_id), "Nodekey:", nodekeystr ])) else: # Handle already exists node_id = cursor.fetchone()[0] logger.info('\n'.join([ "Updating node: #" + str(node_id), "Nodekey:", nodekeystr ])) # Did the IP address change? If so mark that down. selexorhelper.autoretry_mysql_command(cursor, "SELECT (ip_addr) FROM nodes WHERE node_key='"+nodekeystr+"'") old_node_ip = cursor.fetchone()[0] node_ip_changed = node_ip != old_node_ip if node_ip_changed: cmd = ( "UPDATE nodes SET ip_addr='%s', last_ip_change=NOW() WHERE node_id=%i" % (node_ip, node_id) ) selexorhelper.autoretry_mysql_command(cursor, cmd) # Update the node type if necessary if node_ip_changed or settings.force_refresh_node_type: node_type = selexorhelper.get_node_type(node_ip) cmd = "UPDATE nodes SET node_type='%s' WHERE node_id=%i" % (node_type, node_id) selexorhelper.autoretry_mysql_command(cursor, cmd) # Update the last time we saw the node. cmd = "UPDATE nodes SET last_seen=NOW() WHERE node_id=%i" % (node_id) selexorhelper.autoretry_mysql_command(cursor, cmd) # == Update Vessels Table == update_vessels_table(cursor, node_id, node_dict['vessels']) # == Update Userkeys == update_userkeys_table(cursor, node_id, node_dict['vessels']) # == Update Ports == update_ports_table(cursor, node_id, ports) # == Update Location Table == if not geoinfo is None and geoinfo: update_location_table(cursor, node_ip, geoinfo) db.commit()
def contact_vessels_and_update_database(nodes_to_check): ''' Of all nodes that need checking, take one node. Obtain its: IP address name of each vessel on it Location For each vessel, obtain: vessel name ports available Finally, update the information about these nodes. ''' db, cursor = selexorhelper.connect_to_db() while nodes_to_check: nodelocation = nodes_to_check.pop() nodeinfo = selexorhelper.get_node_ip_port_from_nodelocation(nodelocation) # We can't use NAT addresses, nor ipv6 if not selexorhelper.is_ipv4_address(nodeinfo['id']): # if nodeinfo['id'].startswith('NAT'): # self._nat_nodes.append(nodeinfo['id']) continue # Used to communicate with the node node_nmhandle = None try: node_nmhandle = nmclient_createhandle(nodeinfo['id'], nodeinfo['port']) node_dict = nmclient_getvesseldict(node_nmhandle) ports = {} for vesselname in node_dict['vessels']: resources_string = nmclient_rawsay(node_nmhandle, "GetVesselResources", vesselname) ports[vesselname] = selexorhelper.get_ports_from_resource_string(resources_string) node_dict['vessels'][vesselname]['acquirable'] = \ selexorhelper.is_resource_acquirable(resources_string) # Retrieve the geographic information try: # We need to some initial value so that it is not undefined when we check it later. geoinfo = None # Only retrieve the geographic information if we don't have it already # The geoip server's data doesn't change, so we don't need to constantly update it. geoinfo_exists = selexorhelper.autoretry_mysql_command(cursor, "SELECT ip_addr FROM location WHERE ip_addr='"+nodeinfo['id']+"'") == 1L if not geoinfo_exists: logger.info("Location data not in database, looking up on geoip: "+nodelocation) geoinfo = geoip_record_by_addr(nodeinfo['id']) except Exception, e: if not "Unable to contact the geoip server" in str(e): raise # The geoip lookup sometimes returns None. if geoinfo is None: geoinfo = {} format_geoinfo(geoinfo) commit_data_to_database(db, cursor, nodeinfo['id'], nodeinfo['port'], node_dict, ports, geoinfo) except NMClientException, e: if not node_nmhandle: nmclient_destroyhandle(node_nmhandle) # self._bad_node_locations.append(nodelocation) errstr = str(e) if ("timed out" in errstr or 'No connection could be made because the target machine actively refused it' in errstr or "Connection refused" in errstr or "Socket closed" in errstr): continue logger.error("Unknown error contacting " + nodelocation + traceback.format_exc())