def allocate_console_port(instance_uuid): node = config.NODE_NAME consumed = {value['port'] for value in etcd.get_all('console', node)} while True: port = random.randint(30000, 50000) # avoid hitting etcd if it's probably in use if port in consumed: continue s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: # We hold this port open until it's in etcd to prevent # anyone else needing to hit etcd to find out they can't # use it as well as to verify we can use it s.bind(('0.0.0.0', port)) allocatedPort = etcd.create('console', node, port, { 'instance_uuid': instance_uuid, 'port': port, }) if allocatedPort: return port except socket.error as e: LOG.withField('instance', instance_uuid).info( "Exception during port allocation: %s" % e) finally: s.close()
def __iter__(self): for _, n in etcd.get_all('network', None): if n['uuid'] == 'floating': continue out = self.apply_filters(Network(n)) if out: yield out
def get_network_interfaces(network_uuid): see_this_node() for ni in etcd.get_all('networkinterface', None): if ni['state'] == 'deleted': continue if ni['network_uuid'] == network_uuid: yield ni
def _allocate_console_port(self): node = config.NODE_NAME consumed = [ value['port'] for _, value in etcd.get_all('console', node) ] while True: port = random.randint(30000, 50000) # avoid hitting etcd if it's probably in use if port in consumed: continue s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: # We hold this port open until it's in etcd to prevent # anyone else needing to hit etcd to find out they can't # use it as well as to verify we can use it s.bind(('0.0.0.0', port)) allocatedPort = etcd.create('console', node, port, { 'instance_uuid': self.uuid, 'port': port, }) if allocatedPort: return port except socket.error: LOG.with_field('instance', self.uuid).info( 'Collided with in use port %d, selecting another' % port) consumed.append(port) finally: s.close()
def __iter__(self): for _, a in etcd.get_all('artifact', None): a = Artifact.from_db(a['uuid']) if not a: continue out = self.apply_filters(a) if out: yield out
def __iter__(self): for _, b in etcd.get_all('blob', None): b = Blob.from_db(b['uuid']) if not b: continue out = self.apply_filters(b) if out: yield out
def _db_get_attributes(self, attribute_prefix): if self.__in_memory_only: for key in self.__in_memory_values.keys(): if key.startswith(attribute_prefix): yield key, json.loads(self.__in_memory_values[key]) else: for key, data in etcd.get_all('attribute/%s' % self.object_type, self.__uuid, prefix=attribute_prefix): yield key, data
def get_networks(all=False, namespace=None): for n in etcd.get_all('network', None): if n['uuid'] == 'floating': continue if not all: if n['state'] in ['deleted', 'error']: continue if namespace: if namespace not in [n['namespace'], 'system']: continue yield n
def get_instances(only_node=None, all=False, namespace=None): see_this_node() for i in etcd.get_all('instance', None): if only_node and i['node'] != only_node: continue if not all: if i['state'] in ['deleted', 'error']: continue if namespace: if namespace not in [i['namespace'], 'system']: continue yield i
def allocate_console_port(instance_uuid): node = config.parsed.get('NODE_NAME') with etcd.get_lock('console', None, node): consumed = [] for value in etcd.get_all('console', node): consumed.append(value['port']) port = random.randint(30000, 50000) while port in consumed or not _port_free(port): port = random.randint(30000, 50000) etcd.put('console', node, port, { 'instance_uuid': instance_uuid, 'port': port, }) return port
def get_instances(only_node=None, all=False, namespace=None): for i in etcd.get_all('instance', None): if only_node and i['node'] != only_node: continue if not all: if i['state'] in ['deleted', 'error']: continue if namespace: if namespace not in [i['namespace'], 'system']: continue if 'video' not in i: i['video'] = {'model': 'cirrus', 'memory': 16384} if 'error_message' not in i: i['error_message'] = None yield i
def allocate_console_port(instance_uuid): see_this_node() node = config.parsed.get('NODE_NAME') with etcd.get_lock('sf/console/%s' % node) as _: consumed = [] for value in etcd.get_all('console', node): consumed.append(value['port']) port = random.randint(30000, 50000) while port in consumed: port = random.randint(30000, 50000) etcd.put('console', node, port, { 'instance_uuid': instance_uuid, 'port': port, }) return port
def get_nodes(): see_this_node() return etcd.get_all('node', None)
def __iter__(self): for _, i in etcd.get_all('instance', None): out = self.apply_filters(Instance(i)) if out: yield out
def get_nodes(): return etcd.get_all('node', None)
def get_node_ips(): for value in etcd.get_all('node', None): yield value['ip']
def list_namespaces(): return etcd.get_all('namespace', None)
def get_node_ips(): see_this_node() for value in etcd.get_all('node', None): yield value['ip']
def get_events(object_type, object_uuid): for m in etcd.get_all('event/%s' % object_type, object_uuid, sort_order='ascend'): yield m
def get_instance_snapshots(instance_uuid): for m in etcd.get_all('snapshot', instance_uuid, sort_order='ascend'): yield m
def get_instance_interfaces(instance_uuid): for ni in etcd.get_all('networkinterface', None): if ni['state'] == 'deleted': continue if ni['instance_uuid'] == instance_uuid: yield ni
def filter(cls, filters): for _, o in etcd.get_all(cls.object_type, None): obj = cls(o) if all([f(obj) for f in filters]): yield obj
def _cluster_wide_cleanup(self, last_loop_run): LOG.info('Running cluster maintenance') # Cleanup soft deleted objects for objtype in OBJECT_NAMES: for _, objdata in etcd.get_all(objtype, None): try: obj = OBJECT_NAMES_TO_CLASSES[objtype].from_db( objdata['uuid']) if (obj.state.value == dbo.STATE_DELETED and time.time() - obj.state.update_time > config.CLEANER_DELAY): LOG.with_object(obj).info('Hard deleting') obj.hard_delete() except exceptions.BadObjectVersion: LOG.with_fields({ objtype: obj.uuid }).warning('Could not load object for hard delete, bad version') # Prune artifacts which might have too many versions for a in artifact.Artifacts([]): a.delete_old_versions() # Inspect current state of blobs, the actual changes are done below outside # the read only cache. We define being low on disk has having less than three # times the minimum amount of disk. This is so we start to rearrange blobs # before scheduling starts to fail. overreplicated = {} underreplicated = [] low_disk_nodes = nodes_by_free_disk_descending( minimum=0, maximum=(config.MINIMUM_FREE_DISK * 3), intention='blobs') absent_nodes = [] for n in Nodes([node_inactive_states_filter]): LOG.with_fields({ 'node': n.fqdn}).info('Node is absent for blob replication') absent_nodes.append(n.fqdn) LOG.info('Found %d inactive nodes' % len(absent_nodes)) current_fetches = defaultdict(list) for workname, workitem in etcd.get_outstanding_jobs(): # A workname looks like: /sf/queue/sf-3/jobname _, _, phase, node, _ = workname.split('/') if node == 'networknode': continue for task in workitem: if isinstance(task, FetchBlobTask): if node in absent_nodes: LOG.with_fields({ 'blob': task.blob_uuid, 'node': node, 'phase': phase }).warning('Node is absent, ignoring fetch') else: LOG.with_fields({ 'blob': task.blob_uuid, 'node': node, 'phase': phase }).info('Node is fetching blob') current_fetches[task.blob_uuid].append(node) with etcd.ThreadLocalReadOnlyCache(): for b in blob.Blobs([active_states_filter]): # If there is current work for a blob, we ignore it until that # work completes if b.uuid in current_fetches: LOG.with_fields({ 'blob': task.blob_uuid }).info('Blob has current fetches, ignoring') continue locations = b.locations ignored_locations = [] for n in absent_nodes: if n in locations: locations.remove(n) ignored_locations.append(n) if ignored_locations: LOG.with_fields({ 'blob': b, 'ignored_locations': ignored_locations }).info('Ignored some blob locations as nodes are absent') delta = len(locations) - config.BLOB_REPLICATION_FACTOR if delta > 0: # So... The blob replication factor is a target not a limit. # Specifically, if there are more locations than the target # but we aren't low on disk, we don't clean them up. That's # because its hard for us to predict which machine will run # out of disk first, and copying a blob back to a machine if # its needed there is slow and annoying. # Work out where the blob is in active use. excess_locations = b.locations in_use_locations = [] for instance_uuid in b.instances: i = instance.Instance.from_db(instance_uuid) node = i.placement.get('node') if node in excess_locations: excess_locations.remove(node) in_use_locations.append(node) # Only remove excess copies from nodes which are running # low on disk. Do not end up with too few replicas. overreplicated[b.uuid] = [] target = (config.BLOB_REPLICATION_FACTOR - len(in_use_locations)) for n in low_disk_nodes: if n in excess_locations: overreplicated[b.uuid].append(n) if len(overreplicated[b.uuid]) == target: break elif delta < 0: # The tuple is blob UUID, and how much to over replicate by. underreplicated.append((b.uuid, 0)) else: # We have exactly the right number of copies, but what if # the blob is on a really full node? for n in low_disk_nodes: if n in b.locations: # We have at least one space constrained node with # this blob. Request an extra temporary copy of the # blob elsewhere so we can hopefully clean up one of # these next pass. The tuple is blob UUID, and how # much to over replicate by. underreplicated.append((b.uuid, 1)) break # Prune over replicated blobs for blob_uuid in overreplicated: b = blob.Blob.from_db(blob_uuid) for node in overreplicated[blob_uuid]: LOG.with_fields({ 'blob': b, 'node': node }).info('Blob over replicated, removing from node with no users') b.drop_node_location(node) # Replicate under replicated blobs, but only if we don't have heaps of # queued replications already for blob_uuid, excess in underreplicated: LOG.with_fields({ 'current': len(current_fetches), 'maximum': config.MAX_CONCURRENT_BLOB_TRANSFERS }).info('Concurrent blob transfers') if len(current_fetches) > config.MAX_CONCURRENT_BLOB_TRANSFERS: LOG.info( 'Too many concurrent blob transfers queued, not queueing more') break b = blob.Blob.from_db(blob_uuid) LOG.with_fields({ 'blob': b }).info('Blob under replicated, attempting to correct') b.request_replication(allow_excess=excess) current_fetches[blob_uuid].append('unknown') # Node management for n in Nodes([node_inactive_states_filter]): age = time.time() - n.last_seen # Find nodes which have returned from being missing if age < config.NODE_CHECKIN_MAXIMUM: n.state = Node.STATE_CREATED LOG.with_object(n).info('Node returned from being missing') # Find nodes which have been offline for a long time, unless # this machine has been asleep for a long time (think developer # laptop). if (time.time() - last_loop_run < config.NODE_CHECKIN_MAXIMUM and age > config.NODE_CHECKIN_MAXIMUM * 10): n.state = Node.STATE_ERROR for i in instance.healthy_instances_on_node(n): LOG.with_object(i).with_object(n).info( 'Node in error state, erroring instance') # Note, this queue job is just in case the node comes # back. i.enqueue_delete_due_error('Node in error state') # Find nodes which haven't checked in recently for n in Nodes([node_active_states_filter]): age = time.time() - n.last_seen if age > config.NODE_CHECKIN_MAXIMUM: n.state = Node.STATE_MISSING # And we're done LOG.info('Cluster maintenance loop complete')
def list_namespaces(): for _, value in etcd.get_all('namespace', None): yield value
def __iter__(self): for _, ni in etcd.get_all('networkinterface', None): out = self.apply_filters(NetworkInterface(ni)) if out: yield out
def get_stale_instances(delay): for i in etcd.get_all('instance', None): if i['state'] in ['deleted', 'error']: if time.time() - i['state_updated'] > delay: yield i
def get_stale_network_interfaces(delay): for n in etcd.get_all('networkinterface', None): if n['state'] in ['deleted', 'error']: if time.time() - n['state_updated'] > delay: yield n
def get_stale_networks(delay): see_this_node() for n in etcd.get_all('network', None): if n['state'] in ['deleted', 'error']: if time.time() - n['state_updated'] > delay: yield n