def list_networks(self, persist=True): """Return list of networks for cloud A list of networks is fetched from libcloud, data is processed, stored on network models, and a list of network models is returned. Subclasses SHOULD NOT override or extend this method. This method wraps `_list_networks` which contains the core implementation. """ task_key = 'cloud:list_networks:%s' % self.cloud.id task = PeriodicTaskInfo.get_or_add(task_key) first_run = False if task.last_success else True async def _list_subnets_async(networks): loop = asyncio.get_event_loop() subnets = [ loop.run_in_executor(None, network.ctl.list_subnets) for network in networks ] return await asyncio.gather(*subnets) with task.task_runner(persist=persist): # Get cached networks as dict cached_networks = { '%s-%s' % (n.id, n.network_id): n.as_dict() for n in self.list_cached_networks() } networks = self._list_networks() loop = asyncio.get_event_loop() loop.run_until_complete(_list_subnets_async(networks)) # Publish patches to rabbitmq. new_networks = { '%s-%s' % (n.id, n.network_id): n.as_dict() for n in networks } # Exclude last seen and probe field if cached_networks or new_networks: # Publish patches to rabbitmq. patch = jsonpatch.JsonPatch.from_diff(cached_networks, new_networks).patch if patch: if not first_run and self.cloud.observation_logs_enabled: from mist.api.logs.methods import log_observations log_observations(self.cloud.owner.id, self.cloud.id, 'network', patch, cached_networks, new_networks) if amqp_owner_listening(self.cloud.owner.id): amqp_publish_user(self.cloud.owner.id, routing_key='patch_networks', data={ 'cloud_id': self.cloud.id, 'patch': patch }) return networks
def ping_probe(self, persist=True): if not self.machine.cloud.enabled: return False from mist.api.methods import ping from mist.api.machines.models import PingProbe def _get_probe_dict(): data = {} if self.machine.ping_probe is not None: data = self.machine.ping_probe.as_dict() return { '%s-%s' % (self.machine.id, self.machine.machine_id): { 'probe': { 'ping': data } } } try: host = self.machine.ctl.get_host() if host in ['localhost', '127.0.0.1']: return except RuntimeError: return old_probe_data = _get_probe_dict() task_key = 'machine:ping_probe:%s' % self.machine.id task = PeriodicTaskInfo.get_or_add(task_key) with task.task_runner(persist=persist): try: data = ping(self.machine.cloud.owner, self.get_host()) except: probe = self.machine.ping_probe if probe is not None: probe.unreachable_since = datetime.datetime.now() raise else: probe = PingProbe() probe.update_from_dict(data) finally: self.machine.ping_probe = probe self.machine.save() new_probe_data = _get_probe_dict() patch = jsonpatch.JsonPatch.from_diff(old_probe_data, new_probe_data).patch if patch: amqp_publish_user(self.machine.cloud.owner.id, routing_key='patch_machines', data={ 'cloud_id': self.machine.cloud.id, 'patch': patch }) probe_result = self.machine.ping_probe return probe_result and probe_result.as_dict()
def list_networks(self, persist=True): """Return list of networks for cloud A list of networks is fetched from libcloud, data is processed, stored on network models, and a list of network models is returned. Subclasses SHOULD NOT override or extend this method. This method wraps `_list_networks` which contains the core implementation. """ task_key = 'cloud:list_networks:%s' % self.cloud.id task = PeriodicTaskInfo.get_or_add(task_key) with task.task_runner(persist=persist): cached_networks = { '%s' % n.id: n.as_dict() for n in self.list_cached_networks() } networks = self._list_networks() # Initialize AMQP connection to reuse for multiple messages. amqp_conn = Connection(config.AMQP_URI) if amqp_owner_listening(self.cloud.owner.id): networks_dict = [n.as_dict() for n in networks] if cached_networks and networks_dict: # Publish patches to rabbitmq. new_networks = {'%s' % n['id']: n for n in networks_dict} patch = jsonpatch.JsonPatch.from_diff(cached_networks, new_networks).patch if patch: amqp_publish_user(self.cloud.owner.id, routing_key='patch_networks', connection=amqp_conn, data={ 'cloud_id': self.cloud.id, 'patch': patch }) else: # TODO: remove this block, once patches # are implemented in the UI amqp_publish_user(self.cloud.owner.id, routing_key='list_networks', connection=amqp_conn, data={ 'cloud_id': self.cloud.id, 'networks': networks_dict }) return networks
def ssh_probe(self, persist=True): from mist.api.methods import probe_ssh_only from mist.api.machines.models import SSHProbe def _get_probe_dict(): data = {} if self.machine.ssh_probe is not None: data = self.machine.ssh_probe.as_dict() return { '%s-%s' % (self.machine.id, self.machine.machine_id): { 'probe': { 'ssh': data } } } old_probe_data = _get_probe_dict() task_key = 'machine:ssh_probe:%s' % self.machine.id task = PeriodicTaskInfo.get_or_add(task_key) with task.task_runner(persist=persist): try: data = probe_ssh_only( self.machine.cloud.owner, self.machine.cloud.id, self.machine.machine_id, self.get_host(), ) except: probe = self.machine.ssh_probe if probe is not None: probe.unreachable_since = datetime.datetime.now() raise else: probe = SSHProbe() probe.update_from_dict(data) finally: self.machine.ssh_probe = probe self.machine.save() new_probe_data = _get_probe_dict() patch = jsonpatch.JsonPatch.from_diff(old_probe_data, new_probe_data).patch if patch: amqp_publish_user(self.machine.cloud.owner.id, routing_key='patch_machines', data={ 'cloud_id': self.machine.cloud.id, 'patch': patch }) return self.machine.ssh_probe.as_dict()
def list_zones(self, persist=True): """Return list of zones for cloud A list of zones is fetched from libcloud, data is processed, stored on zone models, and a list of zone models is returned. Subclasses SHOULD NOT override or extend this method. This method wraps `_list_zones` which contains the core implementation. """ task_key = 'cloud:list_zones:%s' % self.cloud.id task = PeriodicTaskInfo.get_or_add(task_key) first_run = False if task.last_success else True with task.task_runner(persist=persist): cached_zones = { '%s-%s' % (z.id, z.zone_id): z.as_dict() for z in self.list_cached_zones() } zones = self._list_zones() for zone in zones: self.list_records(zone) # Initialize AMQP connection to reuse for multiple messages. if amqp_owner_listening(self.cloud.owner.id): zones_dict = [z.as_dict() for z in zones] if cached_zones or zones_dict: # Publish patches to rabbitmq. new_zones = { '%s-%s' % (z['id'], z['zone_id']): z for z in zones_dict } patch = jsonpatch.JsonPatch.from_diff(cached_zones, new_zones).patch if patch: if not first_run and self.cloud.observation_logs_enabled: from mist.api.logs.methods import log_observations log_observations(self.cloud.owner.id, self.cloud.id, 'zone', patch, cached_zones, new_zones) amqp_publish_user(self.cloud.owner.id, routing_key='patch_zones', data={ 'cloud_id': self.cloud.id, 'patch': patch }) return zones
def list_volumes(self, persist=True): """Return list of volumes for cloud A list of volumes is fetched from libcloud, data is processed, stored on volume models, and a list of volume models is returned. Subclasses SHOULD NOT override or extend this method. This method wraps `_list_volumes` which contains the core implementation. """ task_key = 'cloud:list_volumes:%s' % self.cloud.id task = PeriodicTaskInfo.get_or_add(task_key) first_run = False if task.last_success else True with task.task_runner(persist=persist): cached_volumes = { '%s-%s' % (v.id, v.external_id): v.as_dict() for v in self.list_cached_volumes() } volumes = self._list_volumes() volumes_dict = [v.as_dict() for v in volumes] if cached_volumes or volumes: # Publish patches to rabbitmq. new_volumes = { '%s-%s' % (v['id'], v['external_id']): v for v in volumes_dict } patch = jsonpatch.JsonPatch.from_diff(cached_volumes, new_volumes).patch if patch: if not first_run and self.cloud.observation_logs_enabled: from mist.api.logs.methods import log_observations log_observations(self.cloud.owner.id, self.cloud.id, 'volume', patch, cached_volumes, new_volumes) if amqp_owner_listening(self.cloud.owner.id): amqp_publish_user(self.cloud.owner.id, routing_key='patch_volumes', data={ 'cloud_id': self.cloud.id, 'patch': patch }) return volumes
def ping_probe(self, persist=True): from mist.api.methods import ping from mist.api.machines.models import PingProbe def _get_probe_dict(): data = {} if self.machine.ping_probe is not None: data = self.machine.ping_probe.as_dict() return { '%s-%s' % (self.machine.id, self.machine.machine_id): { 'probe': { 'ping': data } } } old_probe_data = _get_probe_dict() task_key = 'machine:ping_probe:%s' % self.machine.id task = PeriodicTaskInfo.get_or_add(task_key) with task.task_runner(persist=persist): data = ping(self.machine.cloud.owner, self.get_host()) probe = PingProbe() probe.update_from_dict(data) self.machine.ping_probe = probe self.machine.save() new_probe_data = _get_probe_dict() patch = jsonpatch.JsonPatch.from_diff(old_probe_data, new_probe_data).patch if patch: amqp_publish_user(self.machine.cloud.owner.id, routing_key='patch_machines', data={ 'cloud_id': self.machine.cloud.id, 'patch': patch }) return self.machine.ping_probe.as_dict()
def list_machines(self, persist=True): """Return list of machines for cloud A list of nodes is fetched from libcloud, the data is processed, stored on machine models, and a list of machine models is returned. Subclasses SHOULD NOT override or extend this method. This method wraps `_list_machines` which contains the core implementation. """ task_key = 'cloud:list_machines:%s' % self.cloud.id task = PeriodicTaskInfo.get_or_add(task_key) try: with task.task_runner(persist=persist): old_machines = { '%s-%s' % (m.id, m.machine_id): m.as_dict() for m in self.list_cached_machines() } machines = self._list_machines() except PeriodicTaskThresholdExceeded: self.cloud.disable() raise # Initialize AMQP connection to reuse for multiple messages. amqp_conn = Connection(config.AMQP_URI) if amqp_owner_listening(self.cloud.owner.id): if not config.MACHINE_PATCHES: amqp_publish_user( self.cloud.owner.id, routing_key='list_machines', connection=amqp_conn, data={ 'cloud_id': self.cloud.id, 'machines': [machine.as_dict() for machine in machines] }) else: # Publish patches to rabbitmq. new_machines = { '%s-%s' % (m.id, m.machine_id): m.as_dict() for m in machines } # Exclude last seen and probe fields from patch. for md in old_machines, new_machines: for m in md.values(): m.pop('last_seen') m.pop('probe') patch = jsonpatch.JsonPatch.from_diff(old_machines, new_machines).patch if patch: amqp_publish_user(self.cloud.owner.id, routing_key='patch_machines', connection=amqp_conn, data={ 'cloud_id': self.cloud.id, 'patch': patch }) # Push historic information for inventory and cost reporting. for machine in machines: data = { 'owner_id': self.cloud.owner.id, 'machine_id': machine.id, 'cost_per_month': machine.cost.monthly } amqp_publish(exchange='machines_inventory', routing_key='', auto_delete=False, data=data, connection=amqp_conn) return machines