def test_is_public_and_is_private_subnet(self): public_ips = [ '213.151.0.8', '86.87.86.1', '8.8.8.8', '8.8.4.4' ] private_ips = [ '192.168.1.100', '10.0.0.1', '172.16.0.0' ] for address in public_ips: is_public = is_public_subnet(ip=address) is_private = is_private_subnet(ip=address) self.assertTrue(is_public) self.assertFalse(is_private) for address in private_ips: is_public = is_public_subnet(ip=address) is_private = is_private_subnet(ip=address) self.assertFalse(is_public) self.assertTrue(is_private)
def _to_node(self, data): state = NODE_STATE_MAP[data["state"]] public_ips = [] private_ips = [] extra = {} for ip in data["ips"]: if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) if "credentials" in data["metadata"]: extra["password"] = data["metadata"]["credentials"]["root"] node = Node( id=data["id"], name=data["name"], state=state, public_ips=public_ips, private_ips=private_ips, driver=self.connection.driver, extra=extra, ) return node
def _to_node(self, data, public_ips=None): """ :param data: Node data object. :type data: ``dict`` :param public_ips: A list of additional IP addresses belonging to this node. (optional) :type public_ips: ``list`` or ``None`` """ id = data['id'] if 'name' in data: name = data['name'] elif 'displayname' in data: name = data['displayname'] else: name = None state = self.NODE_STATE_MAP[data['state']] public_ips = public_ips if public_ips else [] private_ips = [] for nic in data['nic']: if 'ipaddress' in nic: if is_private_subnet(nic['ipaddress']): private_ips.append(nic['ipaddress']) else: public_ips.append(nic['ipaddress']) security_groups = data.get('securitygroup', []) if security_groups: security_groups = [sg['name'] for sg in security_groups] affinity_groups = data.get('affinitygroup', []) if affinity_groups: affinity_groups = [ag['id'] for ag in affinity_groups] created = data.get('created', False) extra = self._get_extra_dict(data, RESOURCE_EXTRA_ATTRIBUTES_MAP['node']) # Add additional parameters to extra extra['security_group'] = security_groups extra['affinity_group'] = affinity_groups extra['ip_addresses'] = [] extra['ip_forwarding_rules'] = [] extra['port_forwarding_rules'] = [] extra['created'] = created if 'tags' in data: extra['tags'] = self._get_resource_tags(data['tags']) node = CloudStackNode(id=id, name=name, state=state, public_ips=public_ips, private_ips=private_ips, driver=self, extra=extra) return node
def check_host(self, hostname, ssh_port=22): """Check if host is running. Initially attempt a connection to ssh port specified for host and also to a list of common ports. If connection is successfull, then consider host as running. If not, send an ICMP package with ping. If this fails too, consider host state as stopped. Still needs to be improved to perform more robust checks. """ state = NODE_STATE_MAP['unknown'] if not hostname: return state socket.setdefaulttimeout(5) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ports_list = [22, 80, 443, 3389] if ssh_port not in ports_list: ports_list.insert(0, ssh_port, ) for port in ports_list: try: s.connect((hostname, port)) s.shutdown(2) state = NODE_STATE_MAP['on'] break except: pass if state == NODE_STATE_MAP['unknown']: if is_private_subnet(hostname): hostname = '-I ' + config.VIRTUAL_INTERFACES[0] + ' ' + str(hostname) ping_response = self.ping_host(hostname) if ping_response == 0: state = NODE_STATE_MAP['on'] return state
def _to_node(self, data): """Convert node in Node instances """ state = NODE_STATE_MAP.get(data.get('power_status'), '4') public_ips = [] private_ips = [] ip_addresses = data.get('ipaddresses', '') #E.g. "ipaddresses": "198.120.14.6, 10.132.60.1" if ip_addresses: for ip in ip_addresses.split(','): ip = ip.replace(' ', '') if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) extra = { 'zone_data': data.get('zone'), 'zone': data.get('zone', {}).get('name'), 'image': data.get('image', {}).get('friendly_name'), 'create_time': data.get('create_time'), 'network_ports': data.get('network_ports'), 'is_console_enabled': data.get('is_console_enabled'), 'service_type': data.get('service_type', {}).get('friendly_name'), 'hostname': data.get('hostname') } node = Node(id=data.get('id'), name=data.get('name'), state=state, public_ips=public_ips, private_ips=private_ips, driver=self, extra=extra) return node
def _to_node(self, data): """Convert node in Node instances """ state = NODE_STATE_MAP.get(data.get('power_status'), '4') public_ips = [] private_ips = [] ip_addresses = data.get('ipaddresses', '') # E.g. "ipaddresses": "198.120.14.6, 10.132.60.1" if ip_addresses: for ip in ip_addresses.split(','): ip = ip.replace(' ', '') if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) extra = { 'zone_data': data.get('zone'), 'zone': data.get('zone', {}).get('name'), 'image': data.get('image', {}).get('friendly_name'), 'create_time': data.get('create_time'), 'network_ports': data.get('network_ports'), 'is_console_enabled': data.get('is_console_enabled'), 'service_type': data.get('service_type', {}).get('friendly_name'), 'hostname': data.get('hostname') } node = Node(id=data.get('id'), name=data.get('name'), state=state, public_ips=public_ips, private_ips=private_ips, driver=self, extra=extra) return node
def _extract_public_private_ips(self, ips): private_ips = [] public_ips = [] for ip in ips: if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) return private_ips, public_ips
def test_is_public_and_is_private_subnet(self): public_ips = ["213.151.0.8", "86.87.86.1", "8.8.8.8", "8.8.4.4"] private_ips = ["192.168.1.100", "10.0.0.1", "172.16.0.0"] for address in public_ips: is_public = is_public_subnet(ip=address) is_private = is_private_subnet(ip=address) self.assertTrue(is_public) self.assertFalse(is_private) for address in private_ips: is_public = is_public_subnet(ip=address) is_private = is_private_subnet(ip=address) self.assertFalse(is_public) self.assertTrue(is_private)
def find_public_ips(ips): public_ips = [] for ip in ips: # is_private_subnet does not check for ipv6 try: if not is_private_subnet(ip): public_ips.append(ip) except: pass return public_ips
def _to_node(self, data, public_ips=None): """ :param data: Node data object. :type data: ``dict`` :param public_ips: A list of additional IP addresses belonging to this node. (optional) :type public_ips: ``list`` or ``None`` """ id = data['id'] if 'name' in data: name = data['name'] elif 'displayname' in data: name = data['displayname'] else: name = None state = self.NODE_STATE_MAP[data['state']] public_ips = public_ips if public_ips else [] private_ips = [] for nic in data['nic']: if is_private_subnet(nic['ipaddress']): private_ips.append(nic['ipaddress']) else: public_ips.append(nic['ipaddress']) zone_id = str(data['zoneid']) password = data.get('password', None) keypair = data.get('keypair', None) security_groups = data.get('securitygroup', []) if security_groups: security_groups = [sg['name'] for sg in security_groups] created = data.get('created', False) extra = { 'zoneid': zone_id, 'ip_addresses': [], 'ip_forwarding_rules': [], 'port_forwarding_rules': [], 'password': password, 'keyname': keypair, 'securitygroup': security_groups, 'created': created } node = CloudStackNode(id=id, name=name, state=state, public_ips=public_ips, private_ips=private_ips, driver=self, extra=extra) return node
def _to_node(self, data): state = NODE_STATE_MAP[data['state']] public_ips = [] private_ips = [] extra = {} for ip in data['ips']: if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) if 'credentials' in data['metadata']: extra['password'] = data['metadata']['credentials']['root'] node = Node(id=data['id'], name=data['name'], state=state, public_ips=public_ips, private_ips=private_ips, driver=self.connection.driver, extra=extra) return node
def _to_node(self, data): identifier = data["identifier"] name = data["label"] private_ips = [] public_ips = [] for ip in data["ip_addresses"]: address = ip["ip_address"]['address'] if is_private_subnet(address): private_ips.append(address) else: public_ips.append(address) extra = OnAppNodeDriver._get_extra_dict( data, RESOURCE_EXTRA_ATTRIBUTES_MAP["node"]) return Node(identifier, name, extra['state'], public_ips, private_ips, self, extra=extra)
def _to_node(self, vm): """ Turns a (json) dictionary into a Node object. This returns only running VMs. """ # Check state if not vm['state'] == "running": return None # IPs iplist = [ interface['ip'] for interface in vm['interfaces'] if interface['ip'] != '127.0.0.1' ] public_ips = [] private_ips = [] for ip in iplist: try: socket.inet_aton(ip) except socket.error: # not a valid ip continue if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) # Create the node object n = Node( id=vm['uuid'], name=vm['name'], state=NodeState.RUNNING, public_ips=public_ips, private_ips=private_ips, driver=self, ) return n
def _to_node(self, data): identifier = data["identifier"] name = data["label"] private_ips = [] public_ips = [] for ip in data["ip_addresses"]: address = ip["ip_address"]['address'] if is_private_subnet(address): private_ips.append(address) else: public_ips.append(address) extra = OnAppNodeDriver._get_extra_dict( data, RESOURCE_EXTRA_ATTRIBUTES_MAP["node"] ) return Node(identifier, name, extra['state'], public_ips, private_ips, self, extra=extra)
def _to_node(self, vm): """ Turns a (json) dictionary into a Node object. This returns only running VMs. """ # Check state if not vm['state'] == "running": return None # IPs iplist = [interface['ip'] for interface in vm['interfaces'] if interface['ip'] != '127.0.0.1'] public_ips = [] private_ips = [] for ip in iplist: try: socket.inet_aton(ip) except socket.error: # not a valid ip continue if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) # Create the node object n = Node( id=vm['uuid'], name=vm['name'], state=NodeState.RUNNING, public_ips=public_ips, private_ips=private_ips, driver=self, ) return n
def _to_node(self, data): """Convert node in Node instances""" state = NODE_STATE_MAP.get(data.get("power_status"), "4") public_ips = [] private_ips = [] ip_addresses = data.get("ipaddresses", "") # E.g. "ipaddresses": "198.120.14.6, 10.132.60.1" if ip_addresses: for ip in ip_addresses.split(","): ip = ip.replace(" ", "") if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) extra = { "zone_data": data.get("zone"), "zone": data.get("zone", {}).get("name"), "image": data.get("image", {}).get("friendly_name"), "create_time": data.get("create_time"), "network_ports": data.get("network_ports"), "is_console_enabled": data.get("is_console_enabled"), "service_type": data.get("service_type", {}).get("friendly_name"), "hostname": data.get("hostname"), } node = Node( id=data.get("id"), name=data.get("name"), state=state, public_ips=public_ips, private_ips=private_ips, driver=self, extra=extra, ) return node
def add(self, fail_on_error=True, fail_on_invalid_params=False, **kwargs): from mist.api.machines.models import Machine if not kwargs.get('hosts'): raise RequiredParameterMissingError('hosts') try: self.cloud.save() except me.ValidationError as exc: raise BadRequestError({'msg': str(exc), 'errors': exc.to_dict()}) except me.NotUniqueError: raise CloudExistsError("Cloud with name %s already exists" % self.cloud.title) total_errors = {} for _host in kwargs['hosts']: self._add__preparse_kwargs(_host) errors = {} for key in list(_host.keys()): if key not in ('host', 'alias', 'username', 'port', 'key', 'images_location'): error = "Invalid parameter %s=%r." % (key, _host[key]) if fail_on_invalid_params: self.cloud.delete() raise BadRequestError(error) else: log.warning(error) _host.pop(key) for key in ('host', 'key'): if key not in _host or not _host.get(key): error = "Required parameter missing: %s" % key errors[key] = error if fail_on_error: self.cloud.delete() raise RequiredParameterMissingError(key) else: log.warning(error) total_errors.update({key: error}) if not errors: try: ssh_port = int(_host.get('ssh_port', 22)) except (ValueError, TypeError): ssh_port = 22 images_location = _host.get('images_location', '/var/lib/libvirt/images') extra = { 'images_location': images_location, 'tags': {'type': 'hypervisor'}, 'username': _host.get('username') } # Create and save machine entry to database. machine = Machine( cloud=self.cloud, machine_id=_host.get('host').replace('.', '-'), name=_host.get('alias') or _host.get('host'), ssh_port=ssh_port, last_seen=datetime.datetime.utcnow(), hostname=_host.get('host'), state=NodeState.RUNNING, extra=extra ) # Sanitize inputs. host = sanitize_host(_host.get('host')) check_host(_host.get('host')) machine.hostname = host if is_private_subnet(socket.gethostbyname(_host.get('host'))): machine.private_ips = [_host.get('host')] else: machine.public_ips = [_host.get('host')] try: machine.save(write_concern={'w': 1, 'fsync': True}) except me.NotUniqueError: error = 'Duplicate machine entry. Maybe the same \ host has been added twice?' if fail_on_error: self.cloud.delete() raise MistError(error) else: total_errors.update({_host.get('host'): error}) continue # associate key and attempt to connect try: machine.ctl.associate_key(_host.get('key'), username=_host.get('username'), port=ssh_port) except MachineUnauthorizedError as exc: log.error("Could not connect to host %s." % _host.get('host')) machine.delete() if fail_on_error: self.cloud.delete() raise CloudUnauthorizedError(exc) except ServiceUnavailableError as exc: log.error("Could not connect to host %s." % _host.get('host')) machine.delete() if fail_on_error: self.cloud.delete() raise MistError("Couldn't connect to host '%s'." % _host.get('host')) # check if host was added successfully # if not, delete the cloud and raise if Machine.objects(cloud=self.cloud): if amqp_owner_listening(self.cloud.owner.id): old_machines = [m.as_dict() for m in self.cloud.ctl.compute.list_cached_machines()] new_machines = self.cloud.ctl.compute.list_machines() self.cloud.ctl.compute.produce_and_publish_patch( old_machines, new_machines) self.cloud.errors = total_errors else: self.cloud.delete() raise BadRequestError(total_errors)
def add_machine(self, name, host='', ssh_user='******', ssh_port=22, ssh_key=None, os_type='unix', rdp_port=3389, fail_on_error=True): """Add machine to this dummy Cloud This is a special method that exists only on this Cloud subclass. """ old_machines = [m.as_dict() for m in self.cloud.ctl.compute.list_cached_machines()] # FIXME: Move ssh command to Machine controller once it is migrated. from mist.api.methods import ssh_command try: ssh_port = int(ssh_port) except (ValueError, TypeError): ssh_port = 22 try: rdp_port = int(rdp_port) except (ValueError, TypeError): rdp_port = 3389 if ssh_key: ssh_key = Key.objects.get(owner=self.cloud.owner, id=ssh_key, deleted=None) from mist.api.machines.models import Machine # Create and save machine entry to database. machine = Machine( cloud=self.cloud, name=name, machine_id=uuid.uuid4().hex, os_type=os_type, ssh_port=ssh_port, rdp_port=rdp_port, last_seen=datetime.datetime.utcnow() ) if host: # Sanitize inputs. host = sanitize_host(host) check_host(host) machine.hostname = host if is_private_subnet(socket.gethostbyname(host)): machine.private_ips = [host] else: machine.public_ips = [host] machine.save(write_concern={'w': 1, 'fsync': True}) # Attempt to connect. if os_type == 'unix' and ssh_key: if not ssh_user: ssh_user = '******' # Try to connect. If it works, it will create the association. try: if not host: raise BadRequestError("You have specified an SSH key but " "machine hostname is empty.") to_tunnel(self.cloud.owner, host) # May raise VPNTunnelError ssh_command( self.cloud.owner, self.cloud.id, machine.id, host, 'uptime', key_id=ssh_key.id, username=ssh_user, port=ssh_port ) except MachineUnauthorizedError as exc: if fail_on_error: machine.delete() raise CloudUnauthorizedError(exc) except ServiceUnavailableError as exc: if fail_on_error: machine.delete() raise MistError("Couldn't connect to host '%s'." % host) except: if fail_on_error: machine.delete() raise if amqp_owner_listening(self.cloud.owner.id): new_machines = self.cloud.ctl.compute.list_cached_machines() self.cloud.ctl.compute.produce_and_publish_patch( old_machines, new_machines) return machine
def _to_node(self, node): """ Convert the request into a node Node """ ATTRIBUTE_NAME_MAP = { 'dataCenterId': 'datacenter_id', 'dataCenterVersion': 'datacenter_version', 'serverId': 'node_id', 'serverName': 'node_name', 'cores': 'cores', 'ram': 'ram', 'internetAccess': 'internet_access', 'provisioningState': 'provisioning_state', 'virtualMachineState': 'virtual_machine_state', 'creationTime': 'creation_time', 'lastModificationTime': 'last_modification_time', 'osType': 'os_type', 'availabilityZone': 'availability_zone', 'cpuHotPlug': 'cpu_hotpluggable', 'ramHotPlug': 'memory_hotpluggable', 'nicHotPlug': 'nic_hotpluggable', 'discVirtioHotPlug': 'disc_virtio_hotplug', 'discVirtioHotUnPlug': 'disc_virtio_hotunplug' } extra = {} for attribute_name, extra_name in ATTRIBUTE_NAME_MAP.items(): elem = node.find(attribute_name) if ET.iselement(elem): value = elem.text else: value = None extra[extra_name] = value public_ips = [] private_ips = [] if ET.iselement(node.find('nics')): for nic in node.findall('.//nics'): n_elements = list(nic.findall('.//ips')) if len(n_elements) > 0: ip = n_elements[0].text if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) extra['provisioning_state'] = self.PROVISIONING_STATE.get( extra['provisioning_state'], NodeState.UNKNOWN) node_id = extra['node_id'] node_name = extra['node_name'] state = self.NODE_STATE_MAP.get(extra['virtual_machine_state'], NodeState.UNKNOWN) return Node(id=node_id, name=node_name, state=state, public_ips=public_ips, private_ips=private_ips, driver=self.connection.driver, extra=extra)
def _to_node(self, node): """ Convert the request into a node Node """ ATTRIBUTE_NAME_MAP = { 'dataCenterId': 'datacenter_id', 'dataCenterVersion': 'datacenter_version', 'serverId': 'node_id', 'serverName': 'node_name', 'cores': 'cores', 'ram': 'ram', 'internetAccess': 'internet_access', 'provisioningState': 'provisioning_state', 'virtualMachineState': 'virtual_machine_state', 'creationTime': 'creation_time', 'lastModificationTime': 'last_modification_time', 'osType': 'os_type', 'availabilityZone': 'availability_zone', 'cpuHotPlug': 'cpu_hotpluggable', 'ramHotPlug': 'memory_hotpluggable', 'nicHotPlug': 'nic_hotpluggable', 'discVirtioHotPlug': 'disc_virtio_hotplug', 'discVirtioHotUnPlug': 'disc_virtio_hotunplug' } extra = {} for attribute_name, extra_name in ATTRIBUTE_NAME_MAP.items(): elem = node.find(attribute_name) if ET.iselement(elem): value = elem.text else: value = None extra[extra_name] = value public_ips = [] private_ips = [] if ET.iselement(node.find('nics')): for nic in node.findall('.//nics'): n_elements = list(nic.findall('.//ips')) if len(n_elements) > 0: ip = n_elements[0].text if is_private_subnet(ip): private_ips.append(ip) else: public_ips.append(ip) extra['provisioning_state'] = self.PROVISIONING_STATE.get( extra['provisioning_state'], NodeState.UNKNOWN) node_id = extra['node_id'] node_name = extra['node_name'] state = self.NODE_STATE_MAP.get(extra['virtual_machine_state'], NodeState.UNKNOWN) return Node( id=node_id, name=node_name, state=state, public_ips=public_ips, private_ips=private_ips, driver=self.connection.driver, extra=extra)
def add_machine(self, host, ssh_user='******', ssh_port=22, ssh_key=None, **kwargs): try: ssh_port = int(ssh_port) except (ValueError, TypeError): ssh_port = 22 if not ssh_key: raise RequiredParameterMissingError('machine_key') try: ssh_key = Key.objects.get(owner=self.cloud.owner, id=ssh_key, deleted=None) except Key.DoesNotExist: raise NotFoundError("Key does not exist.") images_location = kwargs.get('images_location', '/var/lib/libvirt/images') extra = { 'images_location': images_location, 'tags': {'type': 'hypervisor'}, 'username': ssh_user } from mist.api.machines.models import Machine # Create and save machine entry to database. # first check if the host has already been added to the cloud try: machine = Machine.objects.get(cloud=self.cloud, machine_id=host.replace('.', '-')) machine.name = kwargs.get('name') or host machine.ssh_port = ssh_port machine.extra = extra machine.last_seen = datetime.datetime.utcnow() machine.missing_since = None except me.DoesNotExist: machine = Machine( cloud=self.cloud, name=kwargs.get('name') or host, hostname=host, machine_id=host.replace('.', '-'), ssh_port=ssh_port, extra=extra, state=NodeState.RUNNING, last_seen=datetime.datetime.utcnow(), ) # Sanitize inputs. host = sanitize_host(host) check_host(host) machine.hostname = host if is_private_subnet(socket.gethostbyname(host)): machine.private_ips = [host] else: machine.public_ips = [host] machine.save(write_concern={'w': 1, 'fsync': True}) # associate key and attempt to connect try: machine.ctl.associate_key(ssh_key, username=ssh_user, port=ssh_port) except MachineUnauthorizedError as exc: log.error("Could not connect to host %s." % host) machine.delete() raise CloudUnauthorizedError(exc) except ServiceUnavailableError as exc: log.error("Could not connect to host %s." % host) machine.delete() raise MistError("Couldn't connect to host '%s'." % host) if amqp_owner_listening(self.cloud.owner.id): old_machines = [] for cached_machine in \ self.cloud.ctl.compute.list_cached_machines(): # make sure that host just added becomes visible if cached_machine.id != machine.id: old_machines.append(cached_machine) old_machines = [m.as_dict() for m in old_machines] new_machines = self.cloud.ctl.compute.list_machines() self.cloud.ctl.compute.produce_and_publish_patch( old_machines, new_machines) return machine