def _calculate_numa(numa_nodes, vcpu, memory, name): numa = [] if numa_nodes: cpus_per_numa = vcpu // numa_nodes if cpus_per_numa * numa_nodes != vcpu: raise error.DevopsError( "NUMA_NODES={0} is not a multiple of the number of CPU={1}" " for node '{2}'".format(numa_nodes, vcpu, name)) memory_per_numa = memory // numa_nodes if memory_per_numa * numa_nodes != memory: raise error.DevopsError( "NUMA_NODES={0} is not a multiple of the amount of " "MEMORY={1} for node '{2}'".format(numa_nodes, memory, name)) for x in range(numa_nodes): # List of cpu IDs for the numa node # pylint: disable=range-builtin-not-iterating cpus = range(x * cpus_per_numa, (x + 1) * cpus_per_numa) # pylint: enable=range-builtin-not-iterating cell = { 'cpus': ','.join(map(str, cpus)), 'memory': memory_per_numa, } numa.append(cell) return numa
def _safe_create_network(cls, name, pool, environment, **params): for ip_network in pool: if cls.objects.filter(net=str(ip_network)).exists(): continue new_params = deepcopy(params) new_params['net'] = ip_network try: with transaction.atomic(): return cls.objects.create( environment=environment, name=name, **new_params ) except IntegrityError as e: logger.debug(e) if 'name' in str(e): raise error.DevopsError( 'AddressPool with name "{}" already exists' ''.format(name)) continue raise error.DevopsError( "There is no network pool available for creating " "address pool {}".format(name))
def next_ip(self): """Get next IP address from the address pool If 'dhcp' ip_range specified for the address pool, then the IP addresses will be taken from this pool. Else, IP addresses will be taken from the range [ x.x.x.x + 2 : x.x.x.x - 2 ] """ range_start = netaddr.IPAddress( self.ip_range_start('dhcp') or self.ip_network[2]) range_end = netaddr.IPAddress( self.ip_range_end('dhcp') or self.ip_network[-2]) for ip in self.ip_network.iter_hosts(): # if ip < self.ip_pool_start or ip > self.ip_pool_end: # Skip net, gw and broadcast addresses in the address pool if ip < range_start or ip > range_end: continue already_exists = Address.objects.filter( interface__l2_network_device__address_pool=self, ip_address=str(ip)).exists() if already_exists: continue return ip raise error.DevopsError( "No more free addresses in the address pool {0}" " with CIDR {1}".format(self.name, self.net))
def test_main_devops_error(self): err = error.DevopsError('my error') self.shell_inst.execute.side_effect = err shell.main(['start']) self.shell_mock.assert_called_once_with(['start']) self.shell_inst.execute.assert_called_once_with() self.sys_mock.exit.assert_called_once_with('Error: my error')
def yaml_include(loader, node): file_name = os.path.join(dirname, node.value) if not os.path.isfile(file_name): raise error.DevopsError( "Cannot load the environment template {0} : include file {1} " "doesn't exist.".format(dirname, file_name)) inputfile = utils.render_template(file_name) return yaml.load(inputfile, TemplateLoader)
def yaml_include(loader, node): file_name = os.path.join(os.path.dirname(loader.name), node.value) if not os.path.isfile(file_name): raise error.DevopsError( "Cannot load the environment template {0} : include file {1} " "doesn't exist.".format(loader.name, file_name)) with open(file_name) as inputfile: return yaml.load(inputfile, TemplateLoader)
def yaml_template_load(config_file): """Temporary moved from fuel_devops to use jinja2""" dirname = os.path.dirname(config_file) class TemplateLoader(yaml.Loader): pass def yaml_include(loader, node): file_name = os.path.join(dirname, node.value) if not os.path.isfile(file_name): raise error.DevopsError( "Cannot load the environment template {0} : include file {1} " "doesn't exist.".format(dirname, file_name)) inputfile = utils.render_template(file_name) return yaml.load(inputfile, TemplateLoader) def yaml_get_env_variable(loader, node): if not node.value.strip(): raise error.DevopsError( "Environment variable is required after {tag} in " "{filename}".format(tag=node.tag, filename=loader.name)) node_value = node.value.split(',', 1) # Get the name of environment variable env_variable = node_value[0].strip() # Get the default value for environment variable if it exists in config if len(node_value) > 1: default_val = node_value[1].strip() else: default_val = None value = os.environ.get(env_variable, default_val) if value is None: raise error.DevopsError( "Environment variable {var} is not set from shell" " environment! No default value provided in file " "{filename}".format(var=env_variable, filename=loader.name)) return yaml.load(value, TemplateLoader) def construct_mapping(loader, node): loader.flatten_mapping(node) return collections.OrderedDict(loader.construct_pairs(node)) if not os.path.isfile(config_file): raise error.DevopsError( "Cannot load the environment template {0} : file " "doesn't exist.".format(config_file)) TemplateLoader.add_constructor("!include", yaml_include) TemplateLoader.add_constructor("!os_env", yaml_get_env_variable) TemplateLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) f = utils.render_template(config_file) return yaml.load(f, TemplateLoader)
def create(cls, name): """Create Environment instance with given name. :rtype: devops.models.Environment """ try: return cls.objects.create(name=name) except IntegrityError: raise error.DevopsError( 'Environment with name {!r} already exists. ' 'Please, set another environment name.' ''.format(name))
def snapshot(self, name=None, description=None, force=False): if name is None: name = str(int(time.time())) if self.has_snapshot(name) and not force: raise error.DevopsError( 'Snapshot with name {0} already exists.'.format( self.params.snapshot_name)) for nod in self.get_nodes(): nod.snapshot(name=name, description=description, force=force, external=settings.SNAPSHOTS_EXTERNAL)
def __setitem__(self, key, value): rw = ['stdout', 'stderr', 'exit_code'] if key in rw: setattr(self, key, value) return if key in deprecated_aliases: logger.warning( '{key} is read-only and calculated automatically'.format( key=key)) return if key in dir(self): raise error.DevopsError('{key} is read-only!'.format(key=key)) raise IndexError('{key} not found in {dir}'.format(key=key, dir=rw))
def yaml_get_env_variable(loader, node): if not node.value.strip(): raise error.DevopsError( "Environment variable is required after {tag} in " "{filename}".format(tag=node.tag, filename=loader.name)) node_value = node.value.split(',', 1) # Get the name of environment variable env_variable = node_value[0].strip() # Get the default value for environment variable if it exists in config if len(node_value) > 1: default_val = node_value[1].strip() else: default_val = None value = os.environ.get(env_variable, default_val) if value is None: raise error.DevopsError( "Environment variable {var} is not set from shell" " environment! No default value provided in file " "{filename}".format(var=env_variable, filename=loader.name)) return yaml.load(value, TemplateLoader)
def find_node_ip(self, node_name): node = self.get_node(name=node_name) for interface in node.interfaces: ip = interface.address_set.get(interface=interface).ip_address try: helpers.wait_tcp( host=ip, port=node.ssh_port, timeout=10, timeout_msg=("Node {ip} is not accessible by SSH." .format(ip=ip))) return ip except error.TimeoutError: pass raise error.DevopsError("Cannot find SSH endpoint for node {0}" .format(node.name))
def get_slave_ip_by_mac(self, mac): nodes = self.get_nodes_json() def poor_mac(mac_addr): return [ m.lower() for m in mac_addr if m.lower() in '01234546789abcdef' ] for node in nodes: for interface in node['meta']['interfaces']: if poor_mac(interface['mac']) == poor_mac(mac): logger.debug('For mac {0} found ip {1}'.format( mac, node['ip'])) return node['ip'] raise error.DevopsError( 'There is no match between MAC {0} and Nailgun MACs'.format(mac))
def ip_range_set(self, range_name, ip_range_start, ip_range_end): """Set IP range in the address pool :param range_name: str, name of the range :param ip_range_start: str, first IP of the range :param ip_range_end: str, last IP of the range :rtype: None or exception DevopsError If range_name already exists, then DevopsError raises. """ if range_name in self.ip_ranges: raise error.DevopsError( "Setting IP range '{0}' for address pool '{1}' failed: range " "already exists".format(range_name, self.name)) self.ip_ranges[range_name] = (ip_range_start, ip_range_end) self.save()
def get_ntp(remote, node_name): # Detect how NTPD is managed - by init script or by pacemaker. pcs_cmd = "ps -C pacemakerd && crm_resource --resource p_ntp --locate" systemd_cmd = "systemctl list-unit-files| grep ntpd" chronyd_cmd = "systemctl is-active chronyd" initd_cmd = "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable" if remote.execute(pcs_cmd)['exit_code'] == 0: # Pacemaker service found return NtpPacemaker(remote, node_name) elif remote.execute(systemd_cmd)['exit_code'] == 0: return NtpSystemd(remote, node_name) elif remote.execute(chronyd_cmd)['exit_code'] == 0: return NtpChronyd(remote, node_name) elif len(remote.execute(initd_cmd)['stdout']): return NtpInitscript(remote, node_name) else: raise error.DevopsError( 'No suitable NTP service found on node {!r}' ''.format(node_name))
def __deserialize(self, fmt): """Deserialize stdout as data format :type fmt: str :rtype: object :raises: DevopsError """ try: if fmt == 'json': return json.loads(self.stdout_str, encoding='utf-8') elif fmt == 'yaml': return yaml.safe_load(self.stdout_str) except BaseException: tmpl = ("'{cmd}' stdout is not valid {fmt}:\n" '{{stdout!r}}\n'.format(cmd=self.cmd, fmt=fmt)) logger.exception(tmpl.format(stdout=self.stdout_str)) raise error.DevopsError(tmpl.format(stdout=self.stdout_brief)) msg = '{fmt} deserialize target is not implemented'.format(fmt=fmt) logger.error(msg) raise error.DevopsNotImplementedError(msg)
def snapshot(self, name=None, description=None, force=False, suspend=True): """Snapshot the environment :param name: name of the snapshot. Current timestamp, if name is None :param description: any string that will be placed to the 'description' section in the snapshot XML :param force: If True - overwrite the existing snapshot. Default: False :param suspend: suspend environment before snapshot if True (default) """ if name is None: name = str(int(time.time())) if self.has_snapshot(name) and not force: raise error.DevopsError( 'Snapshot with name "{0}" already exists.'.format(name)) if suspend: for nod in self.get_nodes(): nod.suspend() for nod in self.get_nodes(): nod.snapshot(name=name, description=description, force=force, external=settings.SNAPSHOTS_EXTERNAL)
def get_admin(self): if self.has_admin(): return self._env.get_node(name='admin') raise error.DevopsError('Environment {!r} has no admin node'.format( self._env.name))
def _start_setup(self): if self.node.kernel_cmd is None: raise error.DevopsError('kernel_cmd is None') self.node.start() self.send_kernel_keys(self.node.kernel_cmd)
def __run_ipmi(self, cmd): """Run command through ipmitool :param cmd: list - ipmi command :return: object - data if successful, None otherwise """ try: ipmitool_cmd = subprocess.check_output(["which", "ipmitool"]).strip() if not ipmitool_cmd: raise error.DevopsError('ipmitool not found') except Exception: raise error.DevopsError( 'Node:{} ipmi_host:{} ipmitool has not installed. ' 'No chance to go over'.format(self.nodename, self.remote_host)) ipmi_cmd_dict = { 'ipmitool': ipmitool_cmd, 'remote_lan_interface': self.remote_lan_interface, 'remote_host': self.remote_host, 'remote_port': self.remote_port, 'user': self.user, 'password': self.password, 'level': self.level, 'cmd': cmd } lerrors = [(key, value) for (key, value) in ipmi_cmd_dict.items() if value is None] if lerrors: raise error.DevopsError('Node:{} ipmi_host:{} ' 'ipmitool arguments ' 'key={}, value={}' 'are not valid'.format( self.nodename, self.remote_host, lerrors[0], lerrors[1])) ipmi_cmd = [ ipmitool_cmd, '-I', self.remote_lan_interface, '-H', self.remote_host, '-U', self.user, '-P', self.password, '-L', self.level, '-p', str(self.remote_port) ] if isinstance(cmd, list): ipmi_cmd.extend(cmd) else: # workaround for commands like "stats get" args = " ".join(cmd).split(" ") ipmi_cmd.extend(args) args = " ".join(ipmi_cmd).split(" ") try: # let's break down it again and prepare args pipe = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = pipe.communicate() code = pipe.returncode except Exception as message: logger.debug('{}'.format(message)) raise error.DevopsError('Node:{} Remote:{} ipmitool command [{}] ' 'has failed with' ' the exception: {}'.format( self.nodename, self.remote_host, cmd, message)) if (out is None) or code != 0: logger.debug("rcode ={} or err ={}".format(code, err)) raise error.DevopsError('Node:{} Remote:{} ipmitool command [{}] ' 'has failed with the message: {}'.format( self.nodename, self.remote_host, cmd, err)) return out
def get_free_port(): for port in range(32000, 32100): if not tcp_ping('localhost', port): return port raise error.DevopsError('No free ports available')