def check_stderr( self, command, verbose=False, timeout=None, error_info=None, raise_on_err=True, **kwargs): """Execute command expecting return code 0 and empty STDERR :type command: str :type verbose: bool :type timeout: int :type error_info: str :type raise_on_err: bool :rtype: ExecResult :raises: DevopsCalledProcessError """ ret = self.check_call( command, verbose, timeout=timeout, error_info=error_info, raise_on_err=raise_on_err, **kwargs) if ret['stderr']: message = ( log_templates.CMD_UNEXPECTED_STDERR.format( append=error_info + '\n' if error_info else '', cmd=command, code=ret['exit_code'], )) logger.error(message) if raise_on_err: raise error.DevopsCalledProcessError( command, ret['exit_code'], stdout=ret['stdout_brief'], stderr=ret['stdout_brief']) return ret
def erase_snapshot(self, name): if self.has_snapshot(name): snapshot = self._get_snapshot(name) snap_type = snapshot.get_type if snap_type == 'external': # EXTERNAL SNAPSHOT DELETE if snapshot.children_num > 0: logger.error("Cannot delete external snapshots " "with children") return self.destroy() # Update domain to snapshot state xml_domain = snapshot._xml_tree.find('domain') self.driver.conn.defineXML(ET.tostring(xml_domain)) snapshot.delete_snapshot_files() snapshot.delete(2) for disk in self.disk_devices: if disk.device == 'disk': snap_disk = disk.volume # update disk on node disk.volume = disk.volume.backing_store disk.save() snap_disk.remove() else: # ORIGINAL DELETE snapshot.delete(0)
def node_delete_snapshot(self, node, name=None): """Delete snapshot :type node: Node :type name: String """ domain = self.conn.lookupByUUIDString(node.uuid) snapshot = self._get_snapshot(domain, name) snap_type = Snapshot(snapshot).get_type if snap_type == 'external': if snapshot.numChildren() > 0: logger.error("Cannot delete external snapshots with children") return if domain.isActive(): domain.destroy() # Update domain to snapshot state xml_tree = ET.fromstring(snapshot.getXMLDesc()) xml_domain = xml_tree.find('domain') self.conn.defineXML(ET.tostring(xml_domain)) self._delete_snapshot_files(snapshot) snapshot.delete(2) else: snapshot.delete(0)
def check_stderr( cls, command, verbose=False, timeout=None, error_info=None, raise_on_err=True, **kwargs): """Execute command expecting return code 0 and empty STDERR Timeout limitation: read tick is 100 ms. :type command: str :type verbose: bool :type timeout: int :type error_info: str :type raise_on_err: bool :rtype: ExecResult :raises: DevopsCalledProcessError """ ret = cls.check_call( command, verbose, timeout=timeout, error_info=error_info, raise_on_err=raise_on_err, **kwargs) if ret['stderr']: message = ( log_templates.CMD_UNEXPECTED_STDERR.format( append=error_info + '\n' if error_info else '', cmd=command, code=ret['exit_code'] )) logger.error(message) if raise_on_err: raise error.DevopsCalledProcessError( command, ret['exit_code'], stdout=ret['stdout_brief'], stderr=ret['stderr_brief']) return ret
def _nwfilter(self): """Returns NWFilter object""" try: return self.driver.conn.nwfilterLookupByName(self.network_name) except libvirt.libvirtError: logger.error("NWFilter not found by name: {}".format( self.network_name)) return None
def check_call( cls, command, verbose=False, timeout=None, error_info=None, expected=None, raise_on_err=True, **kwargs): """Execute command and check for return code Timeout limitation: read tick is 100 ms. :type command: str :type verbose: bool :type timeout: int :type error_info: str :type expected: list :type raise_on_err: bool :rtype: ExecResult :raises: DevopsCalledProcessError """ if expected is None: expected = [proc_enums.ExitCodes.EX_OK] else: expected = [ proc_enums.ExitCodes(code) if ( isinstance(code, int) and code in proc_enums.ExitCodes.__members__.values()) else code for code in expected ] ret = cls.execute(command, verbose, timeout, **kwargs) if ret['exit_code'] not in expected: message = ( "{append}Command '{cmd!r}' returned exit code {code!s} while " "expected {expected!s}\n" "\tSTDOUT:\n" "{stdout}" "\n\tSTDERR:\n" "{stderr}".format( append=error_info + '\n' if error_info else '', cmd=command, code=ret['exit_code'], expected=expected, stdout=ret['stdout_str'], stderr=ret['stderr_str'] )) logger.error(message) if raise_on_err: raise error.DevopsCalledProcessError( command, ret['exit_code'], expected=expected, stdout=ret['stdout_str'], stderr=ret['stderr_str']) return ret
def create_environment(cls, full_config): """Create a new environment using full_config object :param full_config: object that describes all the parameters of created environment :rtype: Environment """ config = full_config['template']['devops_settings'] environment = cls.create(config['env_name']) try: # create groups and drivers groups = config['groups'] environment.add_groups(groups) # create address pools address_pools = config['address_pools'] environment.add_address_pools(address_pools) # process group items for group_data in groups: group = environment.get_group(name=group_data['name']) # add l2_network_devices group.add_l2_network_devices( group_data.get('l2_network_devices', {})) # add network_pools group.add_network_pools( group_data.get('network_pools', {})) # Connect nodes to already created networks for group_data in groups: group = environment.get_group(name=group_data['name']) # add group volumes group.add_volumes( group_data.get('group_volumes', [])) # add nodes group.add_nodes( group_data.get('nodes', [])) except Exception: logger.error("Creation of the environment '{0}' failed" .format(config['env_name'])) environment.erase() raise return environment
def is_active(self, timeout=5): """Check if node is active :rtype : Boolean """ try: with helpers.RunLimit(timeout): states = self.driver.conn.node.states(self.uuid) return (states['provision_state'] == 'active' and states['power_state'] == 'power on') except error.TimeoutError: logger.error("Ironic API is not responded for {0}sec, assuming " "that node {1} is absent".format(timeout, self.name)) return False
def exists(self, timeout=5): """Check if node exists :rtype : Boolean """ try: with helpers.RunLimit(timeout): return any([ self.uuid == node.uuid for node in self.driver.conn.node.list() ]) except error.TimeoutError: logger.error("Ironic API is not responded for {0}sec, assuming " "that node {1} is absent".format(timeout, self.name)) return False
def wait_pass(raising_predicate, expected=Exception, interval=5, timeout=60, timeout_msg="Waiting timed out", predicate_args=None, predicate_kwargs=None): """Wait for successful return from predicate ignoring expected exception Options: :param interval: - seconds between checks. :param timeout: - raise TimeoutError if predicate still throwing expected exception after this amount of seconds. :param timeout_msg: - text of the TimeoutError :param predicate_args: - positional arguments for given predicate wrapped in list or tuple :param predicate_kwargs: - dict with named arguments for the predicate :param expected_exc: Exception that can be ignored while waiting (its possible to pass several using list/tuple """ predicate_args = predicate_args or [] predicate_kwargs = predicate_kwargs or {} _check_wait_args(raising_predicate, predicate_args, predicate_kwargs, interval, timeout) msg = ("{msg}\nWaited for pass {cmd}: {spent} seconds." "".format(msg=timeout_msg, cmd=repr(raising_predicate), spent="{spent:0.3f}")) start_time = time.time() with RunLimit(timeout, msg): while True: try: result = raising_predicate(*predicate_args, **predicate_kwargs) logger.debug( "wait_pass() completed with result='{0}'".format(result)) return result except expected as e: if start_time + timeout < time.time(): err_msg = msg.format(spent=time.time() - start_time) logger.error(err_msg) raise error.TimeoutError(err_msg) logger.debug("Got expected exception {!r}, continue".format(e)) time.sleep(interval)
def create_environment(cls, full_config): """Create a new environment using full_config object :param full_config: object that describes all the parameters of created environment :rtype: Environment """ config = full_config['template']['devops_settings'] environment = cls.create(config['env_name']) try: # create groups and drivers groups = config['groups'] environment.add_groups(groups) # create address pools address_pools = config['address_pools'] environment.add_address_pools(address_pools) # process group items for group_data in groups: group = environment.get_group(name=group_data['name']) # add l2_network_devices group.add_l2_network_devices( group_data.get('l2_network_devices', {})) # add network_pools group.add_network_pools(group_data.get('network_pools', {})) # Connect nodes to already created networks for group_data in groups: group = environment.get_group(name=group_data['name']) # add group volumes group.add_volumes(group_data.get('group_volumes', [])) # add nodes group.add_nodes(group_data.get('nodes', [])) except Exception: logger.error("Creation of the environment '{0}' failed".format( config['env_name'])) environment.erase() raise return environment
def check_call( cls, command, verbose=False, timeout=None, error_info=None, expected=None, raise_on_err=True, **kwargs): """Execute command and check for return code Timeout limitation: read tick is 100 ms. :type command: str :type verbose: bool :type timeout: int :type error_info: str :type expected: list :type raise_on_err: bool :rtype: ExecResult :raises: DevopsCalledProcessError """ if expected is None: expected = [proc_enums.ExitCodes.EX_OK] else: expected = [ proc_enums.ExitCodes(code) if ( isinstance(code, int) and code in proc_enums.ExitCodes.__members__.values()) else code for code in expected ] ret = cls.execute(command, verbose, timeout, **kwargs) if ret['exit_code'] not in expected: message = ( log_templates.CMD_UNEXPECTED_EXIT_CODE.format( append=error_info + '\n' if error_info else '', cmd=command, code=ret['exit_code'], expected=expected )) logger.error(message) if raise_on_err: raise error.DevopsCalledProcessError( command, ret['exit_code'], expected=expected, stdout=ret['stdout_brief'], stderr=ret['stderr_brief']) return ret
def check_call(cls, command, verbose=False, timeout=None, error_info=None, expected=None, raise_on_err=True, **kwargs): """Execute command and check for return code Timeout limitation: read tick is 100 ms. :type command: str :type verbose: bool :type timeout: int :type error_info: str :type expected: list :type raise_on_err: bool :rtype: ExecResult :raises: DevopsCalledProcessError """ if expected is None: expected = [proc_enums.ExitCodes.EX_OK] else: expected = [ proc_enums.ExitCodes(code) if (isinstance(code, int) and code in proc_enums.ExitCodes.__members__.values()) else code for code in expected ] ret = cls.execute(command, verbose, timeout, **kwargs) if ret['exit_code'] not in expected: message = (log_templates.CMD_UNEXPECTED_EXIT_CODE.format( append=error_info + '\n' if error_info else '', cmd=command, code=ret['exit_code'], expected=expected)) logger.error(message) if raise_on_err: raise error.DevopsCalledProcessError( command, ret['exit_code'], expected=expected, stdout=ret['stdout_brief'], stderr=ret['stderr_brief']) return ret
def wait_pass(raising_predicate, expected=Exception, interval=5, timeout=60, timeout_msg="Waiting timed out", predicate_args=None, predicate_kwargs=None): """Wait for successful return from predicate ignoring expected exception Options: :param interval: - seconds between checks. :param timeout: - raise TimeoutError if predicate still throwing expected exception after this amount of seconds. :param timeout_msg: - text of the TimeoutError :param predicate_args: - positional arguments for given predicate wrapped in list or tuple :param predicate_kwargs: - dict with named arguments for the predicate :param expected_exc: Exception that can be ignored while waiting (its possible to pass several using list/tuple """ predicate_args = predicate_args or [] predicate_kwargs = predicate_kwargs or {} _check_wait_args(raising_predicate, predicate_args, predicate_kwargs, interval, timeout) msg = ( "{msg}\nWaited for pass {cmd}: {spent} seconds." "".format( msg=timeout_msg, cmd=repr(raising_predicate), spent="{spent:0.3f}" )) start_time = time.time() with RunLimit(timeout, msg): while True: try: result = raising_predicate(*predicate_args, **predicate_kwargs) logger.debug("wait_pass() completed with result='{0}'" .format(result)) return result except expected as e: if start_time + timeout < time.time(): err_msg = msg.format(spent=time.time() - start_time) logger.error(err_msg) raise error.TimeoutError(err_msg) logger.debug("Got expected exception {!r}, continue".format(e)) time.sleep(interval)
def erase_snapshot(self, name): if self.has_snapshot(name): snapshot = self.driver.node_get_snapshot(self, name) if snapshot.get_type == "external": if snapshot.children_num == 0: self.driver.node_delete_snapshot(node=self, name=name) for disk in self.disk_devices: if disk.device == "disk": snap_disk = disk.volume # update disk on node disk.volume = disk.volume.backing_store disk.save() snap_disk.remove() else: logger.error("Cannot delete external snapshots with children") else: self.driver.node_delete_snapshot(node=self, name=name)
def wait(predicate, interval=5, timeout=60, timeout_msg="Waiting timed out", predicate_args=None, predicate_kwargs=None): """Wait until predicate will become True. Options: :param interval: - seconds between checks. :param timeout: - raise TimeoutError if predicate won't become True after this amount of seconds. :param timeout_msg: - text of the TimeoutError :param predicate_args: - positional arguments for given predicate wrapped in list or tuple :param predicate_kwargs: - dict with named arguments for the predicate """ predicate_args = predicate_args or [] predicate_kwargs = predicate_kwargs or {} _check_wait_args(predicate, predicate_args, predicate_kwargs, interval, timeout) msg = ("{msg}\nWaited for pass {cmd}: {spent} seconds." "".format(msg=timeout_msg, cmd=repr(predicate), spent="{spent:0.3f}")) start_time = time.time() with RunLimit(timeout, msg): while True: result = predicate(*predicate_args, **predicate_kwargs) if result: logger.debug( "wait() completed with result='{0}'".format(result)) return result if start_time + timeout < time.time(): err_msg = msg.format(spent=time.time() - start_time) logger.error(err_msg) raise error.TimeoutError(err_msg) time.sleep(interval)
def wait(predicate, interval=5, timeout=60, timeout_msg="Waiting timed out", predicate_args=None, predicate_kwargs=None): """Wait until predicate will become True. Options: :param interval: - seconds between checks. :param timeout: - raise TimeoutError if predicate won't become True after this amount of seconds. :param timeout_msg: - text of the TimeoutError :param predicate_args: - positional arguments for given predicate wrapped in list or tuple :param predicate_kwargs: - dict with named arguments for the predicate """ predicate_args = predicate_args or [] predicate_kwargs = predicate_kwargs or {} _check_wait_args(predicate, predicate_args, predicate_kwargs, interval, timeout) msg = ( "{msg}\nWaited for pass {cmd}: {spent} seconds." "".format( msg=timeout_msg, cmd=repr(predicate), spent="{spent:0.3f}" )) start_time = time.time() with RunLimit(timeout, msg): while True: result = predicate(*predicate_args, **predicate_kwargs) if result: logger.debug("wait() completed with result='{0}'" .format(result)) return result if start_time + timeout < time.time(): err_msg = msg.format(spent=time.time() - start_time) logger.error(err_msg) raise error.TimeoutError(err_msg) time.sleep(interval)
def check_stderr(self, command, verbose=False, timeout=None, error_info=None, raise_on_err=True, **kwargs): """Execute command expecting return code 0 and empty STDERR :type command: str :type verbose: bool :type timeout: int :type error_info: str :type raise_on_err: bool :rtype: ExecResult :raises: DevopsCalledProcessError """ ret = self.check_call(command, verbose, timeout=timeout, error_info=error_info, raise_on_err=raise_on_err, **kwargs) if ret['stderr']: message = ("{append}Command '{cmd!r}' STDERR while not expected\n" "\texit code: {code!s}\n" "\tSTDOUT:\n" "{stdout}" "\n\tSTDERR:\n" "{stderr}".format(append=error_info + '\n' if error_info else '', cmd=command, code=ret['exit_code'], stdout=ret['stdout_str'], stderr=ret['stderr_str'])) logger.error(message) if raise_on_err: raise error.DevopsCalledProcessError(command, ret['exit_code'], stdout=ret['stdout_str'], stderr=ret['stderr_str']) return ret
def __deserialize(self, fmt): """Deserialize stdout as data format :type fmt: str :rtype: object :raises: DevopsError """ try: if fmt == 'json': return json.loads(self.stdout_str, encoding='utf-8') elif fmt == 'yaml': return yaml.safe_load(self.stdout_str) except BaseException: tmpl = ("'{cmd}' stdout is not valid {fmt}:\n" '{{stdout!r}}\n'.format(cmd=self.cmd, fmt=fmt)) logger.exception(tmpl.format(stdout=self.stdout_str)) raise error.DevopsError(tmpl.format(stdout=self.stdout_brief)) msg = '{fmt} deserialize target is not implemented'.format(fmt=fmt) logger.error(msg) raise error.DevopsNotImplementedError(msg)
def connect(self): logger.debug( "Connect to '{0}:{1}' as '{2}:{3}'".format( self.host, self.port, self.username, self.password)) for private_key in self.private_keys: try: self._ssh.connect( self.host, port=self.port, username=self.username, password=self.password, pkey=private_key) self.__actual_pkey = private_key return except paramiko.AuthenticationException: continue if self.private_keys: logger.error("Authentication with keys failed") self.__actual_pkey = None self._ssh.connect( self.host, port=self.port, username=self.username, password=self.password)
def wait_pass(raising_predicate, expected=Exception, interval=5, timeout=None, timeout_msg="Waiting timed out"): """Wait for successful return from predicate or expected exception""" if not callable(raising_predicate): raise TypeError('Not callable raising_predicate has been posted') start_time = time.time() while True: try: return raising_predicate() except expected: if timeout and start_time + timeout < time.time(): msg = ("{msg}\nWaited for pass {cmd}: {spent:0.3f} seconds." "".format(msg=timeout_msg, cmd=raising_predicate.func_name, spent=time.time() - start_time)) logger.error(msg) raise time.sleep(interval)
def node_create_snapshot(self, node, name=None, description=None, disk_only=False, external=False): """Create snapshot :type description: String :type name: String :type node: Node :rtype : None """ if self.node_snapshot_exists(node, name): logger.error("Snapshot with name {0} already exists".format(name)) return domain = self.conn.lookupByUUIDString(node.uuid) # If domain has snapshots we must check their type snap_list = self.node_get_snapshots(node) if len(snap_list) > 0: snap_type = snap_list[0].get_type if external and snap_type == 'internal': logger.error( "Cannot create external snapshot when internal exists") return if not external and snap_type == 'external': logger.error( "Cannot create internal snapshot when external exists") return logger.info(domain.state(0)) xml = self.xml_builder.build_snapshot_xml( name, description, node, disk_only, external, settings.SNAPSHOTS_EXTERNAL_DIR) logger.info(xml) if external: # Check whether we have directory for snapshots, if not # create it if not os.path.exists(settings.SNAPSHOTS_EXTERNAL_DIR): os.makedirs(settings.SNAPSHOTS_EXTERNAL_DIR) if domain.isActive() and not disk_only: domain.snapshotCreateXML( xml, libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT) else: domain.snapshotCreateXML( xml, libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY | libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT) self.node_set_snapshot_current(node, name) else: domain.snapshotCreateXML(xml) logger.info(domain.state(0))
def _libvirt_network(self): try: return self.driver.conn.networkLookupByUUIDString(self.uuid) except libvirt.libvirtError: logger.error("Network not found by UUID: {}".format(self.uuid)) return None
def _libvirt_node(self): try: return self.driver.conn.lookupByUUIDString(self.uuid) except libvirt.libvirtError: logger.error("Domain not found by UUID: {}".format(self.uuid)) return None
def _libvirt_volume(self): try: return self.driver.conn.storageVolLookupByKey(self.uuid) except libvirt.libvirtError: logger.error("Volume not found by UUID: {}".format(self.uuid)) return None
def _nwfilter(self): try: return self.driver.conn.nwfilterLookupByName(self.nwfilter_name) except libvirt.libvirtError: logger.error("NWFilter not found by name: {}".format( self.nwfilter_name))