def check_stderr(
self,
command, verbose=False, timeout=None,
error_info=None,
raise_on_err=True, **kwargs):
"""Execute command expecting return code 0 and empty STDERR
:type command: str
:type verbose: bool
:type timeout: int
:type error_info: str
:type raise_on_err: bool
:rtype: ExecResult
:raises: DevopsCalledProcessError
"""
ret = self.check_call(
command, verbose, timeout=timeout,
error_info=error_info, raise_on_err=raise_on_err, **kwargs)
if ret['stderr']:
message = (
log_templates.CMD_UNEXPECTED_STDERR.format(
append=error_info + '\n' if error_info else '',
cmd=command,
code=ret['exit_code'],
))
logger.error(message)
if raise_on_err:
raise error.DevopsCalledProcessError(
command,
ret['exit_code'],
stdout=ret['stdout_brief'],
stderr=ret['stdout_brief'])
return ret
def erase_snapshot(self, name):
if self.has_snapshot(name):
snapshot = self._get_snapshot(name)
snap_type = snapshot.get_type
if snap_type == 'external':
# EXTERNAL SNAPSHOT DELETE
if snapshot.children_num > 0:
logger.error("Cannot delete external snapshots "
"with children")
return
self.destroy()
# Update domain to snapshot state
xml_domain = snapshot._xml_tree.find('domain')
self.driver.conn.defineXML(ET.tostring(xml_domain))
snapshot.delete_snapshot_files()
snapshot.delete(2)
for disk in self.disk_devices:
if disk.device == 'disk':
snap_disk = disk.volume
# update disk on node
disk.volume = disk.volume.backing_store
disk.save()
snap_disk.remove()
else:
# ORIGINAL DELETE
snapshot.delete(0)
def node_delete_snapshot(self, node, name=None):
"""Delete snapshot
:type node: Node
:type name: String
"""
domain = self.conn.lookupByUUIDString(node.uuid)
snapshot = self._get_snapshot(domain, name)
snap_type = Snapshot(snapshot).get_type
if snap_type == 'external':
if snapshot.numChildren() > 0:
logger.error("Cannot delete external snapshots with children")
return
if domain.isActive():
domain.destroy()
# Update domain to snapshot state
xml_tree = ET.fromstring(snapshot.getXMLDesc())
xml_domain = xml_tree.find('domain')
self.conn.defineXML(ET.tostring(xml_domain))
self._delete_snapshot_files(snapshot)
snapshot.delete(2)
else:
snapshot.delete(0)
def check_stderr(
cls,
command, verbose=False, timeout=None,
error_info=None,
raise_on_err=True, **kwargs):
"""Execute command expecting return code 0 and empty STDERR
Timeout limitation: read tick is 100 ms.
:type command: str
:type verbose: bool
:type timeout: int
:type error_info: str
:type raise_on_err: bool
:rtype: ExecResult
:raises: DevopsCalledProcessError
"""
ret = cls.check_call(
command, verbose, timeout=timeout,
error_info=error_info, raise_on_err=raise_on_err, **kwargs)
if ret['stderr']:
message = (
log_templates.CMD_UNEXPECTED_STDERR.format(
append=error_info + '\n' if error_info else '',
cmd=command,
code=ret['exit_code']
))
logger.error(message)
if raise_on_err:
raise error.DevopsCalledProcessError(
command, ret['exit_code'],
stdout=ret['stdout_brief'],
stderr=ret['stderr_brief'])
return ret
def erase_snapshot(self, name):
if self.has_snapshot(name):
snapshot = self._get_snapshot(name)
snap_type = snapshot.get_type
if snap_type == 'external':
# EXTERNAL SNAPSHOT DELETE
if snapshot.children_num > 0:
logger.error("Cannot delete external snapshots "
"with children")
return
self.destroy()
# Update domain to snapshot state
xml_domain = snapshot._xml_tree.find('domain')
self.driver.conn.defineXML(ET.tostring(xml_domain))
snapshot.delete_snapshot_files()
snapshot.delete(2)
for disk in self.disk_devices:
if disk.device == 'disk':
snap_disk = disk.volume
# update disk on node
disk.volume = disk.volume.backing_store
disk.save()
snap_disk.remove()
else:
# ORIGINAL DELETE
snapshot.delete(0)
def _nwfilter(self):
"""Returns NWFilter object"""
try:
return self.driver.conn.nwfilterLookupByName(self.network_name)
except libvirt.libvirtError:
logger.error("NWFilter not found by name: {}".format(
self.network_name))
return None
def _nwfilter(self):
"""Returns NWFilter object"""
try:
return self.driver.conn.nwfilterLookupByName(self.network_name)
except libvirt.libvirtError:
logger.error("NWFilter not found by name: {}".format(
self.network_name))
return None
def check_call(
cls,
command, verbose=False, timeout=None,
error_info=None,
expected=None, raise_on_err=True, **kwargs):
"""Execute command and check for return code
Timeout limitation: read tick is 100 ms.
:type command: str
:type verbose: bool
:type timeout: int
:type error_info: str
:type expected: list
:type raise_on_err: bool
:rtype: ExecResult
:raises: DevopsCalledProcessError
"""
if expected is None:
expected = [proc_enums.ExitCodes.EX_OK]
else:
expected = [
proc_enums.ExitCodes(code)
if (
isinstance(code, int) and
code in proc_enums.ExitCodes.__members__.values())
else code
for code in expected
]
ret = cls.execute(command, verbose, timeout, **kwargs)
if ret['exit_code'] not in expected:
message = (
"{append}Command '{cmd!r}' returned exit code {code!s} while "
"expected {expected!s}\n"
"\tSTDOUT:\n"
"{stdout}"
"\n\tSTDERR:\n"
"{stderr}".format(
append=error_info + '\n' if error_info else '',
cmd=command,
code=ret['exit_code'],
expected=expected,
stdout=ret['stdout_str'],
stderr=ret['stderr_str']
))
logger.error(message)
if raise_on_err:
raise error.DevopsCalledProcessError(
command, ret['exit_code'],
expected=expected,
stdout=ret['stdout_str'],
stderr=ret['stderr_str'])
return ret
def create_environment(cls, full_config):
"""Create a new environment using full_config object
:param full_config: object that describes all the parameters of
created environment
:rtype: Environment
"""
config = full_config['template']['devops_settings']
environment = cls.create(config['env_name'])
try:
# create groups and drivers
groups = config['groups']
environment.add_groups(groups)
# create address pools
address_pools = config['address_pools']
environment.add_address_pools(address_pools)
# process group items
for group_data in groups:
group = environment.get_group(name=group_data['name'])
# add l2_network_devices
group.add_l2_network_devices(
group_data.get('l2_network_devices', {}))
# add network_pools
group.add_network_pools(
group_data.get('network_pools', {}))
# Connect nodes to already created networks
for group_data in groups:
group = environment.get_group(name=group_data['name'])
# add group volumes
group.add_volumes(
group_data.get('group_volumes', []))
# add nodes
group.add_nodes(
group_data.get('nodes', []))
except Exception:
logger.error("Creation of the environment '{0}' failed"
.format(config['env_name']))
environment.erase()
raise
return environment
def is_active(self, timeout=5):
"""Check if node is active
:rtype : Boolean
"""
try:
with helpers.RunLimit(timeout):
states = self.driver.conn.node.states(self.uuid)
return (states['provision_state'] == 'active'
and states['power_state'] == 'power on')
except error.TimeoutError:
logger.error("Ironic API is not responded for {0}sec, assuming "
"that node {1} is absent".format(timeout, self.name))
return False
def exists(self, timeout=5):
"""Check if node exists
:rtype : Boolean
"""
try:
with helpers.RunLimit(timeout):
return any([
self.uuid == node.uuid
for node in self.driver.conn.node.list()
])
except error.TimeoutError:
logger.error("Ironic API is not responded for {0}sec, assuming "
"that node {1} is absent".format(timeout, self.name))
return False
def wait_pass(raising_predicate,
expected=Exception,
interval=5,
timeout=60,
timeout_msg="Waiting timed out",
predicate_args=None,
predicate_kwargs=None):
"""Wait for successful return from predicate ignoring expected exception
Options:
:param interval: - seconds between checks.
:param timeout: - raise TimeoutError if predicate still throwing expected
exception after this amount of seconds.
:param timeout_msg: - text of the TimeoutError
:param predicate_args: - positional arguments for given predicate wrapped
in list or tuple
:param predicate_kwargs: - dict with named arguments for the predicate
:param expected_exc: Exception that can be ignored while waiting (its
possible to pass several using list/tuple
"""
predicate_args = predicate_args or []
predicate_kwargs = predicate_kwargs or {}
_check_wait_args(raising_predicate, predicate_args, predicate_kwargs,
interval, timeout)
msg = ("{msg}\nWaited for pass {cmd}: {spent} seconds."
"".format(msg=timeout_msg,
cmd=repr(raising_predicate),
spent="{spent:0.3f}"))
start_time = time.time()
with RunLimit(timeout, msg):
while True:
try:
result = raising_predicate(*predicate_args, **predicate_kwargs)
logger.debug(
"wait_pass() completed with result='{0}'".format(result))
return result
except expected as e:
if start_time + timeout < time.time():
err_msg = msg.format(spent=time.time() - start_time)
logger.error(err_msg)
raise error.TimeoutError(err_msg)
logger.debug("Got expected exception {!r}, continue".format(e))
time.sleep(interval)
def create_environment(cls, full_config):
"""Create a new environment using full_config object
:param full_config: object that describes all the parameters of
created environment
:rtype: Environment
"""
config = full_config['template']['devops_settings']
environment = cls.create(config['env_name'])
try:
# create groups and drivers
groups = config['groups']
environment.add_groups(groups)
# create address pools
address_pools = config['address_pools']
environment.add_address_pools(address_pools)
# process group items
for group_data in groups:
group = environment.get_group(name=group_data['name'])
# add l2_network_devices
group.add_l2_network_devices(
group_data.get('l2_network_devices', {}))
# add network_pools
group.add_network_pools(group_data.get('network_pools', {}))
# Connect nodes to already created networks
for group_data in groups:
group = environment.get_group(name=group_data['name'])
# add group volumes
group.add_volumes(group_data.get('group_volumes', []))
# add nodes
group.add_nodes(group_data.get('nodes', []))
except Exception:
logger.error("Creation of the environment '{0}' failed".format(
config['env_name']))
environment.erase()
raise
return environment
def check_call(
cls,
command, verbose=False, timeout=None,
error_info=None,
expected=None, raise_on_err=True, **kwargs):
"""Execute command and check for return code
Timeout limitation: read tick is 100 ms.
:type command: str
:type verbose: bool
:type timeout: int
:type error_info: str
:type expected: list
:type raise_on_err: bool
:rtype: ExecResult
:raises: DevopsCalledProcessError
"""
if expected is None:
expected = [proc_enums.ExitCodes.EX_OK]
else:
expected = [
proc_enums.ExitCodes(code)
if (
isinstance(code, int) and
code in proc_enums.ExitCodes.__members__.values())
else code
for code in expected
]
ret = cls.execute(command, verbose, timeout, **kwargs)
if ret['exit_code'] not in expected:
message = (
log_templates.CMD_UNEXPECTED_EXIT_CODE.format(
append=error_info + '\n' if error_info else '',
cmd=command,
code=ret['exit_code'],
expected=expected
))
logger.error(message)
if raise_on_err:
raise error.DevopsCalledProcessError(
command, ret['exit_code'],
expected=expected,
stdout=ret['stdout_brief'],
stderr=ret['stderr_brief'])
return ret
def check_call(cls,
command,
verbose=False,
timeout=None,
error_info=None,
expected=None,
raise_on_err=True,
**kwargs):
"""Execute command and check for return code
Timeout limitation: read tick is 100 ms.
:type command: str
:type verbose: bool
:type timeout: int
:type error_info: str
:type expected: list
:type raise_on_err: bool
:rtype: ExecResult
:raises: DevopsCalledProcessError
"""
if expected is None:
expected = [proc_enums.ExitCodes.EX_OK]
else:
expected = [
proc_enums.ExitCodes(code) if
(isinstance(code, int)
and code in proc_enums.ExitCodes.__members__.values()) else
code for code in expected
]
ret = cls.execute(command, verbose, timeout, **kwargs)
if ret['exit_code'] not in expected:
message = (log_templates.CMD_UNEXPECTED_EXIT_CODE.format(
append=error_info + '\n' if error_info else '',
cmd=command,
code=ret['exit_code'],
expected=expected))
logger.error(message)
if raise_on_err:
raise error.DevopsCalledProcessError(
command,
ret['exit_code'],
expected=expected,
stdout=ret['stdout_brief'],
stderr=ret['stderr_brief'])
return ret
def wait_pass(raising_predicate, expected=Exception,
interval=5, timeout=60, timeout_msg="Waiting timed out",
predicate_args=None, predicate_kwargs=None):
"""Wait for successful return from predicate ignoring expected exception
Options:
:param interval: - seconds between checks.
:param timeout: - raise TimeoutError if predicate still throwing expected
exception after this amount of seconds.
:param timeout_msg: - text of the TimeoutError
:param predicate_args: - positional arguments for given predicate wrapped
in list or tuple
:param predicate_kwargs: - dict with named arguments for the predicate
:param expected_exc: Exception that can be ignored while waiting (its
possible to pass several using list/tuple
"""
predicate_args = predicate_args or []
predicate_kwargs = predicate_kwargs or {}
_check_wait_args(raising_predicate, predicate_args, predicate_kwargs,
interval, timeout)
msg = (
"{msg}\nWaited for pass {cmd}: {spent} seconds."
"".format(
msg=timeout_msg,
cmd=repr(raising_predicate),
spent="{spent:0.3f}"
))
start_time = time.time()
with RunLimit(timeout, msg):
while True:
try:
result = raising_predicate(*predicate_args, **predicate_kwargs)
logger.debug("wait_pass() completed with result='{0}'"
.format(result))
return result
except expected as e:
if start_time + timeout < time.time():
err_msg = msg.format(spent=time.time() - start_time)
logger.error(err_msg)
raise error.TimeoutError(err_msg)
logger.debug("Got expected exception {!r}, continue".format(e))
time.sleep(interval)
def erase_snapshot(self, name):
if self.has_snapshot(name):
snapshot = self.driver.node_get_snapshot(self, name)
if snapshot.get_type == "external":
if snapshot.children_num == 0:
self.driver.node_delete_snapshot(node=self, name=name)
for disk in self.disk_devices:
if disk.device == "disk":
snap_disk = disk.volume
# update disk on node
disk.volume = disk.volume.backing_store
disk.save()
snap_disk.remove()
else:
logger.error("Cannot delete external snapshots with children")
else:
self.driver.node_delete_snapshot(node=self, name=name)
def wait(predicate,
interval=5,
timeout=60,
timeout_msg="Waiting timed out",
predicate_args=None,
predicate_kwargs=None):
"""Wait until predicate will become True.
Options:
:param interval: - seconds between checks.
:param timeout: - raise TimeoutError if predicate won't become True after
this amount of seconds.
:param timeout_msg: - text of the TimeoutError
:param predicate_args: - positional arguments for given predicate wrapped
in list or tuple
:param predicate_kwargs: - dict with named arguments for the predicate
"""
predicate_args = predicate_args or []
predicate_kwargs = predicate_kwargs or {}
_check_wait_args(predicate, predicate_args, predicate_kwargs, interval,
timeout)
msg = ("{msg}\nWaited for pass {cmd}: {spent} seconds."
"".format(msg=timeout_msg,
cmd=repr(predicate),
spent="{spent:0.3f}"))
start_time = time.time()
with RunLimit(timeout, msg):
while True:
result = predicate(*predicate_args, **predicate_kwargs)
if result:
logger.debug(
"wait() completed with result='{0}'".format(result))
return result
if start_time + timeout < time.time():
err_msg = msg.format(spent=time.time() - start_time)
logger.error(err_msg)
raise error.TimeoutError(err_msg)
time.sleep(interval)
def wait(predicate, interval=5, timeout=60, timeout_msg="Waiting timed out",
predicate_args=None, predicate_kwargs=None):
"""Wait until predicate will become True.
Options:
:param interval: - seconds between checks.
:param timeout: - raise TimeoutError if predicate won't become True after
this amount of seconds.
:param timeout_msg: - text of the TimeoutError
:param predicate_args: - positional arguments for given predicate wrapped
in list or tuple
:param predicate_kwargs: - dict with named arguments for the predicate
"""
predicate_args = predicate_args or []
predicate_kwargs = predicate_kwargs or {}
_check_wait_args(predicate, predicate_args, predicate_kwargs,
interval, timeout)
msg = (
"{msg}\nWaited for pass {cmd}: {spent} seconds."
"".format(
msg=timeout_msg,
cmd=repr(predicate),
spent="{spent:0.3f}"
))
start_time = time.time()
with RunLimit(timeout, msg):
while True:
result = predicate(*predicate_args, **predicate_kwargs)
if result:
logger.debug("wait() completed with result='{0}'"
.format(result))
return result
if start_time + timeout < time.time():
err_msg = msg.format(spent=time.time() - start_time)
logger.error(err_msg)
raise error.TimeoutError(err_msg)
time.sleep(interval)
def check_stderr(self,
command,
verbose=False,
timeout=None,
error_info=None,
raise_on_err=True,
**kwargs):
"""Execute command expecting return code 0 and empty STDERR
:type command: str
:type verbose: bool
:type timeout: int
:type error_info: str
:type raise_on_err: bool
:rtype: ExecResult
:raises: DevopsCalledProcessError
"""
ret = self.check_call(command,
verbose,
timeout=timeout,
error_info=error_info,
raise_on_err=raise_on_err,
**kwargs)
if ret['stderr']:
message = ("{append}Command '{cmd!r}' STDERR while not expected\n"
"\texit code: {code!s}\n"
"\tSTDOUT:\n"
"{stdout}"
"\n\tSTDERR:\n"
"{stderr}".format(append=error_info +
'\n' if error_info else '',
cmd=command,
code=ret['exit_code'],
stdout=ret['stdout_str'],
stderr=ret['stderr_str']))
logger.error(message)
if raise_on_err:
raise error.DevopsCalledProcessError(command,
ret['exit_code'],
stdout=ret['stdout_str'],
stderr=ret['stderr_str'])
return ret
def __deserialize(self, fmt):
"""Deserialize stdout as data format
:type fmt: str
:rtype: object
:raises: DevopsError
"""
try:
if fmt == 'json':
return json.loads(self.stdout_str, encoding='utf-8')
elif fmt == 'yaml':
return yaml.safe_load(self.stdout_str)
except BaseException:
tmpl = ("'{cmd}' stdout is not valid {fmt}:\n"
'{{stdout!r}}\n'.format(cmd=self.cmd, fmt=fmt))
logger.exception(tmpl.format(stdout=self.stdout_str))
raise error.DevopsError(tmpl.format(stdout=self.stdout_brief))
msg = '{fmt} deserialize target is not implemented'.format(fmt=fmt)
logger.error(msg)
raise error.DevopsNotImplementedError(msg)
def connect(self):
logger.debug(
"Connect to '{0}:{1}' as '{2}:{3}'".format(
self.host, self.port, self.username, self.password))
for private_key in self.private_keys:
try:
self._ssh.connect(
self.host, port=self.port, username=self.username,
password=self.password, pkey=private_key)
self.__actual_pkey = private_key
return
except paramiko.AuthenticationException:
continue
if self.private_keys:
logger.error("Authentication with keys failed")
self.__actual_pkey = None
self._ssh.connect(
self.host, port=self.port, username=self.username,
password=self.password)
def wait_pass(raising_predicate,
expected=Exception,
interval=5,
timeout=None,
timeout_msg="Waiting timed out"):
"""Wait for successful return from predicate or expected exception"""
if not callable(raising_predicate):
raise TypeError('Not callable raising_predicate has been posted')
start_time = time.time()
while True:
try:
return raising_predicate()
except expected:
if timeout and start_time + timeout < time.time():
msg = ("{msg}\nWaited for pass {cmd}: {spent:0.3f} seconds."
"".format(msg=timeout_msg,
cmd=raising_predicate.func_name,
spent=time.time() - start_time))
logger.error(msg)
raise
time.sleep(interval)
def node_create_snapshot(self, node, name=None, description=None,
disk_only=False, external=False):
"""Create snapshot
:type description: String
:type name: String
:type node: Node
:rtype : None
"""
if self.node_snapshot_exists(node, name):
logger.error("Snapshot with name {0} already exists".format(name))
return
domain = self.conn.lookupByUUIDString(node.uuid)
# If domain has snapshots we must check their type
snap_list = self.node_get_snapshots(node)
if len(snap_list) > 0:
snap_type = snap_list[0].get_type
if external and snap_type == 'internal':
logger.error(
"Cannot create external snapshot when internal exists")
return
if not external and snap_type == 'external':
logger.error(
"Cannot create internal snapshot when external exists")
return
logger.info(domain.state(0))
xml = self.xml_builder.build_snapshot_xml(
name, description, node, disk_only, external,
settings.SNAPSHOTS_EXTERNAL_DIR)
logger.info(xml)
if external:
# Check whether we have directory for snapshots, if not
# create it
if not os.path.exists(settings.SNAPSHOTS_EXTERNAL_DIR):
os.makedirs(settings.SNAPSHOTS_EXTERNAL_DIR)
if domain.isActive() and not disk_only:
domain.snapshotCreateXML(
xml,
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT)
else:
domain.snapshotCreateXML(
xml,
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY |
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT)
self.node_set_snapshot_current(node, name)
else:
domain.snapshotCreateXML(xml)
logger.info(domain.state(0))
def _libvirt_network(self):
try:
return self.driver.conn.networkLookupByUUIDString(self.uuid)
except libvirt.libvirtError:
logger.error("Network not found by UUID: {}".format(self.uuid))
return None
def _libvirt_node(self):
try:
return self.driver.conn.lookupByUUIDString(self.uuid)
except libvirt.libvirtError:
logger.error("Domain not found by UUID: {}".format(self.uuid))
return None
def _libvirt_volume(self):
try:
return self.driver.conn.storageVolLookupByKey(self.uuid)
except libvirt.libvirtError:
logger.error("Volume not found by UUID: {}".format(self.uuid))
return None
def _libvirt_network(self):
try:
return self.driver.conn.networkLookupByUUIDString(self.uuid)
except libvirt.libvirtError:
logger.error("Network not found by UUID: {}".format(self.uuid))
return None
def _nwfilter(self):
try:
return self.driver.conn.nwfilterLookupByName(self.nwfilter_name)
except libvirt.libvirtError:
logger.error("NWFilter not found by name: {}".format(
self.nwfilter_name))
def _libvirt_node(self):
try:
return self.driver.conn.lookupByUUIDString(self.uuid)
except libvirt.libvirtError:
logger.error("Domain not found by UUID: {}".format(self.uuid))
return None
def _libvirt_volume(self):
try:
return self.driver.conn.storageVolLookupByKey(self.uuid)
except libvirt.libvirtError:
logger.error("Volume not found by UUID: {}".format(self.uuid))
return None
def _nwfilter(self):
try:
return self.driver.conn.nwfilterLookupByName(self.nwfilter_name)
except libvirt.libvirtError:
logger.error("NWFilter not found by name: {}".format(
self.nwfilter_name))
