def sync_node_time(env, node_name='admin', cmd=None):
if cmd is None:
cmd = "hwclock -s && NTPD=$(find /etc/init.d/ -regex "
cmd += "'/etc/init.d/ntp.?'); $NTPD stop; killall ntpd;"
cmd += " ntpd -qg && $NTPD start"
if node_name == 'admin':
try:
# If public NTP servers aren't accessible ntpdate will fail and
# ntpd daemon shouldn't be restarted to avoid 'Server has gone
# too long without sync' error while syncing time from slaves
remote = get_admin_remote(env)
remote.execute("ntpdate -d $(awk '/^server/{print"
" $2}' /etc/ntp.conf)")
except AssertionError as e:
logger.warning('Error occurred while synchronizing time on master'
': {0}'.format(e))
else:
remote = get_admin_remote(env)
remote.execute('service ntpd stop && ntpd -qg && '
'service ntpd start')
else:
remote = get_node_remote(env, node_name)
remote.execute(cmd)
remote.execute('hwclock -w')
remote_date = remote.execute('date')['stdout']
logger.info("Node time: {0}".format(remote_date))
def synchronize_all(cls):
driver = cls.get_driver()
nodes = {
driver._get_name(e.name, n.name): n
for e in cls.list_all() for n in e.get_nodes()
}
domains = set(driver.node_list())
# FIXME (AWoodward) This willy nilly wacks domains when you run this
# on domains that are outside the scope of devops, if anything this
# should cause domains to be imported into db instead of undefined.
# It also leaves network and volumes around too
# Disabled untill a safer implmentation arrives
# Undefine domains without devops nodes
#
# domains_to_undefine = domains - set(nodes.keys())
# for d in domains_to_undefine:
# driver.node_undefine_by_name(d)
# Remove devops nodes without domains
nodes_to_remove = set(nodes.keys()) - domains
for n in nodes_to_remove:
nodes[n].delete()
cls.erase_empty()
logger.info('Undefined domains: {0}, removed nodes: {1}'.format(
0, len(nodes_to_remove)))
def describe_environment(cls, boot_from='cdrom'):
environment = cls.create(settings.ENV_NAME)
networks = []
interfaces = settings.INTERFACE_ORDER
if settings.MULTIPLE_NETWORKS:
logger.info('Multiple cluster networks feature is enabled!')
if settings.BONDING:
interfaces = settings.BONDING_INTERFACES.keys()
for name in interfaces:
networks.append(environment.create_networks(name))
for name in environment.node_roles.admin_names:
environment.describe_admin_node(name, networks, boot_from)
for name in environment.node_roles.other_names:
networks_to_describe = networks
if settings.MULTIPLE_NETWORKS:
# If slave index is even number, then attach to
# it virtual networks from the second network group,
# if it is odd, then attach from the first network group.
nodegroups_idx = 1 - int(name[-2:]) % 2
networks_to_describe = [
net for net in networks if net.name
in settings.NODEGROUPS[nodegroups_idx]['pools']
]
environment.describe_empty_node(name, networks_to_describe)
for name in environment.node_roles.ironic_names:
ironic_net = []
for net in networks:
if net.name == 'ironic':
ironic_net.append(net)
environment.describe_empty_node(name, ironic_net)
return environment
def execute(self, command, verbose=False, timeout=None, **kwargs):
"""Execute command and wait for return code
:type command: str
:type verbose: bool
:type timeout: int
:rtype: ExecResult
:raises: TimeoutError
"""
chan, _, stderr, stdout = self.execute_async(command, **kwargs)
result = self.__exec_command(command,
chan,
stdout,
stderr,
timeout,
verbose=verbose)
message = ('\n{cmd!r} execution results: Exit code: {code!s}'.format(
cmd=command, code=result.exit_code))
if verbose:
logger.info(message)
else:
logger.debug(message)
return result
def revert(self, name=None, destroy=True):
"""Method to revert node in state from snapshot
For external snapshots in libvirt we use restore function.
After reverting in this way we get situation when node is connected
to original volume disk, without snapshot point. To solve this
problem we need to switch it to correct volume.
In case of usage external snapshots we clean snapshot disk when
revert to snapshot without childs and create new snapshot point
when reverting to snapshots with childs.
"""
if destroy:
self.destroy()
if self.has_snapshot(name):
snapshot = self._get_snapshot(name)
if snapshot.get_type == 'external':
# EXTERNAL SNAPSHOT
self._revert_external_snapshot(name)
else:
# ORIGINAL SNAPSHOT
logger.info("Revert {0} ({1}) to internal snapshot {2}".format(
self.name, snapshot.state, name))
self._libvirt_node.revertToSnapshot(snapshot._snapshot, 0)
else:
raise DevopsError(
'Domain snapshot for {0} node not found: no domain '
'snapshot with matching'
' name {1}'.format(self.name, name))
def revert(self, name=None, destroy=True):
"""Method to revert node in state from snapshot
For external snapshots in libvirt we use restore function.
After reverting in this way we get situation when node is connected
to original volume disk, without snapshot point. To solve this
problem we need to switch it to correct volume.
In case of usage external snapshots we clean snapshot disk when
revert to snapshot without childs and create new snapshot point
when reverting to snapshots with childs.
"""
if destroy:
self.destroy()
if self.has_snapshot(name):
snapshot = self._get_snapshot(name)
if snapshot.get_type == 'external':
# EXTERNAL SNAPSHOT
self._revert_external_snapshot(name)
else:
# ORIGINAL SNAPSHOT
logger.info("Revert {0} ({1}) to internal snapshot {2}".format(
self.name, snapshot.state, name))
self._libvirt_node.revertToSnapshot(snapshot._snapshot, 0)
else:
raise DevopsError(
'Domain snapshot for {0} node not found: no domain '
'snapshot with matching'
' name {1}'.format(self.name, name))
def describe_environment(cls, boot_from='cdrom'):
environment = cls.create(settings.ENV_NAME)
networks = []
interfaces = settings.INTERFACE_ORDER
if settings.MULTIPLE_NETWORKS:
logger.info('Multiple cluster networks feature is enabled!')
if settings.BONDING:
interfaces = settings.BONDING_INTERFACES.keys()
for name in interfaces:
networks.append(environment.create_networks(name))
for name in environment.node_roles.admin_names:
environment.describe_admin_node(name, networks, boot_from)
for name in environment.node_roles.other_names:
if settings.MULTIPLE_NETWORKS:
networks1 = [net for net in networks if net.name
in settings.NODEGROUPS[0]['pools']]
networks2 = [net for net in networks if net.name
in settings.NODEGROUPS[1]['pools']]
# If slave index is even number, then attach to
# it virtual networks from the second network group.
if int(name[-2:]) % 2 == 1:
environment.describe_empty_node(name, networks1)
elif int(name[-2:]) % 2 == 0:
environment.describe_empty_node(name, networks2)
else:
environment.describe_empty_node(name, networks)
return environment
def synchronize_all(cls):
driver = cls.get_driver()
nodes = {driver._get_name(e.name, n.name): n
for e in cls.list_all()
for n in e.get_nodes()}
domains = set(driver.node_list())
# FIXME (AWoodward) This willy nilly wacks domains when you run this
# on domains that are outside the scope of devops, if anything this
# should cause domains to be imported into db instead of undefined.
# It also leaves network and volumes around too
# Disabled untill a safer implmentation arrives
# Undefine domains without devops nodes
#
# domains_to_undefine = domains - set(nodes.keys())
# for d in domains_to_undefine:
# driver.node_undefine_by_name(d)
# Remove devops nodes without domains
nodes_to_remove = set(nodes.keys()) - domains
for n in nodes_to_remove:
nodes[n].delete()
cls.erase_empty()
logger.info('Undefined domains: {0}, removed nodes: {1}'.format(
0, len(nodes_to_remove)
))
def describe_environment(cls, boot_from='cdrom'):
environment = cls.create(settings.ENV_NAME)
networks = []
interfaces = settings.INTERFACE_ORDER
if settings.MULTIPLE_NETWORKS:
logger.info('Multiple cluster networks feature is enabled!')
if settings.BONDING:
interfaces = settings.BONDING_INTERFACES.keys()
for name in interfaces:
networks.append(environment.create_networks(name))
for name in environment.node_roles.admin_names:
environment.describe_admin_node(name, networks, boot_from)
for name in environment.node_roles.other_names:
if settings.MULTIPLE_NETWORKS:
networks1 = [
net for net in networks
if net.name in settings.NODEGROUPS[0]['pools']
]
networks2 = [
net for net in networks
if net.name in settings.NODEGROUPS[1]['pools']
]
# If slave index is even number, then attach to
# it virtual networks from the second network group.
if int(name[-2:]) % 2 == 1:
environment.describe_empty_node(name, networks1)
elif int(name[-2:]) % 2 == 0:
environment.describe_empty_node(name, networks2)
else:
environment.describe_empty_node(name, networks)
return environment
def bootstrap_and_wait(self):
self.node.start()
ip = self.node.get_ip_address_by_network_name(
settings.SSH_CREDENTIALS['admin_network'])
wait_tcp(host=ip, port=self.node.ssh_port,
timeout=self.node.bootstrap_timeout,
timeout_msg='Failed to bootstrap centos master')
logger.info('Centos cloud image bootstrap complete')
def bootstrap_and_wait(self):
self.node.start()
ip = self.node.get_ip_address_by_network_name(
settings.SSH_CREDENTIALS['admin_network'])
helpers.wait_tcp(host=ip,
port=self.node.ssh_port,
timeout=self.node.bootstrap_timeout,
timeout_msg='Failed to bootstrap centos master')
logger.info('Centos cloud image bootstrap complete')
def sync_node_time(env, node_name='admin', cmd=None):
if cmd is None:
cmd = "hwclock -s"
if node_name == 'admin':
remote = get_admin_remote(env)
else:
remote = get_node_remote(env, node_name)
remote.execute(cmd)
remote_date = remote.execute('date')['stdout']
logger.info("Node time: {0}".format(remote_date))
return remote_date
def sync_node_time(env, node_name='admin', cmd=None):
if cmd is None:
cmd = "hwclock -s"
if node_name == 'admin':
remote = get_admin_remote(env)
else:
remote = get_node_remote(env, node_name)
remote.execute(cmd)
remote_date = remote.execute('date')['stdout']
logger.info("Node time: {0}".format(remote_date))
return remote_date
def __init__(
self,
host, port=22,
username=None, password=None, private_keys=None,
auth=None, verbose=True
):
"""SSHClient helper
:type host: str
:type port: int
:type username: str
:type password: str
:type private_keys: list
:type auth: SSHAuth
:type verbose: bool, show additional error/warning messages
"""
self.__lock = threading.RLock()
self.__hostname = host
self.__port = port
self.sudo_mode = False
self.__ssh = paramiko.SSHClient()
self.__ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.__sftp = None
self.__auth = auth if auth is None else auth.copy()
self.__verbose = verbose
if auth is None:
msg = (
'SSHClient(host={host}, port={port}, username={username}): '
'initialization by username/password/private_keys '
'is deprecated in favor of SSHAuth usage. '
'Please update your code'.format(
host=host, port=port, username=username
))
warnings.warn(msg, DeprecationWarning)
logger.debug(msg)
self.__auth = SSHAuth(
username=username,
password=password,
keys=private_keys
)
self.__connect()
_MemorizedSSH.record(ssh=self)
if auth is None:
logger.info(
'{0}:{1}> SSHAuth was made from old style creds: '
'{2}'.format(self.hostname, self.port, self.auth))
def node_define(self, node):
"""
:type node: Node
:rtype : None
"""
emulator = self.get_capabilities(
).find(
'guest/arch[@name="{0:>s}"]/'
'domain[@type="{1:>s}"]/emulator'.format(
node.architecture, node.hypervisor)).text
node_xml = self.xml_builder.build_node_xml(node, emulator)
logger.info(node_xml)
node.uuid = self.conn.defineXML(node_xml).UUIDString()
def execute_through_host(
self,
target_host,
cmd,
username=None,
password=None,
key=None,
target_port=22):
if username is None and password is None and key is None:
username = self.username
password = self.__password
key = self.private_key
intermediate_channel = self._ssh.get_transport().open_channel(
'direct-tcpip', (target_host, target_port), (self.host, 0))
transport = paramiko.Transport(intermediate_channel)
transport.start_client()
logger.info("Passing authentication to: {}".format(target_host))
if password is None and key is None:
logger.debug('auth_none')
transport.auth_none(username=username)
elif key is not None:
logger.debug('auth_publickey')
transport.auth_publickey(username=username, key=key)
else:
logger.debug('auth_password')
transport.auth_password(username=username, password=password)
logger.debug("Opening session")
channel = transport.open_session()
# Make proxy objects for read
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
logger.info("Executing command: {}".format(cmd))
channel.exec_command(cmd)
# TODO(astepanov): make a logic for controlling channel state
# noinspection PyDictCreation
result = {}
result['exit_code'] = channel.recv_exit_status()
result['stdout'] = stdout.read()
result['stderr'] = stderr.read()
channel.close()
result['stdout_str'] = ''.join(result['stdout']).strip()
result['stderr_str'] = ''.join(result['stderr']).strip()
return result
def node_revert_snapshot(self, node, name=None):
"""Revert snapshot for node
:type node: Node
:type name: String
:rtype : None
"""
domain = self.conn.lookupByUUIDString(node.uuid)
snapshot = Snapshot(self._get_snapshot(domain, name))
if snapshot.get_type == 'external':
logger.info("Revert {0} ({1}) from external snapshot {2}".format(
node.name, snapshot.state, name))
if self.node_active(node):
self.node_destroy(node)
# When snapshot dont have children we need to update disks in XML
# used for reverting, standard revert function will restore links
# to original disks, but we need to use disks with snapshot point,
# we dont want to change original data
#
# For snapshot with children we need to create new snapshot chain
# and we need to start from original disks, this disks will get new
# snapshot point in node class
xml_domain = snapshot._xml_tree.find('domain')
if snapshot.children_num == 0:
domain_disks = xml_domain.findall('./devices/disk')
for s_disk, s_disk_data in snapshot.disks.items():
for d_disk in domain_disks:
d_disk_dev = d_disk.find('target').get('dev')
d_disk_device = d_disk.get('device')
if d_disk_dev == s_disk and d_disk_device == 'disk':
d_disk.find('source').set('file', s_disk_data)
if snapshot.state == 'shutoff':
# Redefine domain for snapshot without memory save
self.conn.defineXML(ET.tostring(xml_domain))
else:
self.conn.restoreFlags(
snapshot.memory_file,
dxml=ET.tostring(xml_domain),
flags=libvirt.VIR_DOMAIN_SAVE_PAUSED)
# set snapshot as current
self.node_set_snapshot_current(node, name)
else:
logger.info("Revert {0} ({1}) to internal snapshot {2}".format(
node.name, snapshot.state, name))
domain.revertToSnapshot(snapshot._snapshot, 0)
def delete_snapshot_files(self):
"""Delete snapshot external files"""
snap_type = self.get_type
if snap_type == 'external':
for snap_file in self.__snapshot_files:
if os.path.isfile(snap_file):
try:
os.remove(snap_file)
logger.info("Delete external snapshot file {0}".format(
snap_file))
except Exception:
logger.info("Cannot delete external snapshot file {0}"
" must be deleted from cron script".format(
snap_file))
def execute(self, command, verbose=False):
chan, stdin, stderr, stdout = self.execute_async(command)
result = {'stdout': [], 'stderr': [], 'exit_code': 0}
for line in stdout:
result['stdout'].append(line)
if verbose:
logger.info(line)
for line in stderr:
result['stderr'].append(line)
if verbose:
logger.info(line)
result['exit_code'] = chan.recv_exit_status()
chan.close()
return result
def start(self):
"""Start the node (power on)"""
logger.info(
"Starting Ironic node {0}(uuid={1}) with timeout={2}".format(
self.name, self.uuid, self.wait_active_timeout))
self.wait_for_state(expected_state='active',
timeout=self.wait_active_timeout)
self.driver.conn.node.set_power_state(
node_id=self.uuid,
state='on',
)
super(IronicNode, self).start()
def _node_revert_snapshot_recreate_disks(self, name):
"""Recreate snapshot disks."""
snapshot = self._get_snapshot(name)
if snapshot.children_num == 0:
for s_disk_data in snapshot.disks.values():
logger.info("Recreate {0}".format(s_disk_data))
# Save actual volume XML, delete volume and create
# new from saved XML
volume = self.driver.conn.storageVolLookupByKey(s_disk_data)
volume_xml = volume.XMLDesc()
volume_pool = volume.storagePoolLookupByVolume()
volume.delete()
volume_pool.createXML(volume_xml)
def _node_revert_snapshot_recreate_disks(self, name):
"""Recreate snapshot disks."""
snapshot = self._get_snapshot(name)
if snapshot.children_num == 0:
for s_disk_data in snapshot.disks.values():
logger.info("Recreate {0}".format(s_disk_data))
# Save actual volume XML, delete volume and create
# new from saved XML
volume = self.driver.conn.storageVolLookupByKey(s_disk_data)
volume_xml = volume.XMLDesc()
volume_pool = volume.storagePoolLookupByVolume()
volume.delete()
volume_pool.createXML(volume_xml)
def node_revert_snapshot_recreate_disks(self, node, name):
"""Recreate snapshot disks."""
domain = self.conn.lookupByUUIDString(node.uuid)
snapshot = Snapshot(self._get_snapshot(domain, name))
if snapshot.children_num == 0:
for s_disk, s_disk_data in snapshot.disks.items():
logger.info("Recreate {0}".format(s_disk_data))
# Save actual volume XML, delete volume and create
# new from saved XML
volume = self.conn.storageVolLookupByKey(s_disk_data)
volume_xml = volume.XMLDesc()
volume_pool = volume.storagePoolLookupByVolume()
volume.delete()
volume_pool.createXML(volume_xml)
def delete_snapshot_files(self):
"""Delete snapshot external files"""
snap_type = self.get_type
if snap_type == 'external':
for snap_file in self.__snapshot_files:
if os.path.isfile(snap_file):
try:
os.remove(snap_file)
logger.info(
"Delete external snapshot file {0}".format(
snap_file))
except Exception:
logger.info(
"Cannot delete external snapshot file {0}"
" must be deleted from cron script".format(
snap_file))
def remove(self, *args, **kwargs):
if self.uuid:
if self.exists():
# self.destroy()
logger.info("Removing Ironic node {0}(uuid={1})".format(
self.name, self.uuid))
try:
self.driver.conn.node.set_maintenance(
node_id=self.uuid,
state=True,
maint_reason="Removing the node from devops environment"
)
self.driver.conn.node.delete(self.uuid)
except common.apiclient.exceptions.BadRequest:
# Allow to remove node from fuel-devops if ironic API down
pass
super(IronicNode, self).remove()
def execute(self, command, verbose=False):
chan, _, stderr, stdout = self.execute_async(command)
result = {
'stdout': [],
'stderr': [],
'exit_code': 0
}
for line in stdout:
result['stdout'].append(line)
if verbose:
logger.info(line)
for line in stderr:
result['stderr'].append(line)
if verbose:
logger.info(line)
result['exit_code'] = chan.recv_exit_status()
chan.close()
return result
def synchronize_all(self):
nodes = {self.get_driver()._get_name(e.name, n.name): n
for e in Environment.objects.all()
for n in e.nodes}
domains = set(self.driver.node_list())
# Undefine domains without devops nodes
domains_to_undefine = domains - set(nodes.keys())
for d in domains_to_undefine:
self.driver.node_undefine_by_name(d)
# Remove devops nodes without domains
nodes_to_remove = set(nodes.keys()) - domains
for n in nodes_to_remove:
nodes[n].delete()
Environment.erase_empty()
logger.info('Undefined domains: %s, removed nodes: %s',
(len(domains_to_undefine), len(nodes_to_remove)))
def synchronize_all(self):
nodes = {
self.get_driver()._get_name(e.name, n.name): n
for e in Environment.objects.all() for n in e.nodes
}
domains = set(self.driver.node_list())
# Undefine domains without devops nodes
domains_to_undefine = domains - set(nodes.keys())
for d in domains_to_undefine:
self.driver.node_undefine_by_name(d)
# Remove devops nodes without domains
nodes_to_remove = set(nodes.keys()) - domains
for n in nodes_to_remove:
nodes[n].delete()
Environment.erase_empty()
logger.info('Undefined domains: %s, removed nodes: %s',
(len(domains_to_undefine), len(nodes_to_remove)))
def execute(cls, command, verbose=False, timeout=None, **kwargs):
"""Execute command and wait for return code
Timeout limitation: read tick is 100 ms.
:type command: str
:type verbose: bool
:type timeout: int
:rtype: ExecResult
:raises: TimeoutError
"""
result = cls.__exec_command(command=command, timeout=timeout,
verbose=verbose, **kwargs)
message = log_templates.CMD_RESULT.format(
cmd=command, code=result.exit_code)
if verbose:
logger.info(message)
else:
logger.debug(message)
return result
def _start_nfs(self):
if os.path.isfile('/etc/exports'):
cmd = ['sudo', 'mv', '/etc/exports', '/etc/exports-devops-last']
output = subprocess.check_output(cmd)
LOGGER.info(output)
cmd = ['sudo', 'touch', '/etc/exports']
subprocess.call(cmd)
cmd = ['sudo', 'chown', os.getlogin(), '/etc/exports']
subprocess.call(cmd)
f = open('/etc/exports', 'w+')
f.write('{0}/tftpboot/fuel/ {1}(ro,async,no_subtree_check,fsid=1,'
'no_root_squash)'.format(self.ipmi_driver_root_dir,
self.ip_node_admin))
f.close()
if self.system_init.find('systemd') == 1:
cmd = ['sudo', 'systemctl', 'restart', 'nfsd']
else:
cmd = ['sudo', 'service', 'nfs-kernel-server', 'restart']
output = subprocess.check_output(cmd)
LOGGER.debug('NFS server started, output is {0}'.format(output))
return True
def _start_nfs(self):
if os.path.isfile('/etc/exports'):
cmd = ['sudo', 'mv', '/etc/exports', '/etc/exports-devops-last']
output = subprocess.check_output(cmd)
LOGGER.info(output)
cmd = ['sudo', 'touch', '/etc/exports']
subprocess.call(cmd)
cmd = ['sudo', 'chown', os.getlogin(), '/etc/exports']
subprocess.call(cmd)
f = open('/etc/exports', 'w+')
f.write('{0}/tftpboot/fuel/ {1}(ro,async,no_subtree_check,fsid=1,'
'no_root_squash)'.format(self.ipmi_driver_root_dir,
self.ip_node_admin))
f.close()
if self.system_init.find('systemd') == 1:
cmd = ['sudo', 'systemctl', 'restart', 'nfsd']
else:
cmd = ['sudo', 'service', 'nfs-kernel-server', 'restart']
output = subprocess.check_output(cmd)
LOGGER.debug('NFS server started, output is {0}'.format(output))
return True
def execute(self, command, verbose=False, timeout=None, **kwargs):
"""Execute command and wait for return code
:type command: str
:type verbose: bool
:type timeout: int
:rtype: ExecResult
:raises: TimeoutError
"""
chan, _, stderr, stdout = self.execute_async(command, **kwargs)
result = self.__exec_command(
command, chan, stdout, stderr, timeout,
verbose=verbose
)
message = (log_templates.CMD_RESULT.format(
cmd=command.rstrip(), code=result.exit_code))
if verbose:
logger.info(message)
else:
logger.debug(message)
return result
def execute(cls, command, verbose=False, timeout=None, **kwargs):
"""Execute command and wait for return code
Timeout limitation: read tick is 100 ms.
:type command: str
:type verbose: bool
:type timeout: int
:rtype: ExecResult
:raises: TimeoutError
"""
result = cls.__exec_command(command=command,
timeout=timeout,
verbose=verbose,
**kwargs)
message = ('\n{cmd!r} execution results: Exit code: {code!s}'.format(
cmd=command, code=result.exit_code))
if verbose:
logger.info(message)
else:
logger.debug(message)
return result
def _revert_external_snapshot(self, name=None):
snapshot = self._get_snapshot(name)
self.destroy()
if snapshot.children_num == 0:
logger.info("Reuse last snapshot")
# Update current node disks
self._update_disks_from_snapshot(name)
# Recreate volumes for snapshot and reuse it.
self._node_revert_snapshot_recreate_disks(name)
# Revert snapshot
# self.driver.node_revert_snapshot(node=self, name=name)
self._redefine_external_snapshot(name=name)
else:
# Looking for last reverted snapshot without children
# or create new and start next snapshot chain
revert_name = name + '-revert'
revert_count = 0
create_new = True
while self.has_snapshot(revert_name):
# Check wheter revert snapshot has children
snapshot_revert = self._get_snapshot(revert_name)
if snapshot_revert.children_num == 0:
logger.info(
"Revert snapshot exists, clean and reuse it")
# Update current node disks
self._update_disks_from_snapshot(revert_name)
# Recreate volumes
self._node_revert_snapshot_recreate_disks(revert_name)
# Revert snapshot
# self.driver.node_revert_snapshot(
# node=self, name=revert_name)
self._redefine_external_snapshot(name=revert_name)
create_new = False
break
else:
revert_name = name + '-revert' + str(revert_count)
revert_count += 1
if create_new:
logger.info("Create new revert snapshot")
# Update current node disks
self._update_disks_from_snapshot(name)
# Revert snapshot
# self.driver.node_revert_snapshot(node=self, name=name)
self._redefine_external_snapshot(name=name)
# Create new snapshot
self.snapshot(name=revert_name, external=True)
def _revert_external_snapshot(self, name=None):
snapshot = self._get_snapshot(name)
self.destroy()
if snapshot.children_num == 0:
logger.info("Reuse last snapshot")
# Update current node disks
self._update_disks_from_snapshot(name)
# Recreate volumes for snapshot and reuse it.
self._node_revert_snapshot_recreate_disks(name)
# Revert snapshot
# self.driver.node_revert_snapshot(node=self, name=name)
self._redefine_external_snapshot(name=name)
else:
# Looking for last reverted snapshot without children
# or create new and start next snapshot chain
revert_name = name + '-revert'
revert_count = 0
create_new = True
while self.has_snapshot(revert_name):
# Check wheter revert snapshot has children
snapshot_revert = self._get_snapshot(revert_name)
if snapshot_revert.children_num == 0:
logger.info("Revert snapshot exists, clean and reuse it")
# Update current node disks
self._update_disks_from_snapshot(revert_name)
# Recreate volumes
self._node_revert_snapshot_recreate_disks(revert_name)
# Revert snapshot
# self.driver.node_revert_snapshot(
# node=self, name=revert_name)
self._redefine_external_snapshot(name=revert_name)
create_new = False
break
else:
revert_name = name + '-revert' + str(revert_count)
revert_count += 1
if create_new:
logger.info("Create new revert snapshot")
# Update current node disks
self._update_disks_from_snapshot(name)
# Revert snapshot
# self.driver.node_revert_snapshot(node=self, name=name)
self._redefine_external_snapshot(name=name)
# Create new snapshot
self.snapshot(name=revert_name, external=True)
def _redefine_external_snapshot(self, name=None):
snapshot = self._get_snapshot(name)
logger.info("Revert {0} ({1}) from external snapshot {2}".format(
self.name, snapshot.state, name))
self.destroy()
# When snapshot dont have children we need to update disks in XML
# used for reverting, standard revert function will restore links
# to original disks, but we need to use disks with snapshot point,
# we dont want to change original data
#
# For snapshot with children we need to create new snapshot chain
# and we need to start from original disks, this disks will get new
# snapshot point in node class
xml_domain = snapshot._xml_tree.find('domain')
if snapshot.children_num == 0:
domain_disks = xml_domain.findall('./devices/disk')
for s_disk, s_disk_data in snapshot.disks.items():
for d_disk in domain_disks:
d_disk_dev = d_disk.find('target').get('dev')
d_disk_device = d_disk.get('device')
if d_disk_dev == s_disk and d_disk_device == 'disk':
d_disk.find('source').set('file', s_disk_data)
if snapshot.state == 'shutoff':
# Redefine domain for snapshot without memory save
self.driver.conn.defineXML(ET.tostring(xml_domain))
else:
self.driver.conn.restoreFlags(snapshot.memory_file,
dxml=ET.tostring(xml_domain),
flags=libvirt.VIR_DOMAIN_SAVE_PAUSED)
# set snapshot as current
self.set_snapshot_current(name)
def node_create_snapshot(self, node, name=None, description=None,
disk_only=False, external=False):
"""Create snapshot
:type description: String
:type name: String
:type node: Node
:rtype : None
"""
if self.node_snapshot_exists(node, name):
logger.error("Snapshot with name {0} already exists".format(name))
return
domain = self.conn.lookupByUUIDString(node.uuid)
# If domain has snapshots we must check their type
snap_list = self.node_get_snapshots(node)
if len(snap_list) > 0:
snap_type = snap_list[0].get_type
if external and snap_type == 'internal':
logger.error(
"Cannot create external snapshot when internal exists")
return
if not external and snap_type == 'external':
logger.error(
"Cannot create internal snapshot when external exists")
return
logger.info(domain.state(0))
xml = self.xml_builder.build_snapshot_xml(
name, description, node, disk_only, external,
settings.SNAPSHOTS_EXTERNAL_DIR)
logger.info(xml)
if external:
# Check whether we have directory for snapshots, if not
# create it
if not os.path.exists(settings.SNAPSHOTS_EXTERNAL_DIR):
os.makedirs(settings.SNAPSHOTS_EXTERNAL_DIR)
if domain.isActive() and not disk_only:
domain.snapshotCreateXML(
xml,
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT)
else:
domain.snapshotCreateXML(
xml,
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY |
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT)
self.node_set_snapshot_current(node, name)
else:
domain.snapshotCreateXML(xml)
logger.info(domain.state(0))
def node_create_snapshot(self, node, name=None, description=None):
"""
:type description: String
:type name: String
:type node: Node
:rtype : None
"""
xml = self.xml_builder.build_snapshot_xml(name, description)
logger.info(xml)
domain = self.conn.lookupByUUIDString(node.uuid)
logger.info(domain.state(0))
domain.snapshotCreateXML(xml, 0)
logger.info(domain.state(0))
def define(self):
"""Define node
:rtype : None
"""
name = underscored(
deepgetattr(self, 'group.environment.name'),
self.name,
)
local_disk_devices = []
for disk in self.disk_devices:
local_disk_devices.append(dict(
disk_type=disk.type,
disk_device=disk.device,
disk_volume_format=disk.volume.format,
disk_volume_path=disk.volume.get_path(),
disk_bus=disk.bus,
disk_target_dev=disk.target_dev,
disk_serial=uuid.uuid4().hex,
))
local_interfaces = []
for interface in self.interfaces:
if interface.type != 'network':
raise NotImplementedError(
message='Interface types different from network are not '
'implemented yet')
l2_dev = interface.l2_network_device
local_interfaces.append(dict(
interface_type=interface.type,
interface_mac_address=interface.mac_address,
interface_network_name=l2_dev.network_name(),
interface_id=interface.id,
interface_model=interface.model,
))
emulator = self.driver.get_capabilities().find(
'guest/arch[@name="{0:>s}"]/'
'domain[@type="{1:>s}"]/emulator'.format(
self.architecture, self.hypervisor)).text
node_xml = LibvirtXMLBuilder.build_node_xml(
name=name,
hypervisor=self.hypervisor,
use_host_cpu=self.driver.use_host_cpu,
vcpu=self.vcpu,
memory=self.memory,
use_hugepages=self.driver.use_hugepages,
hpet=self.driver.hpet,
os_type=self.os_type,
architecture=self.architecture,
boot=self.boot,
reboot_timeout=self.driver.reboot_timeout,
bootmenu_timeout=self.bootmenu_timeout,
emulator=emulator,
has_vnc=self.has_vnc,
vnc_password=self.driver.vnc_password,
local_disk_devices=local_disk_devices,
interfaces=local_interfaces,
)
logger.info(node_xml)
self.uuid = self.driver.conn.defineXML(node_xml).UUIDString()
super(Node, self).define()
def __exec_command(cls,
command,
cwd=None,
env=None,
timeout=None,
verbose=False):
"""Command executor helper
:type command: str
:type cwd: str
:type env: dict
:type timeout: int
:rtype: ExecResult
"""
def poll_stream(src, verb_logger=None):
dst = []
try:
for line in src:
dst.append(line)
if verb_logger is not None:
verb_logger(
line.decode('utf-8',
errors='backslashreplace').rstrip())
except IOError:
pass
return dst
def poll_streams(result, stdout, stderr, verbose):
rlist, _, _ = select.select([stdout, stderr], [], [])
if rlist:
if stdout in rlist:
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
if stderr in rlist:
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
@decorators.threaded(started=True)
def poll_pipes(proc, result, stop):
"""Polling task for FIFO buffers
:type proc: subprocess.Popen
:type result: ExecResult
:type stop: threading.Event
"""
# Get file descriptors for stdout and stderr streams
fd_stdout = proc.stdout.fileno()
fd_stderr = proc.stderr.fileno()
# Get flags of stdout and stderr streams
fl_stdout = fcntl.fcntl(fd_stdout, fcntl.F_GETFL)
fl_stderr = fcntl.fcntl(fd_stderr, fcntl.F_GETFL)
# Set nonblock mode for stdout and stderr streams
fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fl_stdout | os.O_NONBLOCK)
fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fl_stderr | os.O_NONBLOCK)
while not stop.isSet():
time.sleep(0.1)
poll_streams(result=result,
stdout=proc.stdout,
stderr=proc.stderr,
verbose=verbose)
proc.poll()
if proc.returncode is not None:
result.exit_code = proc.returncode
result.stdout += poll_stream(
src=proc.stdout,
verb_logger=logger.info if verbose else logger.debug)
result.stderr += poll_stream(
src=proc.stderr,
verb_logger=logger.error if verbose else logger.debug)
stop.set()
# 1 Command per run
with cls.__lock:
result = exec_result.ExecResult(cmd=command)
stop_event = threading.Event()
if verbose:
logger.info("\nExecuting command: {!r}".format(
command.rstrip()))
else:
logger.debug("\nExecuting command: {!r}".format(
command.rstrip()))
# Run
process = subprocess.Popen(args=[command],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=cwd,
env=env,
universal_newlines=False)
# Poll output
poll_pipes(process, result, stop_event)
# wait for process close
stop_event.wait(timeout)
# Process closed?
if stop_event.isSet():
stop_event.clear()
return result
# Kill not ended process and wait for close
try:
process.kill() # kill -9
stop_event.wait(5)
except OSError:
# Nothing to kill
logger.warning("{!r} has been completed just after timeout: "
"please validate timeout.".format(command))
wait_err_msg = (
'Wait for {0!r} during {1}s: no return code!\n'.format(
command, timeout))
output_brief_msg = ('\tSTDOUT:\n'
'{0}\n'
'\tSTDERR"\n'
'{1}'.format(result.stdout_brief,
result.stderr_brief))
logger.debug(wait_err_msg)
raise error.TimeoutError(wait_err_msg + output_brief_msg)
def snapshot(self, name=None, force=False, description=None,
disk_only=False, external=False):
# Erase existing snapshot or raise an error if already exists
if self.has_snapshot(name):
if force:
self.erase_snapshot(name)
else:
raise DevopsError("Snapshot with name {0} already exists"
.format(name))
# Check that existing snapshot has the same type
self._assert_snapshot_type(external=external)
local_disk_devices = []
if external:
# EXTERNAL SNAPSHOTS
if self.driver.get_libvirt_version() < 1002012:
raise DevopsError(
"For external snapshots we need libvirtd >= 1.2.12")
# Check whether we have directory for snapshots, if not
# create it
if not os.path.exists(settings.SNAPSHOTS_EXTERNAL_DIR):
os.makedirs(settings.SNAPSHOTS_EXTERNAL_DIR)
# create new volume which will be used as
# disk for snapshot changes
self.snapshot_create_child_volumes(name)
base_memory_file = '{0}/snapshot-memory-{1}_{2}.{3}'.format(
settings.SNAPSHOTS_EXTERNAL_DIR,
deepgetattr(self, 'group.environment.name'),
self.name,
name)
file_count = 0
memory_file = base_memory_file
while os.path.exists(memory_file):
memory_file = base_memory_file + '-' + str(file_count)
file_count += 1
for disk in self.disk_devices:
if disk.device == 'disk':
local_disk_devices.append(dict(
disk_volume_path=disk.volume.get_path(),
disk_target_dev=disk.target_dev,
))
if self.is_active() and not disk_only:
create_xml_flag = libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT
else:
create_xml_flag = (
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY |
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT
)
else:
# ORIGINAL SNAPSHOTS
memory_file = ''
create_xml_flag = 0
xml = LibvirtXMLBuilder.build_snapshot_xml(
name=name,
description=description,
external=external,
disk_only=disk_only,
memory_file=memory_file,
domain_isactive=self.is_active(),
local_disk_devices=local_disk_devices
)
domain = self._libvirt_node
logger.info(xml)
logger.info(domain.state(0))
domain.snapshotCreateXML(xml, create_xml_flag)
if external:
self.set_snapshot_current(name)
logger.info(domain.state(0))
def __exec_command(cls, command, cwd=None, env=None, timeout=None,
verbose=False):
"""Command executor helper
:type command: str
:type cwd: str
:type env: dict
:type timeout: int
:rtype: ExecResult
"""
def poll_stream(src, verb_logger=None):
dst = []
try:
for line in src:
dst.append(line)
if verb_logger is not None:
verb_logger(
line.decode('utf-8',
errors='backslashreplace').rstrip()
)
except IOError:
pass
return dst
def poll_streams(result, stdout, stderr, verbose):
rlist, _, _ = select.select(
[stdout, stderr],
[],
[])
if rlist:
if stdout in rlist:
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
if stderr in rlist:
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
@decorators.threaded(started=True)
def poll_pipes(proc, result, stop):
"""Polling task for FIFO buffers
:type proc: subprocess.Popen
:type result: ExecResult
:type stop: threading.Event
"""
# Get file descriptors for stdout and stderr streams
fd_stdout = proc.stdout.fileno()
fd_stderr = proc.stderr.fileno()
# Get flags of stdout and stderr streams
fl_stdout = fcntl.fcntl(fd_stdout, fcntl.F_GETFL)
fl_stderr = fcntl.fcntl(fd_stderr, fcntl.F_GETFL)
# Set nonblock mode for stdout and stderr streams
fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fl_stdout | os.O_NONBLOCK)
fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fl_stderr | os.O_NONBLOCK)
while not stop.isSet():
time.sleep(0.1)
poll_streams(
result=result,
stdout=proc.stdout,
stderr=proc.stderr,
verbose=verbose
)
proc.poll()
if proc.returncode is not None:
result.exit_code = proc.returncode
result.stdout += poll_stream(
src=proc.stdout,
verb_logger=logger.info if verbose else logger.debug)
result.stderr += poll_stream(
src=proc.stderr,
verb_logger=logger.error if verbose else logger.debug)
stop.set()
# 1 Command per run
with cls.__lock:
result = exec_result.ExecResult(cmd=command)
stop_event = threading.Event()
message = log_templates.CMD_EXEC.format(cmd=command.rstrip())
if verbose:
logger.info(message)
else:
logger.debug(message)
# Run
process = subprocess.Popen(
args=[command],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True, cwd=cwd, env=env,
universal_newlines=False)
# Poll output
poll_pipes(process, result, stop_event)
# wait for process close
stop_event.wait(timeout)
# Process closed?
if stop_event.isSet():
stop_event.clear()
return result
# Kill not ended process and wait for close
try:
process.kill() # kill -9
stop_event.wait(5)
except OSError:
# Nothing to kill
logger.warning(
u"{!s} has been completed just after timeout: "
"please validate timeout.".format(command))
wait_err_msg = log_templates.CMD_WAIT_ERROR.format(
cmd=command.rstrip(),
timeout=timeout)
output_brief_msg = ('\tSTDOUT:\n'
'{0}\n'
'\tSTDERR"\n'
'{1}'.format(result.stdout_brief,
result.stderr_brief))
logger.debug(wait_err_msg)
raise error.TimeoutError(wait_err_msg + output_brief_msg)
def define(self):
"""Define node
:rtype : None
"""
name = underscored(
deepgetattr(self, 'group.environment.name'),
self.name,
)
local_disk_devices = []
for disk in self.disk_devices:
local_disk_devices.append(
dict(
disk_type=disk.type,
disk_device=disk.device,
disk_volume_format=disk.volume.format,
disk_volume_path=disk.volume.get_path(),
disk_bus=disk.bus,
disk_target_dev=disk.target_dev,
disk_serial=uuid.uuid4().hex,
))
local_interfaces = []
for interface in self.interfaces:
if interface.type != 'network':
raise NotImplementedError(
message='Interface types different from network are not '
'implemented yet')
l2_dev = interface.l2_network_device
local_interfaces.append(
dict(
interface_type=interface.type,
interface_mac_address=interface.mac_address,
interface_network_name=l2_dev.network_name(),
interface_id=interface.id,
interface_model=interface.model,
))
emulator = self.driver.get_capabilities().find(
'guest/arch[@name="{0:>s}"]/'
'domain[@type="{1:>s}"]/emulator'.format(self.architecture,
self.hypervisor)).text
node_xml = LibvirtXMLBuilder.build_node_xml(
name=name,
hypervisor=self.hypervisor,
use_host_cpu=self.driver.use_host_cpu,
vcpu=self.vcpu,
memory=self.memory,
use_hugepages=self.driver.use_hugepages,
hpet=self.driver.hpet,
os_type=self.os_type,
architecture=self.architecture,
boot=self.boot,
reboot_timeout=self.driver.reboot_timeout,
bootmenu_timeout=self.bootmenu_timeout,
emulator=emulator,
has_vnc=self.has_vnc,
vnc_password=self.driver.vnc_password,
local_disk_devices=local_disk_devices,
interfaces=local_interfaces,
)
logger.info(node_xml)
self.uuid = self.driver.conn.defineXML(node_xml).UUIDString()
super(Node, self).define()
def snapshot(self,
name=None,
force=False,
description=None,
disk_only=False,
external=False):
# Erase existing snapshot or raise an error if already exists
if self.has_snapshot(name):
if force:
self.erase_snapshot(name)
else:
raise DevopsError(
"Snapshot with name {0} already exists".format(name))
# Check that existing snapshot has the same type
self._assert_snapshot_type(external=external)
local_disk_devices = []
if external:
# EXTERNAL SNAPSHOTS
if self.driver.get_libvirt_version() < 1002012:
raise DevopsError(
"For external snapshots we need libvirtd >= 1.2.12")
# Check whether we have directory for snapshots, if not
# create it
if not os.path.exists(settings.SNAPSHOTS_EXTERNAL_DIR):
os.makedirs(settings.SNAPSHOTS_EXTERNAL_DIR)
# create new volume which will be used as
# disk for snapshot changes
self.snapshot_create_child_volumes(name)
base_memory_file = '{0}/snapshot-memory-{1}_{2}.{3}'.format(
settings.SNAPSHOTS_EXTERNAL_DIR,
deepgetattr(self, 'group.environment.name'), self.name, name)
file_count = 0
memory_file = base_memory_file
while os.path.exists(memory_file):
memory_file = base_memory_file + '-' + str(file_count)
file_count += 1
for disk in self.disk_devices:
if disk.device == 'disk':
local_disk_devices.append(
dict(
disk_volume_path=disk.volume.get_path(),
disk_target_dev=disk.target_dev,
))
if self.is_active() and not disk_only:
create_xml_flag = libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT
else:
create_xml_flag = (
libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY
| libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_REUSE_EXT)
else:
# ORIGINAL SNAPSHOTS
memory_file = ''
create_xml_flag = 0
xml = LibvirtXMLBuilder.build_snapshot_xml(
name=name,
description=description,
external=external,
disk_only=disk_only,
memory_file=memory_file,
domain_isactive=self.is_active(),
local_disk_devices=local_disk_devices)
domain = self._libvirt_node
logger.info(xml)
logger.info(domain.state(0))
domain.snapshotCreateXML(xml, create_xml_flag)
if external:
self.set_snapshot_current(name)
logger.info(domain.state(0))
def __exec_command(
cls, command, channel, stdout, stderr, timeout, verbose=False):
"""Get exit status from channel with timeout
:type command: str
:type channel: paramiko.channel.Channel
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type timeout: int
:type verbose: bool
:rtype: ExecResult
:raises: TimeoutError
"""
def poll_stream(src, verb_logger=None):
dst = []
try:
for line in src:
dst.append(line)
if verb_logger is not None:
verb_logger(
line.decode('utf-8',
errors='backslashreplace').rstrip()
)
except IOError:
pass
return dst
def poll_streams(result, channel, stdout, stderr, verbose):
if channel.recv_ready():
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
if channel.recv_stderr_ready():
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
@decorators.threaded(started=True)
def poll_pipes(stdout, stderr, result, stop, channel):
"""Polling task for FIFO buffers
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type result: ExecResult
:type stop: Event
:type channel: paramiko.channel.Channel
"""
while not stop.isSet():
time.sleep(0.1)
poll_streams(
result=result,
channel=channel,
stdout=stdout,
stderr=stderr,
verbose=verbose
)
if channel.status_event.is_set():
result.exit_code = result.exit_code = channel.exit_status
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
stop.set()
# channel.status_event.wait(timeout)
result = exec_result.ExecResult(cmd=command)
stop_event = threading.Event()
message = log_templates.CMD_EXEC.format(cmd=command.rstrip())
if verbose:
logger.info(message)
else:
logger.debug(message)
poll_pipes(
stdout=stdout,
stderr=stderr,
result=result,
stop=stop_event,
channel=channel
)
stop_event.wait(timeout)
# Process closed?
if stop_event.isSet():
stop_event.clear()
channel.close()
return result
stop_event.set()
channel.close()
wait_err_msg = log_templates.CMD_WAIT_ERROR.format(
cmd=command.rstrip(),
timeout=timeout)
output_brief_msg = ('\tSTDOUT:\n'
'{0}\n'
'\tSTDERR"\n'
'{1}'.format(result.stdout_brief,
result.stderr_brief))
logger.debug(wait_err_msg)
raise error.TimeoutError(wait_err_msg + output_brief_msg)
def revert(self, name=None, destroy=True):
"""Method to revert node in state from snapshot
For external snapshots in libvirt we use restore function.
After reverting in this way we get situation when node is connected
to original volume disk, without snapshot point. To solve this
problem we need to switch it to correct volume.
In case of usage external snapshots we clean snapshot disk when
revert to snapshot without childs and create new snapshot point
when reverting to snapshots with childs.
"""
if destroy:
self.destroy(verbose=False)
if self.has_snapshot(name):
snapshot = self.driver.node_get_snapshot(self, name)
if snapshot.get_type == "external":
self.destroy()
if snapshot.children_num == 0:
logger.info("Reuse last snapshot")
# Update current node disks
self._update_disks_from_snapshot(name)
# Recreate volumes for snapshot and reuse it.
self.driver.node_revert_snapshot_recreate_disks(self, name)
# Revert snapshot
self.driver.node_revert_snapshot(node=self, name=name)
else:
# Looking for last reverted snapshot without children
# or create new and start next snapshot chain
revert_name = name + "-revert"
revert_count = 0
create_new = True
while self.has_snapshot(revert_name):
# Check wheter revert snapshot has children
snapshot_revert = self.driver.node_get_snapshot(self, revert_name)
if snapshot_revert.children_num == 0:
logger.info("Revert snapshot exists, clean and reuse it")
# Update current node disks
self._update_disks_from_snapshot(revert_name)
# Recreate volumes
self.driver.node_revert_snapshot_recreate_disks(self, revert_name)
# Revert snapshot
self.driver.node_revert_snapshot(node=self, name=revert_name)
create_new = False
break
else:
revert_name = name + "-revert" + str(revert_count)
revert_count += 1
if create_new:
logger.info("Create new revert snapshot")
# Update current node disks
self._update_disks_from_snapshot(name)
# Revert snapshot
self.driver.node_revert_snapshot(node=self, name=name)
# Create new snapshot
self.snapshot(name=revert_name, external=True)
else:
self.driver.node_revert_snapshot(node=self, name=name)
else:
print(
"Domain snapshot for {0} node not found: no domain "
"snapshot with matching"
" name {1}".format(self.name, name)
)
def __exec_command(cls,
command,
channel,
stdout,
stderr,
timeout,
verbose=False):
"""Get exit status from channel with timeout
:type command: str
:type channel: paramiko.channel.Channel
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type timeout: int
:type verbose: bool
:rtype: ExecResult
:raises: TimeoutError
"""
def poll_stream(src, verb_logger=None):
dst = []
try:
for line in src:
dst.append(line)
if verb_logger is not None:
verb_logger(
line.decode('utf-8',
errors='backslashreplace').rstrip())
except IOError:
pass
return dst
def poll_streams(result, channel, stdout, stderr, verbose):
if channel.recv_ready():
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
if channel.recv_stderr_ready():
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
@decorators.threaded(started=True)
def poll_pipes(stdout, stderr, result, stop, channel):
"""Polling task for FIFO buffers
:type stdout: paramiko.channel.ChannelFile
:type stderr: paramiko.channel.ChannelFile
:type result: ExecResult
:type stop: Event
:type channel: paramiko.channel.Channel
"""
while not stop.isSet():
time.sleep(0.1)
poll_streams(result=result,
channel=channel,
stdout=stdout,
stderr=stderr,
verbose=verbose)
if channel.status_event.is_set():
result.exit_code = result.exit_code = channel.exit_status
result.stdout += poll_stream(
src=stdout,
verb_logger=logger.info if verbose else logger.debug)
result.stderr += poll_stream(
src=stderr,
verb_logger=logger.error if verbose else logger.debug)
stop.set()
# channel.status_event.wait(timeout)
result = exec_result.ExecResult(cmd=command)
stop_event = threading.Event()
if verbose:
logger.info("\nExecuting command: {!r}".format(command.rstrip()))
else:
logger.debug("\nExecuting command: {!r}".format(command.rstrip()))
poll_pipes(stdout=stdout,
stderr=stderr,
result=result,
stop=stop_event,
channel=channel)
stop_event.wait(timeout)
# Process closed?
if stop_event.isSet():
stop_event.clear()
channel.close()
return result
stop_event.set()
channel.close()
wait_err_msg = ('Wait for {0!r} during {1}s: no return code!\n'.format(
command, timeout))
output_brief_msg = ('\tSTDOUT:\n'
'{0}\n'
'\tSTDERR"\n'
'{1}'.format(result.stdout_brief,
result.stderr_brief))
logger.debug(wait_err_msg)
raise error.TimeoutError(wait_err_msg + output_brief_msg)
