def find_vm_files(self):
""" Method to find VirtualMachine files location """
try:
vbox_path = getattr(self, "openstack_vbox_catalog")
except AttributeError:
if os.path.exists(os.path.join(os.path.expanduser("~"), "VirtualBox VMs")):
vbox_path = os.path.join(os.path.expanduser("~"), "VirtualBox VMs")
elif os.path.exists(os.path.join(os.path.expanduser("~"), "virtualbox")):
vbox_path = os.path.join(os.path.expanduser("~"), "virtualbox")
else:
STREAM.error(" -> Virtual box catalog not found!")
STREAM.warning(" -> You may specify it directly by adding 'openstack_vbox_catalog' attribute"
" in your configuration file")
return None
STREAM.debug(" -> VirtualBox directory: %s" % vbox_path)
vm_path = None
for paths, dirs, files in os.walk(vbox_path):
if self.vm_name in dirs:
vm_path = os.path.join(paths, self.vm_name)
return vm_path
if vm_path is None:
STREAM.error("VirtualMachine directory(%s) not found in the VirtualBox catalog directory tree(%s)!\n"
"Make sure that the VirtualMachine name you specify in parameter(vm_name) exists"
" in the directory tree (%s)" % (self.vm_name, vbox_path, vbox_path))
return None
def main(self):
# - Attributes taken from config
self.openstack_cluster = self.openstack_cluster
self.openstack_image_name = self.openstack_image_name
self.openstack_flavor = self.openstack_flavor
self.openstack_network = self.openstack_network
# - Optional attribute taken from config
try:
self.openstack_availability_zone = getattr(
self, "openstack_availability_zone")
nodes = self.get_nodes(self.openstack_availability_zone)
except:
self.openstack_availability_zone = None
nodes = None
# --------------------------------
# List of available clusters
self.clusters = {}
target_cluster = self.openstack_credentials_harvester()
nova = self.cluster_connect(target_cluster)
STREAM.info("==> Creating cache for image %s" %
self.openstack_image_name)
# Check for already created instance with current name
STREAM.debug(" -> Check for running instances with the same name")
self.check_for_running_instances(nova)
if self.openstack_availability_zone is None:
STREAM.info(" -> Running cache on random node.")
self.cache_image(nova)
else:
STREAM.info(" -> Running parallel cache on specified nodes.")
STREAM.info(" -> Number of worker processes: %s" % self.WORKERS)
self.cache_image_multi_nodes(nova, nodes)
def openstack_credentials_harvester(self):
""" Method to get cluster's connection settings from the configuration file """
STREAM.info("==> Get Openstack cluster connection settings")
try:
configfile, section = self.openstack_cluster.split("::")
STREAM.debug(" -> Using user configuration file %s" % configfile)
except ValueError:
configfile = LoadSettings.GENERAL_CONFIG
STREAM.debug(" -> Using general configuration file %s" %
configfile)
section = self.openstack_cluster
config = ConfigParser()
config.read(configfile)
args = {key: value.strip() for key, value in config.items(section)}
self.clusters[section] = args
if self.clusters == {}:
STREAM.error(
" -> There are no connection settings for the Openstack clusters found!\nMake sure"
"that parameter(openstack_cluster) specified correctly.")
STREAM.error(" -> Export in Openstack passed.")
sys.exit(1)
STREAM.info(" -> Found connection settings for the Openstack cluster")
STREAM.info(" -> Target Openstack cluster set to: %s" % section)
target_cluster_name = section
return target_cluster_name
def _process_guard(timeout, process):
# This function kill child proccess if timeout exceed
timer = 0
while 1:
if process.is_alive():
if timer > timeout:
process.terminate()
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug("==> Keyword timeout exceed, Terminated!")
raise Exception("Keyword timeout exceed, Terminated!")
else:
if process.exitcode == 0:
break
else:
raise Exception("Exception in keyword!")
sleep(1)
if timer % 60 == 0:
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug("%s min remaining to terminate Keyword!" %
str((timeout - timer) / 60))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
timer += 1
def create_box(self):
""" Method to create vagrant box from exported configuration """
STREAM.info("==> Creating box...")
with tarfile.open(os.path.join(self.work_dir, self.boxname),
"w") as tar:
for fil in os.listdir(self.tmp_dir):
tar.add(os.path.join(self.tmp_dir, fil), arcname=fil)
STREAM.debug(" -> Clearing temporary files")
shutil.rmtree(self.tmp_dir)
def wrapper(self, *args, **kwargs):
try:
result = f(self, *args, **kwargs)
except KeyboardInterrupt:
sys.exit(1)
except Exception as exc:
STREAM.error(exc)
STREAM.debug(format_exc())
sys.exit(1)
return result
def try_harder():
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"cat /etc/os-release")
data = ssh_stdout.read()
STREAM.debug(data)
name = data.split("\n")[0]
for known_os in known_oses:
if known_os in name.lower():
STREAM.debug(" -> Detected: %s" % known_os)
return known_os
return None
def deletor(recursion_depth):
snapshots = self.get_snapshots_list()
STREAM.debug(" -> VirtualMachine snapshots: %s" % snapshots)
if self.snapshot_name not in snapshots.values():
return
for uuid, name in snapshots.items():
if name == self.snapshot_name:
delete_snap(uuid)
if recursion_depth == 0:
return
recursion_depth -= 1
deletor(recursion_depth)
def get_vboxga_latest_realese(self):
""" Method to get version of the last release of Virtual Box Guest Additions from Virtual Box server """
versions = requests.get(self.vbox_url)
soup = BeautifulSoup(versions.content, 'html.parser')
data = soup.find_all('a')
data = [
a.get("href") for a in data
if re.match(r"\d*\.\d*\.\d*/$", a.get("href"))
]
last_release = data[-1][:-1]
STREAM.debug(" -> last release: %s" % last_release)
return last_release
def parse_options(self):
STREAM.debug(" -> ---------- Options.")
options = {
'listtags': False,
'listtasks': False,
'listhosts': False,
'syntax': False,
'connection': 'smart',
'module_path': None,
'forks': 10,
'become': None,
'become_method': None,
'become_user': None,
'check': False,
'diff': False,
'private_key_file': None,
'ssh_common_args': None,
'ssh_extra_args': None,
'sftp_extra_args': None,
'scp_extra_args': None,
'verbosity': None
}
ansible_kwargs = {
attr[8:]: getattr(self, attr)
for attr in dir(self) if attr.startswith('ansible_')
and not attr.startswith('_') and not attr == "ansible_playbooks"
and not attr == "ansible_inventory_options"
}
user_can_change = [
'connection', 'module_path', 'become', 'become_method',
'become_user', 'check', 'diff', 'private_key_file',
'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args',
'scp_extra_args', 'verbosity'
]
STREAM.debug(" -> Options that user allowed to change: ")
map(lambda x: STREAM.debug(" - %s" % x), user_can_change)
STREAM.debug(" -> User changed: %s" % ansible_kwargs)
for key, val in ansible_kwargs.items():
if key not in user_can_change:
continue
if val.lower() == "true":
options[key] = True
elif val.lower() == "false":
options[key] = False
try:
options[key] = int(val)
except ValueError:
options[key] = val
STREAM.debug(" -> Result options: %s" % options)
Options = namedtuple('Options', [
'listtags', 'listtasks', 'listhosts', 'syntax', 'connection',
'module_path', 'forks', 'private_key_file', 'ssh_common_args',
'ssh_extra_args', 'sftp_extra_args', 'scp_extra_args', 'become',
'become_method', 'become_user', 'verbosity', 'check', 'diff'
])
return Options(**options)
def parse_playbooks(self):
STREAM.debug(" -> ---------- Playbooks.")
ansible_playbooks = [
playbook.strip() for playbook in self.ansible_playbooks.split(",")
]
STREAM.debug(" -> Playbooks: %s" % ansible_playbooks)
for playbook in ansible_playbooks:
if not os.path.exists(playbook):
raise Exception("the playbook: %s could not be found" %
playbook)
if not os.path.isfile(playbook):
raise Exception(
"the playbook: %s does not appear to be a file" % playbook)
return ansible_playbooks
def check_connection(self):
STREAM.info("==> Waiting for the availability of the port: %s:%s" %
(self.ANSIBLE_SERVER, self.ANSIBLE_PORT))
STREAM.debug(" -> Connection timeout set to: %s" %
self.CONNECTION_TIMEOUT)
start_time_point = time()
while True:
if self._port_check(self.ANSIBLE_SERVER, self.ANSIBLE_PORT):
STREAM.info(" -> Port available.")
break
if time() - start_time_point > self.CONNECTION_TIMEOUT:
STREAM.error(" -> Port unavailable.")
break
sleep(1)
def _port_check(self, ip, port):
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.settimeout(5)
try:
conn.connect((ip, port))
conn.send("check")
response = conn.recv(50)
STREAM.debug(" -> Check port %s: %s" %
(self.ANSIBLE_PORT, response))
return True
except Exception as err:
STREAM.debug(" -> Check port %s: %s" % (self.ANSIBLE_PORT, err))
return False
finally:
conn.close()
def vbox_guestadditions_update(self, ssh):
""" Method to update Virtual Box Guest Additions in VirtualMachine """
STREAM.info("==> Updating VboxGuestAdditions.")
if not self.mount_vbox_guestadditions(ssh):
return
STREAM.debug(" -> Execute update GuestAdditions.")
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"/mnt/dvd/VBoxLinuxAdditions.run 2>&1",
environment={"LC_ALL": "C"})
ssh_stdin.write("y\n")
ssh_stdin.flush()
stdout = ssh_stdout.read()
if "Running kernel modules will not be replaced" in stdout:
STREAM.success(" -> VboxGuestAdditions updated")
else:
STREAM.error(stdout)
def clearing(self):
""" Clearing method, invoked if job is interrupted or complete unsuccessfully.
Reverse already done actions by invoking 'clearing' method in each keyword.
If 'clearing' method not implemented in keyword, doing nothing. """
LoggerOptions.set_action("clearing")
for action in reversed(self.actions_progress):
try:
invoke = self.invoke_keyword(action)
getattr(invoke, "clearing")
except AttributeError:
STREAM.debug("==> Reverse action '%s':" % action)
STREAM.debug(
" -> Method 'clearing' not implemented in keyword, nothing to do."
)
else:
STREAM.info("==> Reverse action '%s':" % action)
invoke().clearing()
LoggerOptions.set_action(None)
def get_vbox_guestadditions_iso(self, version):
""" Method to download VirtualBoxGuestAdditions.iso from Virtual Box server """
filename = "VBoxGuestAdditions_%s.iso" % version
download_path = os.path.join(LoadSettings.WORK_DIR, filename)
if os.path.exists(download_path):
return download_path
Popen('rm -rf %s' % os.path.join(LoadSettings.WORK_DIR, "*.iso"),
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
download_link = self.vbox_url + version + "/" + filename
STREAM.debug(" -> download link: %s" % download_link)
iso = requests.get(download_link).content
STREAM.info(" -> Downloading VboxGuestAdditions...")
with open(download_path, "wb") as ga:
ga.write(iso)
STREAM.success(" -> Downloaded: %s" % download_path)
return download_path
def restore_from_snapshot(self):
STREAM.info("==> Restore VirtualMachine state from snapshot: '%s'" %
self.snapshot_name)
result = Popen('VBoxManage snapshot %s restore %s' %
(self.vm_name, self.snapshot_name),
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
if len(result[1]) > 0:
if "0%...10%...20%...30%...40%...50%...60%...70%...80%...90%...100%" not in result[
1]:
if "VBOX_E_OBJECT_NOT_FOUND" in result[1]:
raise Exception(
"Snapshot with name '%s' not found for this VirtualMachine"
% self.snapshot_name)
else:
raise Exception(result[1])
STREAM.debug(result)
STREAM.success(" -> Restore complete.")
def create_vagrant_template(self):
""" Method to create Vagrantfile template """
STREAM.debug("==> Create Vagrantfile.")
template = """
Vagrant::Config.run do |config|
# This Vagrantfile is auto-generated by `vagrant package` to contain
# the MAC address of the box. Custom configuration should be placed in
# the actual `Vagrantfile` in this box.
config.vm.base_mac = "0800274B29D3"
end
# Load include vagrant file if it exists after the auto-generated
# so it can override any of the settings
include_vagrantfile = File.expand_path("../include/_Vagrantfile", __FILE__)
load include_vagrantfile if File.exist?(include_vagrantfile)
"""
with open(os.path.join(self.tmp_dir, "Vagrantfile"),
"w") as vagrant_file:
vagrant_file.write(template)
def _get_timeout():
""" The function searches for a timeout for the keyword termination """
try:
ttk = getattr(self.current_vm_obj, "%s_timeout" % action)
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug(" Assigned 'timeout' for action: %s = %s min" %
(action, ttk))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
except AttributeError:
ttk = LoadSettings.TIMEOUT
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug(
" Parameter 'timeout' not assigned, for action (%s), using global: %s min"
% (action, ttk))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
ttk = int(ttk) * 60
return ttk
def cache_image(self, nova, depth=3):
""" Method to cache image on one random node. """
server = self.create_instance(nova)
STREAM.debug(" -> Created instance: %s" % server)
# if recursion will not breaked, whatever keyword will be terminated by vmaker timeout.
while True:
sleep(2)
status = self.get_instance_status(nova, server.id)
STREAM.debug(" -> Creation status: %s" % status)
if status == "ACTIVE":
self.delete_instance(nova, server)
STREAM.success(" -> Image has been cached.")
break
elif status == "ERROR":
self.delete_instance(nova, server)
STREAM.warning(" -> Unexpected error while launch instance")
if depth == 0:
break
STREAM.warning(" -> Trying to cache image again.")
self.cache_image(nova, depth=depth - 1)
break
def delete_snapshot(self):
def delete_snap(uuid):
result = Popen('VBoxManage snapshot %s delete %s' % (self.vm_name, uuid),
shell=True, stdout=PIPE, stderr=PIPE).communicate()
STREAM.debug(result)
def deletor(recursion_depth):
snapshots = self.get_snapshots_list()
STREAM.debug(" -> VirtualMachine snapshots: %s" % snapshots)
if self.snapshot_name not in snapshots.values():
return
for uuid, name in snapshots.items():
if name == self.snapshot_name:
delete_snap(uuid)
if recursion_depth == 0:
return
recursion_depth -= 1
deletor(recursion_depth)
STREAM.debug(" -> Delete existed snapshots with name: '%s'" % self.snapshot_name)
deletor(5)
def create_metadata_file(self):
""" Method to create metadata.json file """
STREAM.debug("==> Creating metadata.json")
STREAM.debug(" -> Calculating box checksum...")
checksum = self._calculate_box_hash()
STREAM.debug(" -> sha1 checksum: %s" % checksum)
rel_path = self.vagrant_catalog[self.vagrant_catalog.find("html") + 5:]
rel_path = rel_path.split("/")
url_rebuild = "\/".join(rel_path)
url = "\/".join([
self.vagrant_server_url, url_rebuild, self.vm_name,
self.boxname[:-5]
])
name = os.path.basename(url_rebuild) + "\/" + self.vm_name
template = """{
"name": "%s",
"versions": [
{
"version": "%s",
"providers": [
{
"name": "%s",
"url": "%s",
"checksum_type": "sha1",
"checksum": "%s"
}
]
}
]
}
""" % (name, self.version, self.provider, url, checksum)
with open(os.path.join(self.work_dir, "metadata.json"),
"w") as metadata:
metadata.write(template)
def command_exec(self, ssh, command, stdin="", get_pty=False):
""" Method to execute remote command via ssh connection """
def line_buffered(f):
""" Iterator object to get output in realtime from stdout buffer """
while not f.channel.exit_status_ready():
yield self.get_decoded(f.readline().strip())
STREAM.info(" -> Executing command: %s" % command)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command,
get_pty=get_pty)
ssh_stdout._set_mode("rb")
ssh_stdin.write(stdin)
ssh_stdin.flush()
for l in line_buffered(ssh_stdout):
STREAM.debug(l)
stderr = ssh_stderr.read()
if len(stderr) > 0:
try:
raise Exception(stderr)
except UnicodeDecodeError:
raise Exception(self.get_decoded(stderr))
STREAM.success(" -> Command executed successfully")
def get_vm_platform(self, ssh):
""" Method detects platform in VirtualMachine """
known_oses = [
"arch", "altlinux", "centos", "debian", "fedora", "freebsd",
"linuxmint", "opensuse", "redhat", "suse", "ubuntu", "windows"
]
def try_harder():
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"cat /etc/os-release")
data = ssh_stdout.read()
STREAM.debug(data)
name = data.split("\n")[0]
for known_os in known_oses:
if known_os in name.lower():
STREAM.debug(" -> Detected: %s" % known_os)
return known_os
return None
STREAM.debug("==> Detecting platform")
STREAM.debug(" -> Known_oses: %s" % known_oses)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"python -m platform")
if len(ssh_stderr.read()) > 0:
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"python2 -m platform")
if len(ssh_stderr.read()) > 0:
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"python3 -m platform")
if len(ssh_stderr.read()) > 0:
raise KeyError("python not found on remote os!")
osx_full = ssh_stdout.read().lower()
_osx = osx_full.split("-")
del _osx[-1]
osx = ""
for _os in _osx:
osx += _os
STREAM.debug(" -> Platform: %s" % osx_full.strip())
# hack to detect last opensuse versions
if "glibc" in osx_full:
ret = try_harder()
if ret is not None:
return ret
else:
for known_os in known_oses:
if known_os in osx:
STREAM.debug(" -> Detected: %s" % known_os)
return known_os
raise KeyError("Unknown os! (Not in list of 'known_oses')")
def check_vm_status(self):
STREAM.debug("==> Check Vm status.")
rvms = Popen("VBoxManage list runningvms | awk '{print $1}'", shell=True, stdout=PIPE, stderr=PIPE)
data = rvms.stdout.read()
if self.vm_name in data:
STREAM.debug(" -> VirtualMachine is already booted")
return True
STREAM.debug(" -> VirtualMachine is turned off")
return False
def check_for_running_instances(self, nova):
""" Method to check if exists already running instances
with name("vmaker-"+self.openstack_image_name) in Openstack cluster """
images = self.get_running_instances(nova)
STREAM.debug(" -> Running instances: %s" % images)
for image in images:
if image.name == "vmaker-" + self.openstack_image_name:
# if instance exists delete it
STREAM.debug(" -> Detected instance with the same name.")
self.delete_instance(nova, image)
STREAM.debug(" -> Deleted instance that already exist.")
def check_vbox_guestadditions_version(self, ssh):
""" Method to check version of Virtual Box Guest Additions in VirtualMachine """
STREAM.debug(" -> Checking VboxGuestAdditions version")
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"modinfo vboxguest |grep -iw version| awk '{print $2}'")
version = ssh_stdout.read()
if len(version) > 0:
STREAM.debug(" -> Guest VboxGuestAdditions version: %s" %
version.strip())
return version.strip()
else:
STREAM.debug(" -> Guest VboxGuestAdditions version: undefined")
return None
def send_reports(self):
""" Sending all harvested reports """
STREAM.debug("There are %s error(s) in VirtualMachines found." %
self.mail_template.ERRORS)
for mail_uid, report in self.reports.items():
self.mail_template.initialize_caption()
email = report[0].email
desc = report[0].description
for rep in report:
self.mail_template.add_block(rep.vm_name, rep.status,
rep.failed_action)
self.mail_template.initialize_footer()
STREAM.debug("==> Sending a report to: %s" % email)
try:
self._send_report(email,
self.mail_template.generate_subject(desc),
self.mail_template.generate_body())
STREAM.debug(" -> OK")
except Exception as exc:
STREAM.debug(" -> Failed (%s)" % exc)
def command_exec(self, ssh, command, stdin=""):
""" Method to execute remote command via ssh connection """
STREAM.info(" -> Executing command: %s" % command)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command)
ssh_stdin.write(stdin)
ssh_stdin.flush()
stdout = ssh_stdout.read()
stderr = ssh_stderr.read()
STREAM.debug(self.get_decoded(stdout))
if len(stderr) > 0:
STREAM.debug(self.get_decoded(stderr))
exit_code = ssh_stdout.channel.recv_exit_status()
STREAM.debug(" -> Command exitcode: %s" % exit_code)
if exit_code == 0:
STREAM.success(" -> Command executed successfully")
else:
raise Exception("Executed command exit status not 0")
def upload_image(self, connection):
""" Method to upload image to the openstack cluster """
args = self.get_image_properties()
args["name"] = self.openstack_image_name
STREAM.info("==> Uploading image...")
STREAM.debug(" -> Image properties: %s" % args)
# Find where vm files are located
vm_dir = self.find_vm_files()
if vm_dir is None:
return
# Find specified disk format in vm directory.
disk = None
for fil in os.listdir(vm_dir):
if fil.endswith(args["disk_format"]):
disk = os.path.join(vm_dir, fil)
if disk is None:
STREAM.error("%s disk not found in %s\nMake sure that you are specify a right disk_format "
"in parameter(openstack_image_properties) or the disk exists." % (args["disk_format"], vm_dir))
STREAM.error("Export in openstack canceled.")
return
STREAM.debug(" -> VirtualMachine's virtual hard drive location: %s" % disk)
# Get image id, if image with specified name already exists
old_image_id = self.image_exists(connection, args["name"])
# Create image object with specified properties.
image = connection.images.create(**args)
# create file with created image id
image_id_file = os.path.join(LoadSettings.WORK_DIR, ".openstack_export.tmp")
with open(image_id_file, "w") as tmp:
tmp.write(image.id)
# Uploading image.
connection.images.upload(image.id, open(disk, 'rb'))
STREAM.success(" -> Uploading complete.")
# if all ok remove tmp file
os.remove(image_id_file)
if old_image_id is not None:
STREAM.info(" -> Remove old image.")
self.delete_image(connection, old_image_id)
STREAM.debug(" -> Removed image with id: %s" % old_image_id)
STREAM.success(" -> Removed.")