def find_vm_files(self):
""" Method to find VirtualMachine files location """
try:
vbox_path = getattr(self, "openstack_vbox_catalog")
except AttributeError:
if os.path.exists(os.path.join(os.path.expanduser("~"), "VirtualBox VMs")):
vbox_path = os.path.join(os.path.expanduser("~"), "VirtualBox VMs")
elif os.path.exists(os.path.join(os.path.expanduser("~"), "virtualbox")):
vbox_path = os.path.join(os.path.expanduser("~"), "virtualbox")
else:
STREAM.error(" -> Virtual box catalog not found!")
STREAM.warning(" -> You may specify it directly by adding 'openstack_vbox_catalog' attribute"
" in your configuration file")
return None
STREAM.debug(" -> VirtualBox directory: %s" % vbox_path)
vm_path = None
for paths, dirs, files in os.walk(vbox_path):
if self.vm_name in dirs:
vm_path = os.path.join(paths, self.vm_name)
return vm_path
if vm_path is None:
STREAM.error("VirtualMachine directory(%s) not found in the VirtualBox catalog directory tree(%s)!\n"
"Make sure that the VirtualMachine name you specify in parameter(vm_name) exists"
" in the directory tree (%s)" % (self.vm_name, vbox_path, vbox_path))
return None
def start_cache(self):
pool = Pool(self.WORKERS)
for status in pool.map(self.parallel_cache, self.nodes):
if "ERROR" in status:
STREAM.error(status)
else:
STREAM.success(status)
def openstack_credentials_harvester(self):
""" Method to get cluster's connection settings from the configuration file """
STREAM.info("==> Get Openstack cluster connection settings")
try:
configfile, section = self.openstack_cluster.split("::")
STREAM.debug(" -> Using user configuration file %s" % configfile)
except ValueError:
configfile = LoadSettings.GENERAL_CONFIG
STREAM.debug(" -> Using general configuration file %s" %
configfile)
section = self.openstack_cluster
config = ConfigParser()
config.read(configfile)
args = {key: value.strip() for key, value in config.items(section)}
self.clusters[section] = args
if self.clusters == {}:
STREAM.error(
" -> There are no connection settings for the Openstack clusters found!\nMake sure"
"that parameter(openstack_cluster) specified correctly.")
STREAM.error(" -> Export in Openstack passed.")
sys.exit(1)
STREAM.info(" -> Found connection settings for the Openstack cluster")
STREAM.info(" -> Target Openstack cluster set to: %s" % section)
target_cluster_name = section
return target_cluster_name
def __init__(self):
# Invoke Engine
try:
super(Core, self).__init__()
except KeyboardInterrupt:
print "\n[!] Job was interrupted by user."
exit(1)
# inherited attributes:
# self.executions - dict with executions that could be executed like keywords by invoking plugin 'execute_command'
# {action_name: command}
# self.config - dict with vm objects {vm_name: object(vm)}
# self.config_sequence - sequence to work with virtual machines list[vm_name, ...]
# self.loaded_keywords - dict with loaded keywords {keyword_name: object(keyword)}
STREAM.notice("==> BEGIN.")
# Connect notification module
self.reports = Reporter(self.config)
# Current working vm object
self.current_vm_obj = None
# Contains list of already done actions for current working vm object
self.actions_progress = []
try:
self.main()
except KeyboardInterrupt:
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.error("[!] Job was interrupted by user.")
STREAM.notice("==> Clearing ourselves")
self.clearing()
def wrapper(self, *args, **kwargs):
try:
result = f(self, *args, **kwargs)
except KeyboardInterrupt:
sys.exit(1)
except Exception as exc:
STREAM.error(exc)
STREAM.debug(format_exc())
sys.exit(1)
return result
def _restore(exception, action):
""" The function reverse actions and add ERROR report to reporter """
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
self.reports.add_report(self.current_vm_obj.__name__, "ERROR",
action)
STREAM.error(" -> %s" % exception)
STREAM.error(" -> Can't proceed with this vm")
STREAM.notice("==> Clearing ourselves")
self.clearing()
def check_connection(self):
STREAM.info("==> Waiting for the availability of the port: %s:%s" %
(self.ANSIBLE_SERVER, self.ANSIBLE_PORT))
STREAM.debug(" -> Connection timeout set to: %s" %
self.CONNECTION_TIMEOUT)
start_time_point = time()
while True:
if self._port_check(self.ANSIBLE_SERVER, self.ANSIBLE_PORT):
STREAM.info(" -> Port available.")
break
if time() - start_time_point > self.CONNECTION_TIMEOUT:
STREAM.error(" -> Port unavailable.")
break
sleep(1)
def vbox_guestadditions_update(self, ssh):
""" Method to update Virtual Box Guest Additions in VirtualMachine """
STREAM.info("==> Updating VboxGuestAdditions.")
if not self.mount_vbox_guestadditions(ssh):
return
STREAM.debug(" -> Execute update GuestAdditions.")
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"/mnt/dvd/VBoxLinuxAdditions.run 2>&1",
environment={"LC_ALL": "C"})
ssh_stdin.write("y\n")
ssh_stdin.flush()
stdout = ssh_stdout.read()
if "Running kernel modules will not be replaced" in stdout:
STREAM.success(" -> VboxGuestAdditions updated")
else:
STREAM.error(stdout)
def upload_image(self, connection):
""" Method to upload image to the openstack cluster """
args = self.get_image_properties()
args["name"] = self.openstack_image_name
STREAM.info("==> Uploading image...")
STREAM.debug(" -> Image properties: %s" % args)
# Find where vm files are located
vm_dir = self.find_vm_files()
if vm_dir is None:
return
# Find specified disk format in vm directory.
disk = None
for fil in os.listdir(vm_dir):
if fil.endswith(args["disk_format"]):
disk = os.path.join(vm_dir, fil)
if disk is None:
STREAM.error("%s disk not found in %s\nMake sure that you are specify a right disk_format "
"in parameter(openstack_image_properties) or the disk exists." % (args["disk_format"], vm_dir))
STREAM.error("Export in openstack canceled.")
return
STREAM.debug(" -> VirtualMachine's virtual hard drive location: %s" % disk)
# Get image id, if image with specified name already exists
old_image_id = self.image_exists(connection, args["name"])
# Create image object with specified properties.
image = connection.images.create(**args)
# create file with created image id
image_id_file = os.path.join(LoadSettings.WORK_DIR, ".openstack_export.tmp")
with open(image_id_file, "w") as tmp:
tmp.write(image.id)
# Uploading image.
connection.images.upload(image.id, open(disk, 'rb'))
STREAM.success(" -> Uploading complete.")
# if all ok remove tmp file
os.remove(image_id_file)
if old_image_id is not None:
STREAM.info(" -> Remove old image.")
self.delete_image(connection, old_image_id)
STREAM.debug(" -> Removed image with id: %s" % old_image_id)
STREAM.success(" -> Removed.")
def generate_default_config(config_file):
template = """; You can create vm objects and assign them any actions.
; Specify preffered section name.
[debian9-x86-template]
; Mandatory keys.
; Key specifies, which type of object will be created (vm, group, alias).
type = vm
; Key specifies Keywords which will be performed for this VirtualMachine
actions = port_forwarding, vbox_start, execute_command, vbox_stop
; Variable keys
; Key specifies to which group this object belongs.
group = linux
; You may specify email to receive notifications about Keyword's errors.
;alert = [email protected]
; That description will be shown in subject of the email
;alert_description = install curl in vm
; Attributes needed for the correct work of a Keyword's
; name of the virtual machine in VirtualBox.
vm_name = debian9-x86-template
; Command which will be executed in VirtualMachine by Keyword "execute_command"
execute_command = apt-get install -y clamav
[fedora27-amd64]
type = vm
; actions will be inherited from group
group = linux
vm_name = fedora27-amd64
execute_command = dnf install -y clamav
[freebsd10-amd64]
type = vm
group = linux
vm_name = freebsd10-amd64
execute_command = pkg install -y clamav
; You can create groups and combine it with other objects.
; Groups support attribute inheritance (groups attributes have a lower priority than vm attributes).
; Specify name of the group.
[linux]
; Mandatory key.
type = group
; Key specifies keywords which will be performed for the group of VirtualMachines.
actions = port_forwarding, vbox_start, execute_command, vbox_stop
; You can specify a timeout for each Keyword after which the process will be terminated (ex: <Keyword_name>_timeout)
execute_command_timeout = 10
; You can combine some Keywords in one action, named alias.
[linux_aliases]
type = alias
; By default aliases extends to all objects, but you can assign aliases at specific group
;group = linux
reboot = vbox_stop, vbox_start
"""
STREAM.info("==> Generating default configuration file...")
if os.path.exists(config_file):
STREAM.warning(" -> File %s already exists!" % config_file)
STREAM.warning(" -> Do you want to overwrite it? (y/n): ")
answers = ["y", "n"]
while 1:
choice = raw_input().lower()
if choice in answers:
break
STREAM.error("Choose y or n! : ")
if choice == answers[0]:
with open(config_file, "w") as config:
config.write(template)
STREAM.success(" -> Generated %s" % config_file)
else:
STREAM.notice(" -> Cancelled by user.")
sys.exit()
else:
with open(config_file, "w") as config:
config.write(template)
STREAM.success(" -> Generated %s" % config_file)
def load_config(self):
STREAM.info("==> Loading user configuration file...")
config = ConfigParser()
config.read(self.CONFIG_FILE)
aliases, groups, vms, cmds = {}, {}, {}, {}
# - Generating aliases objects
STREAM.debug("==> Generating alias objects...")
for sec in config.sections():
STREAM.debug(" -> Loading section '%s'" % sec)
try:
if config[sec]["type"] == "aliases":
STREAM.debug(" [%s] Section seems like alias object" %
sec)
args = {
key: [val.strip() for val in value.split(",")]
for key, value in config.items(sec) if key != "type"
}
STREAM.debug(" [%s] -> Section attributes: %s" %
(sec, args))
if config.has_option(sec, "group"):
STREAM.debug(
" [%s] -> Section have <group> attribute: assigned to group %s"
% (sec, str(config[sec]["group"])))
aliases[str(config[sec]["group"])] = type(
str(config[sec]["group"]), (object, ),
{"aliases": args})
STREAM.debug(
" [%s] -> Object attributes: %s" %
(sec, dir(aliases[str(config[sec]["group"])])))
else:
STREAM.debug(
" [%s] -> Section don't have <group> attribute: assigned to global context"
% sec)
aliases["global"] = type("global", (object, ),
{"aliases": args})
STREAM.debug(" [%s] -> Object attributes: %s" %
(sec, dir(aliases["global"])))
else:
STREAM.debug(
" [%s] Section doesn't seem like alias object. Passed..."
% sec)
except KeyError as wrong_key:
STREAM.error(
" -> Config Error: Wrong section '%s' Key %s not specified"
% (sec, wrong_key))
sys.exit()
STREAM.debug("[*] ==> Generated alias objects: %s\n" % aliases)
# - Generating group objects
STREAM.debug("==> Generating group objects...")
for sec in config.sections():
STREAM.debug(" -> Loading section '%s'" % sec)
try:
if config[sec]["type"] == "group":
STREAM.debug(" [%s] Section seems like group object" %
sec)
args = {
key: value
for key, value in config.items(sec) if key != "type"
}
STREAM.debug(" [%s] -> Section attributes: %s" %
(sec, args))
if aliases != {}:
STREAM.debug(
" [%s] -> Alias objects detected: object will generated with alias inheritance"
% sec)
if aliases.get(sec) is None and aliases.get(
"global") is None:
# => alias null
STREAM.debug(
" [%s] -> Group alias: False, Global alias: False -> object will generated without alias inheritance"
% sec)
groups[sec] = type(str(sec), (object, ), args)
STREAM.debug(" [%s] -> Object attrs: %s" %
(sec, groups[sec].aliases))
elif aliases.get(sec) is not None and aliases.get(
"global") is not None:
# => alias group + global
STREAM.debug(
" [%s] -> Group alias: True, Global alias: True -> alias group + global alias inheritance"
% sec)
complex_alias = dict(
aliases.get(sec).aliases,
**aliases.get("global").aliases)
aliases.get(sec).aliases = complex_alias
groups[sec] = type(str(sec), (aliases.get(sec), ),
args)
STREAM.debug(" [%s] -> Object aliases: %s" %
(sec, groups[sec].aliases))
elif aliases.get(sec) is not None:
# => alias group
STREAM.debug(
" [%s] -> Group alias: True, Global alias: False -> alias group inheritance"
% sec)
groups[sec] = type(str(sec), (aliases.get(sec), ),
args)
STREAM.debug(" [%s] -> Object attrs: %s" %
(sec, groups[sec].aliases))
elif aliases.get("global") is not None:
# => alias global
STREAM.debug(
" [%s] -> Group alias: False, Global alias: True -> global alias inheritance"
% sec)
groups[sec] = type(str(sec),
(aliases.get("global"), ), args)
STREAM.debug(" [%s] -> Object attrs: %s" %
(sec, groups[sec].aliases))
else:
STREAM.debug(
" [%s] -> Alias objects not detected: object will generated without alias inheritance"
% sec)
# => alias null
groups[sec] = type(str(sec), (object, ), args)
else:
STREAM.debug(
" [%s] Section doesn't seem like group object. Passed..."
% sec)
except KeyError as wrong_key:
STREAM.error(
" -> Config Error: Wrong section '%s' Key '%s' not specified"
% (sec, wrong_key))
sys.exit()
STREAM.debug("[*] ==> Generated group objects: %s\n" % groups)
# - Generating VM objects
STREAM.debug("==> Generating vm objects...")
vms_work_sequence = []
for sec in config.sections():
STREAM.debug(" -> Loading section '%s'" % sec)
try:
if config[sec]["type"] == "vm":
STREAM.debug(" [%s] Section seems like vm object" % sec)
args = {
key: value
for key, value in config.items(sec) if key != "type"
and key != "group" and key != "actions"
}
STREAM.debug(" [%s] -> Section attributes: %s" %
(sec, args))
# firstly check if vm section exists action attr
# then below check maybe it inherit from group
try:
act = config[sec]["actions"]
args["actions"] = act
except KeyError:
pass
# alias inheritance added in group generation step
if config.has_option(sec, "group") and groups.get(
config[sec]["group"]) is not None:
STREAM.debug(
" [%s] Assigned group detected: inherit attributes "
"from group '%s'" % (sec, config[sec]["group"]))
vms[sec] = type(str(sec),
(groups.get(config[sec]["group"]), ),
args)
else:
# if group doesn't exist or no group, adding alias inheritance
STREAM.debug(
" [%s] Assigned group not detected: assign aliases"
% sec)
if aliases.get("global") is None:
STREAM.debug(
" [%s] Aliases not assigned: no aliases" %
sec)
# => alias null
vms[sec] = type(str(sec), (), args)
else:
STREAM.debug(" [%s] Aliases assigned: global" %
sec)
# => alias global
vms[sec] = type(str(sec),
(aliases.get("global"), ), args)
# Check if 'action' attr was inherited from group
try:
acts = getattr(vms[sec], "actions")
setattr(vms[sec], "actions",
[action.strip() for action in acts.split(",")])
retro = " [%s] Section inheritance retrospective:"
final_attrs = {
attr
for attr in dir(vms[sec])
if not attr.startswith('__')
}
for attr in final_attrs:
val = getattr(vms[sec], attr)
retro += "\n\t\t\t\t\t\t%s = %s" % (attr, val)
STREAM.debug(retro % sec)
vms_work_sequence.append(sec)
except AttributeError as wrong_key:
STREAM.error(" -> Config Error: Wrong section '%s'"
" Key %s not specified" %
(sec, str(wrong_key).split(" ")[-1]))
del vms[sec]
sys.exit()
else:
STREAM.debug(
" [%s] Section doesn't seem like vm object. Passed..."
% sec)
except KeyError as wrong_key:
STREAM.error(
" -> Config Error: Wrong section '%s' Key '%s' not specified"
% (sec, wrong_key))
sys.exit()
STREAM.debug("[*] ==> Generated vm objects: %s" % vms)
STREAM.debug("[*] ==> Generated vm objects work sequence: %s" %
vms_work_sequence)
STREAM.debug("==> Finding sections with executions...")
for sec in config.sections():
try:
if config[sec]["type"] == "executions":
STREAM.debug(" -> Found section '%s'" % sec)
args = {
key: value
for key, value in config.items(sec) if key != "type"
}
cmds = dict(cmds, **args)
except KeyError as wrong_key:
STREAM.error(
" -> Config Error: Wrong section '%s' Key '%s' not specified"
% (sec, wrong_key))
sys.exit()
STREAM.debug("[*] ==> Found executions aliases: %s" % cmds)
STREAM.success(" -> User configuration file loaded")
return vms, vms_work_sequence, cmds
def check_attributes_dependencies(self):
STREAM.info("==> Checking for keywords required attributes.")
for vm in self.config_sequence:
# Set of required attributes for all Keywords used in the VirtualMachine
req_args = set()
STREAM.debug("==> VirtualMachine: %s" % vm)
for action in self.config[vm].actions:
try:
# List of required attributes for the Keyword to work
req_attr = self.loaded_keywords[
action].REQUIRED_CONFIG_ATTRS
# Add required attributes of current action to summary set
req_args = set(req_args) | set(req_attr)
except KeyError:
# If action not in executions section, check for aliases
if action not in self.executions.keys():
# Check aliases actions for required attributes
try:
aliases = self.config[vm].aliases[action]
# Intercept if VirtualMachine have no aliases
except KeyError as key:
STREAM.error(
"The keyword (%s) you use in the configuration file does not exist or is not enabled."
% key)
STREAM.warning(
"You can't use this keyword until you turn it on in .vmaker.ini"
)
sys.exit(1)
# Intercept if VirtualMachine have no aliases
except AttributeError:
STREAM.error(
"The keyword (u'%s') you use in the configuration file does not exist or is not enabled."
% action)
STREAM.warning(
"You can't use this keyword until you turn it on in .vmaker.ini"
)
sys.exit(1)
for act in aliases:
req_attr = self.loaded_keywords[
act].REQUIRED_CONFIG_ATTRS
req_args = set(req_args) | set(req_attr)
vm_attrs = [
name for name in dir(self.config[vm])
if not name.startswith('__')
]
req_args = set(req_args)
vm_attrs = set(vm_attrs)
STREAM.debug(" -> [%s] Required attributes for actions: %s" %
(vm, req_args))
STREAM.debug(" -> [%s] VirtualMachines attributes: %s" %
(vm, vm_attrs))
# Attributes comparison
result = req_args - vm_attrs
if len(result) > 0:
STREAM.error(
" -> Section <%s> missed required attributes %s." %
(vm, list(result)))
STREAM.error(
" -> This causes problems in the operation of some keywords. Check your user configuration file."
)
sys.exit()
STREAM.success(" -> All attributes are present.")
def export_vm_configuration(self):
""" Method to export VirtualMachine configuration from Virtual Vox """
STREAM.info("==> Checking if vm exists...")
vms = Popen("vboxmanage list vms |awk '{print $1}'",
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
vms = vms[0]
if not self.vm_name in vms:
STREAM.error(
" -> VirtualMachine doesn't exist!\nMake sure that the VirtualMachine"
" you specified in the parameter(vm_name) are exists.")
return False
STREAM.success(" -> Exists: True")
STREAM.info("==> Exporting configuration...")
STREAM.debug(" -> vagrant catalog directory: %s" %
self.vagrant_catalog)
if not os.path.exists(self.vagrant_catalog):
STREAM.error(
" -> Vagrant catalog (%s) does not exist!\nMake sure that the catalog"
" you specified in the parameter(vagrant_catalog) are exists."
% self.vagrant_catalog)
STREAM.warning(" -> Export in vagrant, passed.")
return False
self.work_dir = os.path.join(self.vagrant_catalog, self.vm_name)
self.tmp_dir = os.path.join(self.vagrant_catalog, self.vm_name, "tmp")
try:
os.makedirs(self.tmp_dir)
except OSError as errno:
if "Errno 17" in str(errno):
STREAM.debug(
"==> Temporary directory detected, cleaning before start..."
)
shutil.rmtree(self.tmp_dir)
STREAM.debug(" -> Removed: %s" % self.tmp_dir)
os.makedirs(self.tmp_dir)
else:
STREAM.error(errno)
return False
result = Popen(
'VBoxManage export %s --output %s' %
(self.vm_name, os.path.join(self.tmp_dir, self.vm_name + ".ovf")),
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
if len(result[1]) > 0:
if "0%...10%...20%...30%...40%...50%...60%...70%...80%...90%...100%" not in result[
1]:
raise Exception(result[1])
diskname = ""
for fil in os.listdir(self.tmp_dir):
if fil.endswith(".vmdk"):
diskname = fil
os.rename(os.path.join(self.tmp_dir, fil),
os.path.join(self.tmp_dir, "box-disk.vmdk"))
elif fil.endswith(".ovf"):
os.rename(os.path.join(self.tmp_dir, fil),
os.path.join(self.tmp_dir, "box.ovf"))
with open(os.path.join(self.tmp_dir, "box.ovf"), "r") as ovf:
ovf_file = ovf.read()
with open(os.path.join(self.tmp_dir, "box.ovf"), "w") as ovf:
ovf.write(ovf_file.replace(diskname, "box-disk.vmdk"))
return True
def do_actions(self, actions_list):
def _restore(exception, action):
""" The function reverse actions and add ERROR report to reporter """
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
self.reports.add_report(self.current_vm_obj.__name__, "ERROR",
action)
STREAM.error(" -> %s" % exception)
STREAM.error(" -> Can't proceed with this vm")
STREAM.notice("==> Clearing ourselves")
self.clearing()
def _get_timeout():
""" The function searches for a timeout for the keyword termination """
try:
ttk = getattr(self.current_vm_obj, "%s_timeout" % action)
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug(" Assigned 'timeout' for action: %s = %s min" %
(action, ttk))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
except AttributeError:
ttk = LoadSettings.TIMEOUT
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug(
" Parameter 'timeout' not assigned, for action (%s), using global: %s min"
% (action, ttk))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
ttk = int(ttk) * 60
return ttk
def _process_guard(timeout, process):
# This function kill child proccess if timeout exceed
timer = 0
while 1:
if process.is_alive():
if timer > timeout:
process.terminate()
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug("==> Keyword timeout exceed, Terminated!")
raise Exception("Keyword timeout exceed, Terminated!")
else:
if process.exitcode == 0:
break
else:
raise Exception("Exception in keyword!")
sleep(1)
if timer % 60 == 0:
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug("%s min remaining to terminate Keyword!" %
str((timeout - timer) / 60))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
timer += 1
for action in actions_list:
if action in self.executions.keys():
keyword = self.execution_get_keyword(self.executions[action])
setattr(self.current_vm_obj, keyword, self.executions[action])
action = keyword
try:
invoked_keyword = self.invoke_keyword(action)
self.actions_progress.append(action)
timeout = _get_timeout()
try:
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
# Execute keyword in child process
keyword_process = Process(target=invoked_keyword().main)
keyword_process.start()
# Monitoring running proccess
_process_guard(timeout, keyword_process)
except Exception as exc:
_restore(exc, action)
return False
except KeyError:
# Going to alias actions list
try:
result = self.do_actions(
self.current_vm_obj.aliases[action])
if result is False:
return False
except KeyError as exc:
STREAM.error(" -> Unknown action! (%s)" % str(exc))
_restore(exc, action)
return False
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
return True