def main(self):
# - Attributes taken from config
self.openstack_cluster = self.openstack_cluster
self.openstack_image_name = self.openstack_image_name
self.openstack_flavor = self.openstack_flavor
self.openstack_network = self.openstack_network
# - Optional attribute taken from config
try:
self.openstack_availability_zone = getattr(
self, "openstack_availability_zone")
nodes = self.get_nodes(self.openstack_availability_zone)
except:
self.openstack_availability_zone = None
nodes = None
# --------------------------------
# List of available clusters
self.clusters = {}
target_cluster = self.openstack_credentials_harvester()
nova = self.cluster_connect(target_cluster)
STREAM.info("==> Creating cache for image %s" %
self.openstack_image_name)
# Check for already created instance with current name
STREAM.debug(" -> Check for running instances with the same name")
self.check_for_running_instances(nova)
if self.openstack_availability_zone is None:
STREAM.info(" -> Running cache on random node.")
self.cache_image(nova)
else:
STREAM.info(" -> Running parallel cache on specified nodes.")
STREAM.info(" -> Number of worker processes: %s" % self.WORKERS)
self.cache_image_multi_nodes(nova, nodes)
def __init__(self, config_file):
self.CONFIG_FILE = config_file
if not os.path.exists(self.CONFIG_FILE):
STREAM.critical(
"Config Error: Configuration file not found!\nSolutions:\n\t - Specify your configuration file by adding '-c <path>' key\n\t - Generate default configuration file by adding '-g' key\nExitting..."
)
sys.exit()
def start_cache(self):
pool = Pool(self.WORKERS)
for status in pool.map(self.parallel_cache, self.nodes):
if "ERROR" in status:
STREAM.error(status)
else:
STREAM.success(status)
def main(self):
# - Attributes taken from config
self.vm_name = self.vm_name
self.vagrant_catalog = self.vagrant_catalog
# ----------------------------
if self.vagrant_catalog.endswith("/"):
self.vagrant_catalog = self.vagrant_catalog[:-1]
self.vagrant_server_url = LoadSettings.VAGRANT_SERVER_URL
if self.vagrant_server_url == "":
raise Exception(
"Parameter 'vagrant_server_url' not specified, you must specify it in vmaker.ini"
)
if self.vagrant_server_url.endswith("/"):
self.vagrant_server_url = self.vagrant_server_url[:-1]
self.vagrant_server_url = LoadSettings.VAGRANT_SERVER_URL.replace(
"//", "\/\/")
self.provider = "virtualbox"
self.version = datetime.now().strftime("%Y%m%d%H%M")
self.boxname = "%s_%s_%s.box.prep" % (self.vm_name, self.version,
self.provider)
result = self.export_vm_configuration()
if result:
self.create_vagrant_template()
self.create_box()
self.create_metadata_file()
self.renew_vm()
STREAM.success(
"==> Exporting into vagrant successfully completed.")
def _process_guard(timeout, process):
# This function kill child proccess if timeout exceed
timer = 0
while 1:
if process.is_alive():
if timer > timeout:
process.terminate()
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug("==> Keyword timeout exceed, Terminated!")
raise Exception("Keyword timeout exceed, Terminated!")
else:
if process.exitcode == 0:
break
else:
raise Exception("Exception in keyword!")
sleep(1)
if timer % 60 == 0:
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug("%s min remaining to terminate Keyword!" %
str((timeout - timer) / 60))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
timer += 1
def connect_to_vm(self):
""" Method connects to VirtualMachine via ssh """
def try_connect(ssh):
""" Recursive function to enable multiple connection attempts """
self.connect_tries += 1
try:
ssh.connect(self.ssh_server, port=int(self.ssh_port), username=self.ssh_user, password=self.ssh_password)
STREAM.success(" -> Connection established")
except Exception as err:
STREAM.warning(" -> Fail (%s)" % err)
if "ecdsakey" in str(err):
STREAM.warning("ECDSAKey error, try to fix.")
Popen('ssh-keygen -f %s -R "[%s]:%s"' %
(os.path.join(os.path.expanduser("~"), ".ssh/known_hosts"), self.ssh_server, self.ssh_port),
shell=True, stdout=PIPE, stderr=PIPE).communicate()
if self.connect_tries == self.connections_limit:
raise paramiko.ssh_exception.SSHException("Connection retries limit(%s) exceed!"
% self.connections_limit)
STREAM.info(" -> Connection retry %s:" % self.connect_tries)
sleep(15)
try_connect(ssh)
STREAM.info("==> Connecting to VirtualMachine (port = %s)." % self.ssh_port)
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.connect_tries = 0
try_connect(ssh)
del self.connect_tries
return ssh
def run_playbook(self, playbook, inventory):
options = self.parse_options()
STREAM.info("==> Execute Ansible playbook: %s" % playbook)
# initialize needed objects
loader = DataLoader()
passwords = {}
inventory = InventoryManager(loader=loader, sources=inventory)
variable_manager = VariableManager(loader=loader, inventory=inventory)
# create the playbook executor, which manages running the plays via a task queue manager
pbex = PlaybookExecutor(playbooks=[playbook],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=passwords)
# run playbook and return exit_code
results = pbex.run()
if results == 0:
STREAM.success(" -> Successfully executed.")
else:
raise Exception(
" -> Ansible playbook(%s) exited with error_code: %s" %
(playbook, results))
# Clean ansible temp files
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
def mount_vbox_guestadditions(self, ssh):
""" Method to mount VirtualBoxGuestAdditions.iso to VirtualMachine """
Popen(
'vboxmanage storageattach %s --storagectl "IDE" --port 1 --device 0'
' --type dvddrive --medium %s --forceunmount' %
(self.vm_name, "emptydrive"),
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
last_realese = self.get_vboxga_latest_realese()
iso = self.get_vbox_guestadditions_iso(last_realese)
if self.check_vbox_guestadditions_version(ssh) == last_realese:
STREAM.success(
" -> VboxGuestAdditions have a latest version (%s)." %
last_realese)
return False
Popen('vboxmanage storageattach %s --storagectl "IDE"'
' --port 1 --device 0 --type dvddrive --medium %s' %
(self.vm_name, iso),
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
sleep(1)
ssh.exec_command("mkdir /mnt/dvd")
ssh.exec_command("mount -t iso9660 -o ro /dev/cdrom /mnt/dvd")
sleep(1)
return True
def renew_vm(self):
""" Method to replace the old box """
for fil in os.listdir(self.work_dir):
if fil.endswith(".box"):
STREAM.info("==> Renew old box...")
os.remove(os.path.join(self.work_dir, fil))
os.rename(os.path.join(self.work_dir, self.boxname),
os.path.join(self.work_dir, self.boxname[:-5]))
def create_box(self):
""" Method to create vagrant box from exported configuration """
STREAM.info("==> Creating box...")
with tarfile.open(os.path.join(self.work_dir, self.boxname),
"w") as tar:
for fil in os.listdir(self.tmp_dir):
tar.add(os.path.join(self.tmp_dir, fil), arcname=fil)
STREAM.debug(" -> Clearing temporary files")
shutil.rmtree(self.tmp_dir)
def load_keywords(self):
lst_of_keywords = self.enabled_keywords
STREAM.info("==> Checking and loading keywords...")
for keyword in lst_of_keywords:
KeywordController.check_keyword(keyword)
loaded_keywords = {}
for keyword in lst_of_keywords:
loaded_keywords[keyword] = self.load_keyword(keyword)
return loaded_keywords
def wrapper(self, *args, **kwargs):
try:
result = f(self, *args, **kwargs)
except KeyboardInterrupt:
sys.exit(1)
except Exception as exc:
STREAM.error(exc)
STREAM.debug(format_exc())
sys.exit(1)
return result
def try_harder():
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
"cat /etc/os-release")
data = ssh_stdout.read()
STREAM.debug(data)
name = data.split("\n")[0]
for known_os in known_oses:
if known_os in name.lower():
STREAM.debug(" -> Detected: %s" % known_os)
return known_os
return None
def _get_connection_settings(self):
self.SMTP_SERVER = LoadSettings.SMTP_SERVER
self.SMTP_PORT = LoadSettings.SMTP_PORT
self.SMTP_USER = LoadSettings.SMTP_USER
self.SMTP_PASS = LoadSettings.SMTP_PASS
self.SMTP_MAIL_FROM = LoadSettings.SMTP_MAIL_FROM
if self.SMTP_SERVER != "":
self.ENABLE_HARVESTER = True
STREAM.notice("Email notifications: ON")
else:
STREAM.notice("Email notifications: OFF")
def deletor(recursion_depth):
snapshots = self.get_snapshots_list()
STREAM.debug(" -> VirtualMachine snapshots: %s" % snapshots)
if self.snapshot_name not in snapshots.values():
return
for uuid, name in snapshots.items():
if name == self.snapshot_name:
delete_snap(uuid)
if recursion_depth == 0:
return
recursion_depth -= 1
deletor(recursion_depth)
def clearing(self):
image_id_file = os.path.join(LoadSettings.WORK_DIR, ".openstack_export.tmp")
if os.path.exists(image_id_file):
with open(image_id_file, "r") as tmp:
old_image_id = tmp.read()
target_cluster = self.openstack_credentials_harvester()
glance = self.cluster_connect(target_cluster)
self.delete_image(glance, old_image_id)
STREAM.info(" -> Removed just created image with id %s" % old_image_id)
os.remove(image_id_file)
else:
STREAM.info(" -> Nothing to clean.")
def main(self):
# - Attributes taken from config
self.vm_name = self.vm_name
# self.forwarding_ports input format: name:guest:host, ... ex: vm_ssh:22:2020, icap:1344:1234
self.forwarding_ports = self.forwarding_ports
self.management_type = self.management_type
# ------------------------------------
STREAM.info("==> Forwarding ports.")
if self.check_vm_status():
raise Exception(
"Unable to forwarding ports, VirtualMachine is booted.")
self.forward()
def get_vboxga_latest_realese(self):
""" Method to get version of the last release of Virtual Box Guest Additions from Virtual Box server """
versions = requests.get(self.vbox_url)
soup = BeautifulSoup(versions.content, 'html.parser')
data = soup.find_all('a')
data = [
a.get("href") for a in data
if re.match(r"\d*\.\d*\.\d*/$", a.get("href"))
]
last_release = data[-1][:-1]
STREAM.debug(" -> last release: %s" % last_release)
return last_release
def parse_options(self):
STREAM.debug(" -> ---------- Options.")
options = {
'listtags': False,
'listtasks': False,
'listhosts': False,
'syntax': False,
'connection': 'smart',
'module_path': None,
'forks': 10,
'become': None,
'become_method': None,
'become_user': None,
'check': False,
'diff': False,
'private_key_file': None,
'ssh_common_args': None,
'ssh_extra_args': None,
'sftp_extra_args': None,
'scp_extra_args': None,
'verbosity': None
}
ansible_kwargs = {
attr[8:]: getattr(self, attr)
for attr in dir(self) if attr.startswith('ansible_')
and not attr.startswith('_') and not attr == "ansible_playbooks"
and not attr == "ansible_inventory_options"
}
user_can_change = [
'connection', 'module_path', 'become', 'become_method',
'become_user', 'check', 'diff', 'private_key_file',
'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args',
'scp_extra_args', 'verbosity'
]
STREAM.debug(" -> Options that user allowed to change: ")
map(lambda x: STREAM.debug(" - %s" % x), user_can_change)
STREAM.debug(" -> User changed: %s" % ansible_kwargs)
for key, val in ansible_kwargs.items():
if key not in user_can_change:
continue
if val.lower() == "true":
options[key] = True
elif val.lower() == "false":
options[key] = False
try:
options[key] = int(val)
except ValueError:
options[key] = val
STREAM.debug(" -> Result options: %s" % options)
Options = namedtuple('Options', [
'listtags', 'listtasks', 'listhosts', 'syntax', 'connection',
'module_path', 'forks', 'private_key_file', 'ssh_common_args',
'ssh_extra_args', 'sftp_extra_args', 'scp_extra_args', 'become',
'become_method', 'become_user', 'verbosity', 'check', 'diff'
])
return Options(**options)
def find_vm_files(self):
""" Method to find VirtualMachine files location """
try:
vbox_path = getattr(self, "openstack_vbox_catalog")
except AttributeError:
if os.path.exists(os.path.join(os.path.expanduser("~"), "VirtualBox VMs")):
vbox_path = os.path.join(os.path.expanduser("~"), "VirtualBox VMs")
elif os.path.exists(os.path.join(os.path.expanduser("~"), "virtualbox")):
vbox_path = os.path.join(os.path.expanduser("~"), "virtualbox")
else:
STREAM.error(" -> Virtual box catalog not found!")
STREAM.warning(" -> You may specify it directly by adding 'openstack_vbox_catalog' attribute"
" in your configuration file")
return None
STREAM.debug(" -> VirtualBox directory: %s" % vbox_path)
vm_path = None
for paths, dirs, files in os.walk(vbox_path):
if self.vm_name in dirs:
vm_path = os.path.join(paths, self.vm_name)
return vm_path
if vm_path is None:
STREAM.error("VirtualMachine directory(%s) not found in the VirtualBox catalog directory tree(%s)!\n"
"Make sure that the VirtualMachine name you specify in parameter(vm_name) exists"
" in the directory tree (%s)" % (self.vm_name, vbox_path, vbox_path))
return None
def parse_playbooks(self):
STREAM.debug(" -> ---------- Playbooks.")
ansible_playbooks = [
playbook.strip() for playbook in self.ansible_playbooks.split(",")
]
STREAM.debug(" -> Playbooks: %s" % ansible_playbooks)
for playbook in ansible_playbooks:
if not os.path.exists(playbook):
raise Exception("the playbook: %s could not be found" %
playbook)
if not os.path.isfile(playbook):
raise Exception(
"the playbook: %s does not appear to be a file" % playbook)
return ansible_playbooks
def generate_from_path(path):
""" Generating config based on path to Virtual box """
cfg = os.path.join(LoadSettings.WORK_DIR, "generated.ini")
config = ConfigParser()
config.read(cfg)
for vm in os.listdir(path):
if os.path.isdir(os.path.join(path, vm)):
config.add_section(vm)
config.set(vm, "type", "vm")
config.set(vm, "vm_name", vm)
with open(cfg, "w") as conf:
config.write(conf)
STREAM.success("Generated %s" % cfg)
def try_connect(ssh):
""" Recursive function to enable multiple connection attempts """
try:
ssh.connect(self.ssh_server,
port=int(self.ssh_port),
username=self.ssh_user,
password=self.ssh_password)
STREAM.success(" -> Connection established")
except Exception as err:
STREAM.warning(" -> Fail (%s)" % err)
if "ecdsakey" in str(err):
STREAM.warning("ECDSAKey error, try to fix.")
Popen('ssh-keygen -f %s -R "[%s]:%s"' % (os.path.join(
os.path.expanduser("~"),
".ssh/known_hosts"), self.ssh_server, self.ssh_port),
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
if self.connect_tries > 20:
raise paramiko.ssh_exception.SSHException(
"Connection retries limit exceed!")
self.connect_tries += 1
STREAM.info(" -> Connection retry %s:" % self.connect_tries)
sleep(15)
try_connect(ssh)
def _port_check(self, ip, port):
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.settimeout(5)
try:
conn.connect((ip, port))
conn.send("check")
response = conn.recv(50)
STREAM.debug(" -> Check port %s: %s" %
(self.ANSIBLE_PORT, response))
return True
except Exception as err:
STREAM.debug(" -> Check port %s: %s" % (self.ANSIBLE_PORT, err))
return False
finally:
conn.close()
def command_exec(self, ssh, command, stdin=""):
""" Method to execute remote command via ssh connection """
STREAM.info(" -> Executing command: %s" % command)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command)
ssh_stdin.write(stdin)
ssh_stdin.flush()
stdout = ssh_stdout.read()
stderr = ssh_stderr.read()
STREAM.debug(self.get_decoded(stdout))
if len(stderr) > 0:
STREAM.debug(self.get_decoded(stderr))
exit_code = ssh_stdout.channel.recv_exit_status()
STREAM.debug(" -> Command exitcode: %s" % exit_code)
if exit_code == 0:
STREAM.success(" -> Command executed successfully")
else:
raise Exception("Executed command exit status not 0")
def create_vagrant_template(self):
""" Method to create Vagrantfile template """
STREAM.debug("==> Create Vagrantfile.")
template = """
Vagrant::Config.run do |config|
# This Vagrantfile is auto-generated by `vagrant package` to contain
# the MAC address of the box. Custom configuration should be placed in
# the actual `Vagrantfile` in this box.
config.vm.base_mac = "0800274B29D3"
end
# Load include vagrant file if it exists after the auto-generated
# so it can override any of the settings
include_vagrantfile = File.expand_path("../include/_Vagrantfile", __FILE__)
load include_vagrantfile if File.exist?(include_vagrantfile)
"""
with open(os.path.join(self.tmp_dir, "Vagrantfile"),
"w") as vagrant_file:
vagrant_file.write(template)
def delete_snapshot(self):
def delete_snap(uuid):
result = Popen('VBoxManage snapshot %s delete %s' % (self.vm_name, uuid),
shell=True, stdout=PIPE, stderr=PIPE).communicate()
STREAM.debug(result)
def deletor(recursion_depth):
snapshots = self.get_snapshots_list()
STREAM.debug(" -> VirtualMachine snapshots: %s" % snapshots)
if self.snapshot_name not in snapshots.values():
return
for uuid, name in snapshots.items():
if name == self.snapshot_name:
delete_snap(uuid)
if recursion_depth == 0:
return
recursion_depth -= 1
deletor(recursion_depth)
STREAM.debug(" -> Delete existed snapshots with name: '%s'" % self.snapshot_name)
deletor(5)
def cache_image(self, nova, depth=3):
""" Method to cache image on one random node. """
server = self.create_instance(nova)
STREAM.debug(" -> Created instance: %s" % server)
# if recursion will not breaked, whatever keyword will be terminated by vmaker timeout.
while True:
sleep(2)
status = self.get_instance_status(nova, server.id)
STREAM.debug(" -> Creation status: %s" % status)
if status == "ACTIVE":
self.delete_instance(nova, server)
STREAM.success(" -> Image has been cached.")
break
elif status == "ERROR":
self.delete_instance(nova, server)
STREAM.warning(" -> Unexpected error while launch instance")
if depth == 0:
break
STREAM.warning(" -> Trying to cache image again.")
self.cache_image(nova, depth=depth - 1)
break
def noforce_stop(self):
STREAM.info("==> Attempting to gracefull shutdown VirtualMachine")
if not self.check_vm_status():
STREAM.info(" -> VirtualMachine is already stopped")
return
process = Popen("VBoxManage controlvm %s acpipowerbutton" %
self.vm_name,
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
stderr = process[1]
if len(stderr) > 0:
raise Exception(stderr)
while 1:
rvms = Popen("VBoxManage list runningvms | awk '{print $1}'",
shell=True,
stdout=PIPE,
stderr=PIPE)
data = rvms.stdout.read()
if self.vm_name not in data:
break
def _get_timeout():
""" The function searches for a timeout for the keyword termination """
try:
ttk = getattr(self.current_vm_obj, "%s_timeout" % action)
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug(" Assigned 'timeout' for action: %s = %s min" %
(action, ttk))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
except AttributeError:
ttk = LoadSettings.TIMEOUT
LoggerOptions.set_component("Core")
LoggerOptions.set_action(None)
STREAM.debug(
" Parameter 'timeout' not assigned, for action (%s), using global: %s min"
% (action, ttk))
LoggerOptions.set_component(self.current_vm_obj.__name__)
LoggerOptions.set_action(action)
ttk = int(ttk) * 60
return ttk
