def main(self): # - Attributes taken from config self.openstack_cluster = self.openstack_cluster self.openstack_image_name = self.openstack_image_name self.openstack_flavor = self.openstack_flavor self.openstack_network = self.openstack_network # - Optional attribute taken from config try: self.openstack_availability_zone = getattr( self, "openstack_availability_zone") nodes = self.get_nodes(self.openstack_availability_zone) except: self.openstack_availability_zone = None nodes = None # -------------------------------- # List of available clusters self.clusters = {} target_cluster = self.openstack_credentials_harvester() nova = self.cluster_connect(target_cluster) STREAM.info("==> Creating cache for image %s" % self.openstack_image_name) # Check for already created instance with current name STREAM.debug(" -> Check for running instances with the same name") self.check_for_running_instances(nova) if self.openstack_availability_zone is None: STREAM.info(" -> Running cache on random node.") self.cache_image(nova) else: STREAM.info(" -> Running parallel cache on specified nodes.") STREAM.info(" -> Number of worker processes: %s" % self.WORKERS) self.cache_image_multi_nodes(nova, nodes)
def __init__(self, config_file): self.CONFIG_FILE = config_file if not os.path.exists(self.CONFIG_FILE): STREAM.critical( "Config Error: Configuration file not found!\nSolutions:\n\t - Specify your configuration file by adding '-c <path>' key\n\t - Generate default configuration file by adding '-g' key\nExitting..." ) sys.exit()
def start_cache(self): pool = Pool(self.WORKERS) for status in pool.map(self.parallel_cache, self.nodes): if "ERROR" in status: STREAM.error(status) else: STREAM.success(status)
def main(self): # - Attributes taken from config self.vm_name = self.vm_name self.vagrant_catalog = self.vagrant_catalog # ---------------------------- if self.vagrant_catalog.endswith("/"): self.vagrant_catalog = self.vagrant_catalog[:-1] self.vagrant_server_url = LoadSettings.VAGRANT_SERVER_URL if self.vagrant_server_url == "": raise Exception( "Parameter 'vagrant_server_url' not specified, you must specify it in vmaker.ini" ) if self.vagrant_server_url.endswith("/"): self.vagrant_server_url = self.vagrant_server_url[:-1] self.vagrant_server_url = LoadSettings.VAGRANT_SERVER_URL.replace( "//", "\/\/") self.provider = "virtualbox" self.version = datetime.now().strftime("%Y%m%d%H%M") self.boxname = "%s_%s_%s.box.prep" % (self.vm_name, self.version, self.provider) result = self.export_vm_configuration() if result: self.create_vagrant_template() self.create_box() self.create_metadata_file() self.renew_vm() STREAM.success( "==> Exporting into vagrant successfully completed.")
def _process_guard(timeout, process): # This function kill child proccess if timeout exceed timer = 0 while 1: if process.is_alive(): if timer > timeout: process.terminate() LoggerOptions.set_component("Core") LoggerOptions.set_action(None) STREAM.debug("==> Keyword timeout exceed, Terminated!") raise Exception("Keyword timeout exceed, Terminated!") else: if process.exitcode == 0: break else: raise Exception("Exception in keyword!") sleep(1) if timer % 60 == 0: LoggerOptions.set_component("Core") LoggerOptions.set_action(None) STREAM.debug("%s min remaining to terminate Keyword!" % str((timeout - timer) / 60)) LoggerOptions.set_component(self.current_vm_obj.__name__) LoggerOptions.set_action(action) timer += 1
def connect_to_vm(self): """ Method connects to VirtualMachine via ssh """ def try_connect(ssh): """ Recursive function to enable multiple connection attempts """ self.connect_tries += 1 try: ssh.connect(self.ssh_server, port=int(self.ssh_port), username=self.ssh_user, password=self.ssh_password) STREAM.success(" -> Connection established") except Exception as err: STREAM.warning(" -> Fail (%s)" % err) if "ecdsakey" in str(err): STREAM.warning("ECDSAKey error, try to fix.") Popen('ssh-keygen -f %s -R "[%s]:%s"' % (os.path.join(os.path.expanduser("~"), ".ssh/known_hosts"), self.ssh_server, self.ssh_port), shell=True, stdout=PIPE, stderr=PIPE).communicate() if self.connect_tries == self.connections_limit: raise paramiko.ssh_exception.SSHException("Connection retries limit(%s) exceed!" % self.connections_limit) STREAM.info(" -> Connection retry %s:" % self.connect_tries) sleep(15) try_connect(ssh) STREAM.info("==> Connecting to VirtualMachine (port = %s)." % self.ssh_port) ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.connect_tries = 0 try_connect(ssh) del self.connect_tries return ssh
def run_playbook(self, playbook, inventory): options = self.parse_options() STREAM.info("==> Execute Ansible playbook: %s" % playbook) # initialize needed objects loader = DataLoader() passwords = {} inventory = InventoryManager(loader=loader, sources=inventory) variable_manager = VariableManager(loader=loader, inventory=inventory) # create the playbook executor, which manages running the plays via a task queue manager pbex = PlaybookExecutor(playbooks=[playbook], inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords) # run playbook and return exit_code results = pbex.run() if results == 0: STREAM.success(" -> Successfully executed.") else: raise Exception( " -> Ansible playbook(%s) exited with error_code: %s" % (playbook, results)) # Clean ansible temp files shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
def mount_vbox_guestadditions(self, ssh): """ Method to mount VirtualBoxGuestAdditions.iso to VirtualMachine """ Popen( 'vboxmanage storageattach %s --storagectl "IDE" --port 1 --device 0' ' --type dvddrive --medium %s --forceunmount' % (self.vm_name, "emptydrive"), shell=True, stdout=PIPE, stderr=PIPE).communicate() last_realese = self.get_vboxga_latest_realese() iso = self.get_vbox_guestadditions_iso(last_realese) if self.check_vbox_guestadditions_version(ssh) == last_realese: STREAM.success( " -> VboxGuestAdditions have a latest version (%s)." % last_realese) return False Popen('vboxmanage storageattach %s --storagectl "IDE"' ' --port 1 --device 0 --type dvddrive --medium %s' % (self.vm_name, iso), shell=True, stdout=PIPE, stderr=PIPE).communicate() sleep(1) ssh.exec_command("mkdir /mnt/dvd") ssh.exec_command("mount -t iso9660 -o ro /dev/cdrom /mnt/dvd") sleep(1) return True
def renew_vm(self): """ Method to replace the old box """ for fil in os.listdir(self.work_dir): if fil.endswith(".box"): STREAM.info("==> Renew old box...") os.remove(os.path.join(self.work_dir, fil)) os.rename(os.path.join(self.work_dir, self.boxname), os.path.join(self.work_dir, self.boxname[:-5]))
def create_box(self): """ Method to create vagrant box from exported configuration """ STREAM.info("==> Creating box...") with tarfile.open(os.path.join(self.work_dir, self.boxname), "w") as tar: for fil in os.listdir(self.tmp_dir): tar.add(os.path.join(self.tmp_dir, fil), arcname=fil) STREAM.debug(" -> Clearing temporary files") shutil.rmtree(self.tmp_dir)
def load_keywords(self): lst_of_keywords = self.enabled_keywords STREAM.info("==> Checking and loading keywords...") for keyword in lst_of_keywords: KeywordController.check_keyword(keyword) loaded_keywords = {} for keyword in lst_of_keywords: loaded_keywords[keyword] = self.load_keyword(keyword) return loaded_keywords
def wrapper(self, *args, **kwargs): try: result = f(self, *args, **kwargs) except KeyboardInterrupt: sys.exit(1) except Exception as exc: STREAM.error(exc) STREAM.debug(format_exc()) sys.exit(1) return result
def try_harder(): ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "cat /etc/os-release") data = ssh_stdout.read() STREAM.debug(data) name = data.split("\n")[0] for known_os in known_oses: if known_os in name.lower(): STREAM.debug(" -> Detected: %s" % known_os) return known_os return None
def _get_connection_settings(self): self.SMTP_SERVER = LoadSettings.SMTP_SERVER self.SMTP_PORT = LoadSettings.SMTP_PORT self.SMTP_USER = LoadSettings.SMTP_USER self.SMTP_PASS = LoadSettings.SMTP_PASS self.SMTP_MAIL_FROM = LoadSettings.SMTP_MAIL_FROM if self.SMTP_SERVER != "": self.ENABLE_HARVESTER = True STREAM.notice("Email notifications: ON") else: STREAM.notice("Email notifications: OFF")
def deletor(recursion_depth): snapshots = self.get_snapshots_list() STREAM.debug(" -> VirtualMachine snapshots: %s" % snapshots) if self.snapshot_name not in snapshots.values(): return for uuid, name in snapshots.items(): if name == self.snapshot_name: delete_snap(uuid) if recursion_depth == 0: return recursion_depth -= 1 deletor(recursion_depth)
def clearing(self): image_id_file = os.path.join(LoadSettings.WORK_DIR, ".openstack_export.tmp") if os.path.exists(image_id_file): with open(image_id_file, "r") as tmp: old_image_id = tmp.read() target_cluster = self.openstack_credentials_harvester() glance = self.cluster_connect(target_cluster) self.delete_image(glance, old_image_id) STREAM.info(" -> Removed just created image with id %s" % old_image_id) os.remove(image_id_file) else: STREAM.info(" -> Nothing to clean.")
def main(self): # - Attributes taken from config self.vm_name = self.vm_name # self.forwarding_ports input format: name:guest:host, ... ex: vm_ssh:22:2020, icap:1344:1234 self.forwarding_ports = self.forwarding_ports self.management_type = self.management_type # ------------------------------------ STREAM.info("==> Forwarding ports.") if self.check_vm_status(): raise Exception( "Unable to forwarding ports, VirtualMachine is booted.") self.forward()
def get_vboxga_latest_realese(self): """ Method to get version of the last release of Virtual Box Guest Additions from Virtual Box server """ versions = requests.get(self.vbox_url) soup = BeautifulSoup(versions.content, 'html.parser') data = soup.find_all('a') data = [ a.get("href") for a in data if re.match(r"\d*\.\d*\.\d*/$", a.get("href")) ] last_release = data[-1][:-1] STREAM.debug(" -> last release: %s" % last_release) return last_release
def parse_options(self): STREAM.debug(" -> ---------- Options.") options = { 'listtags': False, 'listtasks': False, 'listhosts': False, 'syntax': False, 'connection': 'smart', 'module_path': None, 'forks': 10, 'become': None, 'become_method': None, 'become_user': None, 'check': False, 'diff': False, 'private_key_file': None, 'ssh_common_args': None, 'ssh_extra_args': None, 'sftp_extra_args': None, 'scp_extra_args': None, 'verbosity': None } ansible_kwargs = { attr[8:]: getattr(self, attr) for attr in dir(self) if attr.startswith('ansible_') and not attr.startswith('_') and not attr == "ansible_playbooks" and not attr == "ansible_inventory_options" } user_can_change = [ 'connection', 'module_path', 'become', 'become_method', 'become_user', 'check', 'diff', 'private_key_file', 'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args', 'scp_extra_args', 'verbosity' ] STREAM.debug(" -> Options that user allowed to change: ") map(lambda x: STREAM.debug(" - %s" % x), user_can_change) STREAM.debug(" -> User changed: %s" % ansible_kwargs) for key, val in ansible_kwargs.items(): if key not in user_can_change: continue if val.lower() == "true": options[key] = True elif val.lower() == "false": options[key] = False try: options[key] = int(val) except ValueError: options[key] = val STREAM.debug(" -> Result options: %s" % options) Options = namedtuple('Options', [ 'listtags', 'listtasks', 'listhosts', 'syntax', 'connection', 'module_path', 'forks', 'private_key_file', 'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args', 'scp_extra_args', 'become', 'become_method', 'become_user', 'verbosity', 'check', 'diff' ]) return Options(**options)
def find_vm_files(self): """ Method to find VirtualMachine files location """ try: vbox_path = getattr(self, "openstack_vbox_catalog") except AttributeError: if os.path.exists(os.path.join(os.path.expanduser("~"), "VirtualBox VMs")): vbox_path = os.path.join(os.path.expanduser("~"), "VirtualBox VMs") elif os.path.exists(os.path.join(os.path.expanduser("~"), "virtualbox")): vbox_path = os.path.join(os.path.expanduser("~"), "virtualbox") else: STREAM.error(" -> Virtual box catalog not found!") STREAM.warning(" -> You may specify it directly by adding 'openstack_vbox_catalog' attribute" " in your configuration file") return None STREAM.debug(" -> VirtualBox directory: %s" % vbox_path) vm_path = None for paths, dirs, files in os.walk(vbox_path): if self.vm_name in dirs: vm_path = os.path.join(paths, self.vm_name) return vm_path if vm_path is None: STREAM.error("VirtualMachine directory(%s) not found in the VirtualBox catalog directory tree(%s)!\n" "Make sure that the VirtualMachine name you specify in parameter(vm_name) exists" " in the directory tree (%s)" % (self.vm_name, vbox_path, vbox_path)) return None
def parse_playbooks(self): STREAM.debug(" -> ---------- Playbooks.") ansible_playbooks = [ playbook.strip() for playbook in self.ansible_playbooks.split(",") ] STREAM.debug(" -> Playbooks: %s" % ansible_playbooks) for playbook in ansible_playbooks: if not os.path.exists(playbook): raise Exception("the playbook: %s could not be found" % playbook) if not os.path.isfile(playbook): raise Exception( "the playbook: %s does not appear to be a file" % playbook) return ansible_playbooks
def generate_from_path(path): """ Generating config based on path to Virtual box """ cfg = os.path.join(LoadSettings.WORK_DIR, "generated.ini") config = ConfigParser() config.read(cfg) for vm in os.listdir(path): if os.path.isdir(os.path.join(path, vm)): config.add_section(vm) config.set(vm, "type", "vm") config.set(vm, "vm_name", vm) with open(cfg, "w") as conf: config.write(conf) STREAM.success("Generated %s" % cfg)
def try_connect(ssh): """ Recursive function to enable multiple connection attempts """ try: ssh.connect(self.ssh_server, port=int(self.ssh_port), username=self.ssh_user, password=self.ssh_password) STREAM.success(" -> Connection established") except Exception as err: STREAM.warning(" -> Fail (%s)" % err) if "ecdsakey" in str(err): STREAM.warning("ECDSAKey error, try to fix.") Popen('ssh-keygen -f %s -R "[%s]:%s"' % (os.path.join( os.path.expanduser("~"), ".ssh/known_hosts"), self.ssh_server, self.ssh_port), shell=True, stdout=PIPE, stderr=PIPE).communicate() if self.connect_tries > 20: raise paramiko.ssh_exception.SSHException( "Connection retries limit exceed!") self.connect_tries += 1 STREAM.info(" -> Connection retry %s:" % self.connect_tries) sleep(15) try_connect(ssh)
def _port_check(self, ip, port): conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) conn.settimeout(5) try: conn.connect((ip, port)) conn.send("check") response = conn.recv(50) STREAM.debug(" -> Check port %s: %s" % (self.ANSIBLE_PORT, response)) return True except Exception as err: STREAM.debug(" -> Check port %s: %s" % (self.ANSIBLE_PORT, err)) return False finally: conn.close()
def command_exec(self, ssh, command, stdin=""): """ Method to execute remote command via ssh connection """ STREAM.info(" -> Executing command: %s" % command) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command) ssh_stdin.write(stdin) ssh_stdin.flush() stdout = ssh_stdout.read() stderr = ssh_stderr.read() STREAM.debug(self.get_decoded(stdout)) if len(stderr) > 0: STREAM.debug(self.get_decoded(stderr)) exit_code = ssh_stdout.channel.recv_exit_status() STREAM.debug(" -> Command exitcode: %s" % exit_code) if exit_code == 0: STREAM.success(" -> Command executed successfully") else: raise Exception("Executed command exit status not 0")
def create_vagrant_template(self): """ Method to create Vagrantfile template """ STREAM.debug("==> Create Vagrantfile.") template = """ Vagrant::Config.run do |config| # This Vagrantfile is auto-generated by `vagrant package` to contain # the MAC address of the box. Custom configuration should be placed in # the actual `Vagrantfile` in this box. config.vm.base_mac = "0800274B29D3" end # Load include vagrant file if it exists after the auto-generated # so it can override any of the settings include_vagrantfile = File.expand_path("../include/_Vagrantfile", __FILE__) load include_vagrantfile if File.exist?(include_vagrantfile) """ with open(os.path.join(self.tmp_dir, "Vagrantfile"), "w") as vagrant_file: vagrant_file.write(template)
def delete_snapshot(self): def delete_snap(uuid): result = Popen('VBoxManage snapshot %s delete %s' % (self.vm_name, uuid), shell=True, stdout=PIPE, stderr=PIPE).communicate() STREAM.debug(result) def deletor(recursion_depth): snapshots = self.get_snapshots_list() STREAM.debug(" -> VirtualMachine snapshots: %s" % snapshots) if self.snapshot_name not in snapshots.values(): return for uuid, name in snapshots.items(): if name == self.snapshot_name: delete_snap(uuid) if recursion_depth == 0: return recursion_depth -= 1 deletor(recursion_depth) STREAM.debug(" -> Delete existed snapshots with name: '%s'" % self.snapshot_name) deletor(5)
def cache_image(self, nova, depth=3): """ Method to cache image on one random node. """ server = self.create_instance(nova) STREAM.debug(" -> Created instance: %s" % server) # if recursion will not breaked, whatever keyword will be terminated by vmaker timeout. while True: sleep(2) status = self.get_instance_status(nova, server.id) STREAM.debug(" -> Creation status: %s" % status) if status == "ACTIVE": self.delete_instance(nova, server) STREAM.success(" -> Image has been cached.") break elif status == "ERROR": self.delete_instance(nova, server) STREAM.warning(" -> Unexpected error while launch instance") if depth == 0: break STREAM.warning(" -> Trying to cache image again.") self.cache_image(nova, depth=depth - 1) break
def noforce_stop(self): STREAM.info("==> Attempting to gracefull shutdown VirtualMachine") if not self.check_vm_status(): STREAM.info(" -> VirtualMachine is already stopped") return process = Popen("VBoxManage controlvm %s acpipowerbutton" % self.vm_name, shell=True, stdout=PIPE, stderr=PIPE).communicate() stderr = process[1] if len(stderr) > 0: raise Exception(stderr) while 1: rvms = Popen("VBoxManage list runningvms | awk '{print $1}'", shell=True, stdout=PIPE, stderr=PIPE) data = rvms.stdout.read() if self.vm_name not in data: break
def _get_timeout(): """ The function searches for a timeout for the keyword termination """ try: ttk = getattr(self.current_vm_obj, "%s_timeout" % action) LoggerOptions.set_component("Core") LoggerOptions.set_action(None) STREAM.debug(" Assigned 'timeout' for action: %s = %s min" % (action, ttk)) LoggerOptions.set_component(self.current_vm_obj.__name__) LoggerOptions.set_action(action) except AttributeError: ttk = LoadSettings.TIMEOUT LoggerOptions.set_component("Core") LoggerOptions.set_action(None) STREAM.debug( " Parameter 'timeout' not assigned, for action (%s), using global: %s min" % (action, ttk)) LoggerOptions.set_component(self.current_vm_obj.__name__) LoggerOptions.set_action(action) ttk = int(ttk) * 60 return ttk