class EnvAttribute(AnsibleAttribute): atype = "env" keys = Dict([VariableKeys(String("some value"), example="name")]) defaults = {} def get_vars(self): return self.settings()
class CopyAttribute(NodeAttribute, need.NeedGuestFS): atype = "copy" requires = ["image"] keys = Dict( [VariableKeys(String("/path/to/dest"), example="/path/to/source")]) def spawn(self): for dest, source in self.settings().items(): if not os.path.isabs(dest): self.warn( "destination path `{}` needs to be absolute".format(dest)) continue if not os.path.isabs(source): source = os.path.join(os.getcwd(), source) if not os.path.exists(source): self.warn("source path `{}` does not exist".format(source)) continue self.say("{} => {}".format(source, dest)) if os.path.isdir(source): tmp = util.make_temp_name(source + dest) with tarfile.open(tmp, "a") as tar: tar.add(source, arcname=os.path.basename(dest)) self.guest().tar_in(tmp, os.path.split(dest)[0]) else: with open(source, "r") as src: self.guest().write(dest, src.read())
class ModeAttribute(NetworkAttribute): atype = "mode" defaults = {'type': 'nat'} keys = Or([ String("nat"), Dict([ RequiredKey('type', String("route")), RequiredKey('dev', String("eth0")) ]) ]) def validate_settings(self): NetworkAttribute.validate_settings(self) if self._get_mode() not in ['nat', 'route']: raise error.DefError("Unknown network mode '{}'".format( self._get_mode())) def spawn(self): template = 'mode.xml' if self._get_dev(): template = 'mode_route_dev.xml' xml = self.template(template) self.add_xml( xml.safe_substitute({ 'type': self._get_mode(), 'dev': self._get_dev() })) def _get_mode(self): if isinstance(self.settings(), dict): return self.settings('type') return self.settings() def _get_dev(self): if isinstance(self.settings(), dict): return self.settings('dev') return None
class ShellAttribute(NodeAttribute, need.NeedSSH, need.NeedLibvirt): atype = "shell" requires = ["ssh", "user"] keys = List( Dict([ Key("start", String("scripts/start.sh")), Key("stop", String("scripts/stop.sh")), Key("spawn", String("scripts/spawn.sh")), Key("suspend", String("scripts/suspend.sh")), Key("resume", String("scripts/resume.sh")), Key("shell", String("/bin/bash")) ])) def get_default_user(self): return self.other_attribute("user").get_default_user() def get_default_host(self): return self.domain_get_ip(self.component_entity()) def after_start(self): for (script, shell) in self._get_shell_scripts("start"): self.say("[start] running shell provisioner ({})".format(script)) self._run_shell(script, shell) def suspend(self): for (script, shell) in self._get_shell_scripts("suspend"): self.say("[suspend] running shell provisioner ({})".format(script)) self._run_shell(script, shell) def after_resume(self): for (script, shell) in self._get_shell_scripts("resume"): self.say("[resume] running shell provisioner ({})".format(script)) self._run_shell(script, shell) def stop(self): for (script, shell) in self._get_shell_scripts("stop"): self.say("[stop] running shell provisioner ({})".format(script)) self._run_shell(script, shell) def _get_shell_scripts(self, action): scripts = [] for script in self.settings(): if action in script: shell = "/bin/bash" if "shell" in script: shell = script["shell"] scripts.append((script[action], shell)) return scripts def _run_shell(self, script, shell="/bin/bash"): if not os.path.isfile(script): raise error.NotFound( "Could not find privisioner shell script ({})".format(script)) ssh = self.default_ssh() script_location = ssh.copy_to_tmp(script) ssh.run(shell + " " + script_location)
class RunAttribute(AnsibleAttribute, NeedIO): atype = "run" keys = String("/path/to/playbook.yml") def validate(self): AnsibleAttribute.validate(self) if not os.path.exists(self.settings()): raise error.NotFound("Could not find ansible playbook: " "{} no such file or directory".format( self.settings())) def get_playbook(self, tmp): playbook = os.path.join(tmp, "playbook.yml") self.io().copy(self.settings(), playbook) return playbook
class DirectoryAttribute(PoolAttribute): atype = "directory" pool_type = "dir" keys = String("/path/to/pool/directory") def validate(self): PoolAttribute.validate(self) if not os.path.isabs(self.settings()): raise error.ValidatorError( "{} needs to be a valid absolut path.".format(self.entity())) def spawn(self): tpl = self.template("directory.xml") self.add_xml(tpl.safe_substitute({"path": self.settings()})) def after_spawn(self): pool = self.get_pool(self.component_entity()) if not os.path.exists(self.settings()): pool.build()
class PoolAttribute(Attribute, need.NeedLibvirt): atype = "pool" keys = String("default-pool") defaults = "default" def spawn(self): pool = self.used_pool() if not pool.isActive(): pool.create() def used_pool(self): pool = self.get_pool(self.settings(), raise_exception=False) if not pool: if not self.has_component("pool", self.settings()): raise error.NotFound("pool {} does not exist.") for i in range(self.global_get("global/retry_pool", 20)): pool = self.get_pool(name, raise_exception=False) if pool: break self.counted( i, "Waiting for pool {} to become ready.".format( self.settings())) sleep(self.global_get("global/wait", 3)) raise error.ExecError("Coult not start pool {}!".format( self.settings())) return pool def used_pool_name(self): return self.settings() def used_pool_type(self): pool = self.used_pool() xml = etree.fromstring(pool.XMLDesc()) return xml.attrib["type"]
class IPAttribute(NetworkAttribute): keys = Dict([ RequiredKey("ip", Ip("192.168.124.1")), Key("netmask", String("255.255.255.0")), Key( "dhcp", Dict([ RequiredKey("start", Ip("192.168.124.2")), RequiredKey("end", Ip("192.168.124.254")) ])), ]) def spawn(self): settings = { "family": self.atype, "ip": self.settings("ip"), "netmask": self.settings("netmask", self.default_netmask), "start": self.settings("dhcp/start"), "end": self.settings("dhcp/end") } xml = self.template(self.atype + ".xml") self.add_xml(xml.safe_substitute(settings))
class SSHAttribute(NodeAttribute, need.NeedGuestFS): atype = "ssh" requires = ["image", "user"] defaults = None keys = Dict([ Key('copy-key', Dict([ Key('ssh-keys', List(String("/path/to/public_key_file"))), RequiredKey('users', List(String("username"))) ])), Key('distribute-keys', Dict([ RequiredKey('users', List(String("username"))), Key('hosts', List(String("hostname.local"))), Key('same-hosts', Bool(True)) ])) ]) def spawn(self): if not self.settings(): return guest = self.guest() copy_key = self.settings('copy-key') if copy_key: self._add_keys_to_domain(guest, copy_key) def _add_keys_to_domain(self, guest, copy_key): keys = self._get_public_keys() users = self.guest_get_users() for target in copy_key['users']: if target not in users: self.warn("try to add ssh key to not existing user {}. Ignoring!".format(target)) continue user = users[target] user_ssh_dir = os.path.join(user['home'], ".ssh") authorized_file = os.path.join(user_ssh_dir, "authorized_keys") self.say("local keys => {}".format(target)) if not guest.is_dir(user_ssh_dir): guest.mkdir(user_ssh_dir) guest.chown(user['uid'], user['gid'], user_ssh_dir) guest.write_append(authorized_file, "".join(keys)) guest.chown(user['uid'], user['gid'], authorized_file) # FIXME: Find better way to deal with selinux labels if guest.get_selinux(): guest.sh("chcon -R unconfined_u:object_r:user_home_t:s0 {}".format(user_ssh_dir)) def _get_public_keys(self): ssh_keys = self.settings('copy-key/ssh-keys') if ssh_keys: return ssh_keys # fetch keys from local user ssh_keys = [] home = os.path.expanduser('~') pub_keys = [os.path.join(home, '.ssh/id_rsa.pub'), os.path.join(home, '.ssh/id_ecdsa.pub')] for key_path in pub_keys: if os.path.isfile(key_path): with open(key_path, 'r') as hdl: ssh_keys.append(hdl.read()) return ssh_keys def _parse_passwd(self, passwd): parsed = {} for line in passwd: split = line.split(":") if len(split) != 7: continue parsed[split[0]] = split[-2] return parsed def _distribute_keys(self): hosts = self.settings("distribute-keys/hosts") users = self.settings("distribute-keys/users") same_host = self.settings("distribute-keys/same-hosts", False) guest_users = self.guest_get_users() # check if user wants current host to distribute his ssh keys if hosts and self.component_entity() not in hosts: return #keys = self.global_share("ssh_distribute_keys", default={}) for user in users: if user not in guest_users: self.warn("distribute key failed: user {} does " "not exist".format(user)) continue
class HostsAttribute(AnsibleAttribute, NeedLibvirt, NeedIO, NeedSSH): atype = "hosts" defaults = None keys = Or([ List(String("hosts")), Dict([VariableKeys(List(String("hosts")), example="hostname")]) ]) def _fetch_ips(self, hosts): def _to_tuple(host): return (host, self.domain_get_ip(host, quiet=True)) return util.in_parallel(3, hosts, _to_tuple) def _check_ssh_servers(self, hosts): def _check_ssh(host, ip): if self.ssh_host_alive(ip): return (host, ip) return None return filter(None, util.in_parallel(3, hosts, lambda x: _check_ssh(*x))) def generate_inventory(self, tmp): known_hosts = {} hosts = self._get_hosts() inventory = os.path.join(tmp, "inventory") to_write = [] all_hosts = [] # fetch all hostnames map(all_hosts.extend, self._get_hosts().values()) all_hosts = list(set(all_hosts)) # fetch ips and check ssh connectifity ips = self._fetch_ips(all_hosts) ips = self._check_ssh_servers(ips) for group, hosts in self._get_hosts().items(): to_write.append("[{}]".format(group)) for host in hosts: ip = self.domain_get_ip(host, quiet=self.is_verbose()) cmpnt = self.get_component(host) if ip is None: raise error.ExecError("Could not fetch all required ip " "addresses") entry = [ host, "ansible_connection=ssh", "ansible_port=22", "ansible_host={}".format(ip), "ansible_user={}".format(cmpnt.ssh_user()) ] to_write.append("\t".join(entry)) with self.io().open(inventory, "w") as inv: inv.write("\n".join(to_write)) return inventory def _get_hosts(self): if self.settings() is None: return self._generate_hosts() if isinstance(self.settings(), dict): return self.settings() # a list is given if isinstance(self.settings(), list): return {"all": self.settings()} raise error.Bug("Can not create host list for ansible")
class NetworkAttribute(NodeAttribute, need.NeedLibvirt): atype = "network" defaults = "default" keys = Or([ String("default"), Dict([ RequiredKey("source", String("default")), RequiredKey("ip", Ip("192.168.124.87")) ]) ]) def network_name(self): if self._need_ipv4(): return self.settings("source") return self.settings() def start(self): network = self._get_delayed_network(self.network_name()) if network.isActive(): return self.say("starting network...") network.create() self.success("network started!") def after_start(self): network = self._get_delayed_network(self.network_name()) if self._need_ipv4(): mac = self._get_mac_address() self._remove_mac(network, mac, self.settings("ip")) self._announce_static_ip(network, mac, self.settings("ip")) def stop(self): network = self._get_delayed_network(self.network_name()) if self._need_ipv4(): mac = self._get_mac_address() self._remove_mac(network, mac, self.settings("ip")) def spawn(self): network = self._get_delayed_network(self.network_name()) if not network: raise error.NotFound("Network {} for domain " "{}".format(self.network_name(), self.component_entity())) if not network.isActive(): self.start() self.add_xml('devices', self._gen_xml()) def _gen_xml(self): xml = self.template('network.xml') return xml.safe_substitute({'network': self.network_name()}) def _get_mac_address(self): def _uses_network(iface): return iface.find("source").attrib["network"] == self.network_name() node = self.get_domain(self.component_entity()) desc = etree.fromstring(node.XMLDesc()) ifaces = filter(_uses_network, desc.findall("devices/interface")) if len(ifaces) == 0: raise error.NotFound("Could not find domain interface") # FIXME: Add multiple interface support # When multiple interfaces using the same network mac = ifaces[0].find("mac") if mac is None: raise error.NotFound("Could not find interface mac address") return mac.attrib["address"] def _need_ipv4(self): return isinstance(self.settings(), dict) def _announce_static_ip(self, network, mac, ip): command = libvirt.VIR_NETWORK_UPDATE_COMMAND_ADD_LAST flags = (libvirt.VIR_NETWORK_UPDATE_AFFECT_CONFIG | libvirt.VIR_NETWORK_UPDATE_AFFECT_LIVE) section = libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST xml = "<host mac='{}' name='{}' ip='{}' />".format(mac, "xii-" + self.component_entity(), ip) try: network.update(command, section, -1, xml, flags) except libvirt.libvirtError: return False return True def _remove_mac(self, network, mac, ip): command = libvirt.VIR_NETWORK_UPDATE_COMMAND_DELETE flags = (libvirt.VIR_NETWORK_UPDATE_AFFECT_CONFIG | libvirt.VIR_NETWORK_UPDATE_AFFECT_LIVE) section = libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST xml = "<host mac='{}' name='{}' ip='{}' />".format(mac, "xii-" + self.component_entity(), ip) try: network.update(command, section, -1, xml, flags) except libvirt.libvirtError: return False return True def _get_delayed_network(self, name): network = self.get_network(name, raise_exception=False) if not network: if not self.has_component("network", name): raise error.NotFound("Could not find network ({})" .format(name)) # wait for network to become ready for _ in range(self.global_get("global/retry_network", 20)): network = self.get_network(name, raise_exception=False) if network: return network sleep(self.global_get("global/wait", 3)) raise error.ExecError("Network {} has not become ready in " "time. Giving up".format(name)) return network
class ImageAttribute(Attribute, need.NeedIO, need.NeedLibvirt): atype = "image" requires = ['pool'] keys = String("~/images/openSUSE-leap-42.2.qcow2") def get_tmp_volume_path(self): ext = os.path.splitext(self.settings())[1] return self.get_temp_path("image" + ext) def create(self): # create image store if needed self.io().ensure_path_exists(self._image_store_path()) pool_name = self.other_attribute("pool").used_pool_name() # pool_type = self.other_attribute("pool").used_pool_type() volume = self.get_volume(pool_name, self.component_entity(), raise_exception=False) if volume: self._remove_volume(volume) if not self.io().exists(self._image_path()): self._fetch_image() self.say("cloning image...") self.io().copy(self._image_path(), self.get_tmp_volume_path()) def after_spawn(self): pool = self.other_attribute("pool").used_pool() size = self.io().stat(self.get_tmp_volume_path()).st_size volume_tpl = self.template("volume.xml") xml = volume_tpl.safe_substitute({ "name": self.component_entity(), "capacity": size }) # FIXME: Add error handling volume = pool.createXML(xml) def read_handler(stream, data, file_): return file_.read(data) self.say("importing...") with open(self.get_tmp_volume_path(), 'r') as image: stream = self.virt().newStream(0) volume.upload(stream, 0, 0, 0) stream.sendAll(read_handler, image) stream.finish() pool = self.other_attribute("pool").used_pool() disk_tpl = self.template("disk.xml") xml = disk_tpl.safe_substitute({ "pool": pool.name(), "volume": self.component_entity() }) self.parent().add_xml('devices', xml) def destroy(self): pool = self.other_attribute("pool").used_pool() volume = self.get_volume(pool.name(), self.component_entity(), raise_exception=False) if volume: self._remove_volume(volume) def _image_store_path(self): home = self.io().user_home() return paths.xii_home(home, 'images') def _image_path(self): name = util.md5digest(self.settings()) self.parent().add_meta("image", name) return os.path.join(self._image_store_path(), name) def _remove_volume(self, volume): volume.wipe(0) volume.delete(0) def _fetch_image(self): _pending.acquire() if self.io().exists(self._image_path()): _pending.release() return if self.settings().startswith("http"): self.say("downloading image...") self.io().download(self.settings(), self._image_path()) else: self.say("copy image...") self.io().copy(self.settings(), self._image_path()) (md5, sha256) = self._generate_hashes() stats = { "source": self.settings(), "type": os.path.splitext(self.settings())[1][1:], "size": self.io().stat(self._image_path()).st_size, "md5": md5, "sha256": sha256, "added": time.time() } util.yaml_write(self._image_path() + ".yml", stats) _pending.release() def _generate_hashes(self): try: self.say("generate image checksum...") md5_hash = hashlib.md5() sha256_hash = hashlib.sha256() with open(self._image_path(), 'rb') as hdl: buf = hdl.read(65536) while len(buf) > 0: md5_hash.update(buf) sha256_hash.update(buf) buf = hdl.read(65536) return (md5_hash.hexdigest(), sha256_hash.hexdigest()) except IOError as err: raise error.ExecError("Could not create validation hashes")
class SSHMountAttribute(NodeAttribute, need.NeedGuestFS, need.NeedSSH, need.NeedIO): atype = "sshmount" requires = ["image", "ssh", "user", "network"] key_path = ".ssh/xii-sshfs.key" keys = Dict([ VariableKeys(Dict([ RequiredKey("source", String("/path/to/source/directory")), Key("user", String("xii")) ]), example="/path/to/dest/directory") ]) def sshfs_key_path(self, name): home = self.guest_user_home(name) if not home: return None return os.path.join(home, self.key_path) def spawn(self): self._check_sshfs_compability() required = self._get_required_users() host = self._host_name() _pending.acquire() authed_hosts = self._authorized_hosts() for user, home in required.items(): path = os.path.join(home, self.key_path) sign = user + "@" + host (key, pubkey) = util.generate_rsa_key_pair() # save private key to domain_image self.say("{} => {}".format(user, self.component_entity())) self.guest().write(path, key) for idx, host in enumerate(authed_hosts): # check if public key alreay exists. If so update key if host.find(sign) != -1: self.say("update authorized_keys ({})".format(sign)) authed_hosts[idx] = pubkey + " " + sign break else: self.say("{} => local authorized_keys".format(sign)) authed_hosts.append(pubkey + " " + sign) with open(self._authorized_keys_path(), "w") as auth: authed_hosts = filter(None, authed_hosts) auth.write("\n".join(authed_hosts)) _pending.release() def destroy(self): authed_hosts = self._authorized_hosts() host = self._host_name() if not os.path.exists(self._authorized_keys_path()): return # removing key from authorized_keys _pending.acquire() for mount in self.settings().values(): user = self.io().user() if "user" in mount: user = mount["user"] sign = user + "@" + host for idx, host in enumerate(authed_hosts): if host.find(sign) != -1: del authed_hosts[idx] break with open(self._authorized_keys_path(), "w") as auth: authed_hosts = filter(None, authed_hosts) auth.write("\n".join(authed_hosts)) _pending.release() def after_start(self): self._mount_dirs() pass def stop(self): self._umount_dirs() pass def after_resume(self): self._mount_dirs() pass def suspend(self): self._umount_dirs() pass def _mount_dirs(self): # local connection host = self.network_get_host_ip( self.other_attribute("network").network_name()) key = os.path.join("~", self.key_path) # remote connection ssh = self.default_ssh() for dest, settings in self.settings().items(): source = settings["source"] user = self.io().user() if "user" in settings: user = settings["user"] # make a absolute path if neccessary if not os.path.isabs(source): source = os.path.join(os.getcwd(), source) if not os.path.isabs(dest): dest = os.path.join(ssh.user_home(), dest) if not os.path.isdir(source): self.warn( "sshfs source `{}` is not a directory".format(source)) continue ssh.mkdir(dest) options = ("-o StrictHostKeyChecking=no " "-o UserKnownHostsFile=/dev/null " "-o IdentityFile={}".format(key)) self.say("{} => {}".format(source, dest)) ssh.run("sshfs {}@{}:{} {} {}".format(user, host, source, dest, options)) def _umount_dirs(self): ssh = self.default_ssh() for dest, settings in self.settings().items(): if not os.path.isabs(dest): dest = os.path.join(ssh.user_home(), dest) self.say("unmounting {}...".format(dest)) ssh.shell("fusermount -u {}".format(dest)) def _authorized_keys_path(self): local_home = os.path.expanduser('~') return os.path.join(local_home, ".ssh/authorized_keys") def _authorized_hosts(self): authorized_keys = self._authorized_keys_path() # touch the file if not exists if not os.path.isfile(authorized_keys): with open(authorized_keys, "a"): pass with open(authorized_keys, "r") as auth: content = [line.strip() for line in auth.readlines()] return filter(None, content) def _host_name(self): return self.component_entity() def _check_sshfs_compability(self): sshfs_locations = ["/usr/bin/sshfs"] self.say("checking sshfs compability") for location in sshfs_locations: if self.guest().exists(location): return raise error.NotFound("Image for {} seems to not support sshfs.".format( self.component_entity())) def _get_required_users(self): required = {} users = self.guest_get_users() default = self.default_ssh_user() for mount in self.settings().values(): user = default if "user" in mount: user = mount["user"] if user not in users: self.warn( "can not use {} for sshfs. User does not exist".format( user)) continue # we already prepare this user if user in required: continue required[user] = users[user]["home"] return required
class UserAttribute(NodeAttribute, need.NeedGuestFS): atype = "user" requires = ["image"] default_settings = { "username": "******", "description": "xii generated user", "shell": "/bin/bash", "password": "******", "skel": True, "n": 0 } keys = Dict([ VariableKeys(Dict([ RequiredKey('password', String("password")), Key('description', String("A nice user")), Key('shell', String("/bin/bash")), Key('default', Bool(True)) ]), example="username") ]) def default_user(self): if not self.settings(): return "xii" for name, user in self.settings().items(): if "default" in user: return name name = self.settings().iterkeys().next() self.say("Assuming default user for {} is {}...".format( self.component_entity(), name)) return name def spawn(self): if not self.settings(): return self.say("adding user/s...") guest = self.guest() shadow = guest.cat("/etc/shadow").split("\n") passwd = guest.cat("/etc/passwd").split("\n") groups = guest.cat("/etc/group").split("\n") user_index = 0 for name, settings in self.settings().items(): self.say("adding {}".format(name)) user = self.default_settings.copy() user['username'] = name user['n'] = user_index user.update(settings) gid, groups = self._add_user_to_groups(groups, user) user['gid'] = gid uid, passwd = self._add_user_to_passwd(passwd, user) user['uid'] = uid shadow = self._add_user_to_shadow(shadow, user) self._mk_home(guest, user) user_index += 1 guest.write("/etc/shadow", "\n".join(shadow)) guest.write("/etc/passwd", "\n".join(passwd)) guest.write("/etc/group", "\n".join(groups)) def _add_user_to_shadow(self, shadow, user): new = self._gen_shadow(user['username'], user['password']) for i, line in enumerate(shadow): if line.startswith(user['username'] + ":"): shadow[i] = new return shadow shadow.append(new) return shadow def _add_user_to_passwd(self, passwd, user): # Assumption: Every system already has a root user if user['username'] == "root": return 0, passwd uid = 1100 + user['n'] if 'uid' in user: uid = user['uid'] new = self._gen_passwd(uid, user) for i, line in enumerate(passwd): if line.startswith(user['username'] + ":"): passwd[i] = new return uid, passwd passwd.append(new) return uid, passwd def _add_user_to_groups(self, groups, user): # Since root group is defined in lsb it must be there if user['username'] == 'root': return 0, groups search = user['username'] name = user['username'] gid = 1100 + user['n'] # if group is set use this if 'group' in user: search = user['group'] name = user['group'] # prefer gid over group name if 'gid' in user: search = str(user['gid']) gid = user['gid'] for i, line in enumerate(groups): if ":" + search + ":" in line: if line[-1] != ":": groups[i] += "," groups[i] += user['username'] return gid, groups new = ":".join([name, "x", str(gid), user['username']]) groups.append(new) return gid, groups def _gen_passwd(self, uid, user): home = self._user_home(user) elems = [ user['username'], "x", # password is stored in /etc/shadow str(uid), str(user['gid']), user['description'], home, user['shell'] ] return ":".join(elems) def _mk_home(self, guest, user): home = self._user_home(user) if guest.is_dir(home): return if user['skel']: guest.cp_r('/etc/skel', home) else: guest.mkdir(home) paths = [os.path.join(home, path) for path in guest.ls(home)] paths.append(home) for path in paths: guest.chown(user['uid'], user['gid'], path) def _user_home(self, user): if 'home' in user: return user['home'] return os.path.join('/home', user['username']) def _gen_shadow(self, user, password): salt = self._generate_salt() hashed = crypt.crypt(password, "$6$" + salt + "$") return ":".join([user, hashed, "1", "", "99999", "", "", "", ""]) def _generate_salt(self, length=16): chars = string.ascii_letters + string.digits return ''.join([random.choice(chars) for _ in range(length)])