def copy_boot(self): cluster = Cluster(mongo_db = self._mongo_db) image_path = str(self.get('path')) kernver = str(self.get('kernver')) tmp_path = '/tmp' # in chroot env initrdfile = str(self.name) + '-initramfs-' + kernver kernfile = str(self.name) + '-vmlinuz-' + kernver path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None path = str(path) user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) shutil.copy(image_path + '/boot/initramfs-' + kernver + '.img', path_to_store + '/' + initrdfile) shutil.copy(image_path + '/boot/vmlinuz-' + kernver, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) self._logger.warning("Boot files was copied, but luna module might not being added to initrd. Please check /etc/dracut.conf.d in image") return True
def __init__(self, name = None, mongo_db = None, create = False, id = None, path = '', kernver = '', kernopts = '', grab_list = 'grab_default_centos.lst'): """ create - shoulld be True if we need create osimage path - path to / of the image/ can be ralative, if needed (will be converted to absolute) kernver - kernel version (will be checked on creation) kernopt - kernel options grab_list - rsync exclude list for grabbing live node to image """ self._logger.debug("Arguments to function '{}".format(self._debug_function())) self._collection_name = 'osimage' mongo_doc = self._check_name(name, mongo_db, create, id) if bool(kernopts) and type(kernopts) is not str: self._logger.error("Kernel options should be 'str' type") raise RuntimeError self._keylist = {'path': type(''), 'kernver': type(''), 'kernopts': type(''), 'kernmodules': type(''), 'dracutmodules': type(''), 'tarball': type(''), 'torrent': type(''), 'kernfile': type(''), 'initrdfile': type(''), 'grab_exclude_list': type(''), 'grab_filesystems': type('')} if create: cluster = Cluster(mongo_db = self._mongo_db) path = os.path.abspath(path) path_suspected_doc = self._mongo_collection.find_one({'path': path}) if path_suspected_doc and path_suspected_doc['path'] == path: self._logger.error("Cannot create 'osimage' with the same 'path' as name='{}' has".format(path_suspected_doc['name'])) raise RuntimeError if kernver == 'ANY': try: kernver = self.get_package_ver(path, 'kernel')[0] except: pass if not self._check_kernel(path, kernver): raise RuntimeError grab_list_path = cluster.get('path') + '/templates/' + grab_list if not os.path.isfile(grab_list_path): self._logger.error("'{}' is not a file.".format(grab_list_path)) raise RuntimeError with open(grab_list_path) as lst: grab_list_content = lst.read() mongo_doc = {'name': name, 'path': path, 'kernver': kernver, 'kernopts': kernopts, 'dracutmodules': 'luna,-i18n,-plymouth', 'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler', 'grab_exclude_list': grab_list_content, 'grab_filesystems': '/,/boot'} self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) self.link(cluster) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id) self._logger = logging.getLogger(__name__ + '.' + self._name)
def run(self): counter = self.interval cluster = Cluster(mongo_db = self._mongo_db) while self.active: if counter >= self.interval: if cluster.is_active(): self.update() else: self.logger.info("This is passive node. Doing nothing.") time.sleep(60) counter = 0 counter += 1 time.sleep(1)
def run(self): counter = self.interval cluster = Cluster(mongo_db = self._mongo_db) while self.active: if counter >= self.interval: if cluster.is_active(): self.update() else: self.logger.info("This is passive node. Doing nothing.") time.sleep(60) counter = 0 counter += 1 time.sleep(1)
def __init__(self, name=None, mongo_db=None, create=False, id=None, network=None, ip=None, comment=''): """ network - the network the device is connected to ip - device's ip """ self.log.debug("function args {}".format(self._debug_function())) # Define the schema used to represent otherdev objects self._collection_name = 'otherdev' self._keylist = {'comment': type('')} # Check if this device is already present in the datastore # Read it if that is the case dev = self._get_object(name, mongo_db, create, id) if create: cluster = Cluster(mongo_db=self._mongo_db) if not network: connected = {} elif not ip: err_msg = "IP needs to be specified" self.log.error(err_msg) raise RuntimeError, err_msg else: net = Network(name=network, mongo_db=self._mongo_db) ipnum = net.reserve_ip(ip, ignore_errors=False) if not ipnum: err_msg = "Unable to allocate IP in network" self.log.error(err_msg) raise RuntimeError, err_msg connected = {str(net.DBRef.id): ipnum} # Store the new device in the datastore dev = {'name': name, 'connected': connected, 'comment': comment} self.log.debug("Saving dev '{}' to the datastore".format(dev)) self.store(dev) # Link this device to its dependencies and the current cluster self.link(cluster) if connected and net: self.link(net) self.log = logging.getLogger('otherdev.' + self._name)
def create_torrent(self): # TODO check if root tarball_uid = self.get('tarball') if not tarball_uid: self.log.error("No tarball in DB.") return False cluster = Cluster(mongo_db=self._mongo_db) tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz" if not os.path.exists(tarball): self.log.error("Wrong path in DB.") return False tracker_address = cluster.get('frontend_address') if tracker_address == '': self.log.error("Tracker address needs to be configured.") return False tracker_port = cluster.get('frontend_port') if tracker_port == 0: self.log.error("Tracker port needs to be configured.") return False user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid old_cwd = os.getcwd() os.chdir(os.path.dirname(tarball)) uid = str(uuid.uuid4()) torrentfile = cluster.get('path') + "/torrents/" + uid + ".torrent" fs = libtorrent.file_storage() libtorrent.add_files(fs, os.path.basename(tarball)) t = libtorrent.create_torrent(fs) if cluster.get('frontend_https'): proto = 'https' else: proto = 'http' t.add_tracker((proto + "://" + str(tracker_address) + ":" + str(tracker_port) + "/announce")) t.set_creator(torrent_key) t.set_comment(uid) libtorrent.set_piece_hashes(t, ".") f = open(torrentfile, 'w') f.write(libtorrent.bencode(t.generate())) f.close() os.chown(torrentfile, user_id, grp_id) self.set('torrent', str(uid)) os.chdir(old_cwd) return True
def __init__(self, name = None, mongo_db = None, create = False, id = None, prescript = None, bmcsetup = None, bmcnetwork = None, partscript = None, osimage = None, interfaces = None, postscript = None, boot_if = None, torrent_if = None): """ prescript - preinstall script bmcsetup - bmcsetup options bmcnetwork - used for bmc networking partscript - parition script osimage - osimage interfaces - list of the newtork interfaces postscript - postinstall script """ self._logger.debug("Arguments to function '{}".format(self._debug_function())) self._collection_name = 'group' mongo_doc = self._check_name(name, mongo_db, create, id) self._keylist = {'prescript': type(''), 'partscript': type(''), 'postscript': type(''), 'boot_if': type(''), 'torrent_if': type('')} if create: cluster = Cluster(mongo_db = self._mongo_db) (bmcobj, bmcnetobj) = (None, None) if bool(bmcsetup): bmcobj = BMCSetup(bmcsetup).DBRef if bool(bmcnetwork): bmcnetobj = Network(bmcnetwork, mongo_db = self._mongo_db).DBRef osimageobj = OsImage(osimage) if bool(interfaces) and type(interfaces) is not type([]): self._logger.error("'interfaces' should be list") raise RuntimeError if_dict = {} if not bool(interfaces): interfaces = [] for interface in interfaces: if_dict[interface] = {'network': None, 'params': ''} if not bool(partscript): partscript = "mount -t tmpfs tmpfs /sysroot" if not bool(prescript): prescript = "" if not bool(postscript): postscript = """cat <<EOF>>/sysroot/etc/fstab tmpfs / tmpfs defaults 0 0 EOF""" mongo_doc = {'name': name, 'prescript': prescript, 'bmcsetup': bmcobj, 'bmcnetwork': bmcnetobj, 'partscript': partscript, 'osimage': osimageobj.DBRef, 'interfaces': if_dict, 'postscript': postscript, 'boot_if': boot_if, 'torrent_if': torrent_if} self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) self.link(cluster) if bmcobj: self.link(bmcobj) if bmcnetobj: self.link(bmcnetobj) self.link(osimageobj) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id) self._logger = logging.getLogger('group.' + self._name)
def copy_boot(self): image_path = self.get('path') kernver = self.get('kernver') initrdfile = self.name + '-initramfs-' + kernver kernfile = self.name + '-vmlinuz-' + kernver cluster = Cluster(mongo_db=self._mongo_db) user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid path = cluster.get('path') path_to_store = path + "/boot" initrd_path = image_path + '/boot/initramfs-' + kernver + '.img' kernel_path = image_path + '/boot/vmlinuz-' + kernver if not os.path.isfile(kernel_path): self.log.error("Unable to find kernel in {}".format(kernel_path)) return False if not os.path.isfile(initrd_path): self.log.error("Unable to find initrd in {}".format(initrd_path)) return False if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) shutil.copy(initrd_path, path_to_store + '/' + initrdfile) shutil.copy(kernel_path, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) self.log.warning(("Boot files were copied, but luna modules might not " "have been added to initrd. Please check " "/etc/dracut.conf.d in the image")) return True
def copy_boot(self): image_path = self.get('path') kernver = self.get('kernver') initrdfile = self.name + '-initramfs-' + kernver kernfile = self.name + '-vmlinuz-' + kernver cluster = Cluster(mongo_db=self._mongo_db) user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid path = cluster.get('path') path_to_store = path + "/boot" initrd_path = image_path + '/boot/initramfs-' + kernver + '.img' kernel_path = image_path + '/boot/vmlinuz-' + kernver if not os.path.isfile(kernel_path): self.log.error("Unable to find kernel in {}".format(kernel_path)) return False if not os.path.isfile(initrd_path): self.log.error("Unable to find initrd in {}".format(initrd_path)) return False if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) shutil.copy(initrd_path, path_to_store + '/' + initrdfile) shutil.copy(kernel_path, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) self.log.warning(("Boot files were copied, but luna modules might not " "have been added to initrd. Please check " "/etc/dracut.conf.d in the image")) return True
def __init__(self, name=None, mongo_db=None, create=False, id=None, NETWORK=None, PREFIX=None, ns_hostname=None, ns_ip=None): """ create - should be True if we need create osimage NETWORK - network PREFIX - should be specified network bits or """ self._logger.debug("Arguments to function '{}".format( self._debug_function())) self._collection_name = 'network' self._keylist = { 'NETWORK': long, 'PREFIX': type(0), 'ns_hostname': type(''), 'ns_ip': type('') } mongo_doc = self._check_name(name, mongo_db, create, id) if create: cluster = Cluster(mongo_db=self._mongo_db) num_net = self.get_base_net(NETWORK, PREFIX) if not num_net: self._logger.error("Cannot compute NETWORK/PREFIX") raise RuntimeError if not ns_hostname: ns_hostname = self._guess_ns_hostname() freelist = [{'start': 1, 'end': (1 << (32 - PREFIX)) - 2}] mongo_doc = { 'name': name, 'NETWORK': num_net, 'PREFIX': PREFIX, 'freelist': freelist, 'ns_hostname': ns_hostname, 'ns_ip': None } self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) self.link(cluster) if not ns_ip: ns_ip = self.relnum_to_ip(freelist[0]['end']) if not ns_ip: self._logger.error("Cannot configure IP address for NS") else: self.set('ns_ip', ns_ip) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id) self._logger = logging.getLogger(__name__ + '.' + self._name)
def __init__(self, name=None, mongo_db=None, create=False, id=None, group=None, localboot=False, setupbmc=True, service=False): """ name - can be ommited group - group belongs to; should be specified Flags localboot - boot from localdisk setupbmc - whether we need to setup ipmi on install service - do not perform install but boot to installer (dracut environment) """ self._logger.debug("Arguments to function '{}".format( self._debug_function())) self._collection_name = 'node' if not bool(name) and bool(create): name = self._generate_name(mongo_db=mongo_db) mongo_doc = self._check_name(name, mongo_db, create, id) self._keylist = { 'port': type(''), 'localboot': type(True), 'setupbmc': type(True), 'service': type(True) } if create: cluster = Cluster(mongo_db=self._mongo_db) group = Group(group, mongo_db=self._mongo_db) mongo_doc = { 'name': name, 'group': group.DBRef, 'interfaces': None, 'mac': None, 'switch': None, 'port': None, 'localboot': localboot, 'setupbmc': setupbmc, 'service': service } self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) for interface in group._get_json()['interfaces']: self.add_ip(interface) self.add_bmc_ip() self.link(group) self.link(cluster) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id) self._logger = logging.getLogger(__name__ + '.' + self._name)
def _generate_name(self, mongo_db): cluster = Cluster(mongo_db) prefix = cluster.get('nodeprefix') digits = cluster.get('nodedigits') back_links = cluster.get_back_links() max_num = 0 for link in back_links: if not link['collection'] == self._collection_name: continue node = Node(id = link['DBRef'].id, mongo_db = mongo_db) name = node.name try: nnode = int(name.lstrip(prefix)) except ValueError: continue if nnode > max_num: max_num = nnode ret_name = prefix + str(max_num + 1).zfill(digits) return ret_name
def _generate_name(self, mongo_db): cluster = Cluster(mongo_db) prefix = cluster.get('nodeprefix') digits = cluster.get('nodedigits') back_links = cluster.get_back_links() max_num = 0 for link in back_links: if not link['collection'] == self._collection_name: continue node = Node(id = link['DBRef'].id, mongo_db = mongo_db) name = node.name try: nnode = int(name.lstrip(prefix)) except ValueError: continue if nnode > max_num: max_num = nnode ret_name = prefix + str(max_num + 1).zfill(digits) return ret_name
def __init__(self, name=None, mongo_db=None, create=False, id=None, userid=3, user='******', password='******', netchannel=1, mgmtchannel=1): """ userid - default user id user - username password - pasword netchannel - network channel mgmtchannel - management channel """ self._logger.debug("Arguments to function '{}".format( self._debug_function())) self._collection_name = 'bmcsetup' mongo_doc = self._check_name(name, mongo_db, create, id) self._keylist = { 'userid': type(0), 'user': type(''), 'password': type(''), 'netchannel': type(0), 'mgmtchannel': type(0) } if create: cluster = Cluster(mongo_db=self._mongo_db) passed_vars = inspect.currentframe().f_locals for key in self._keylist: if type(passed_vars[key]) is not self._keylist[key]: self._logger.error("Argument '{}' should be '{}'".format( key, self._keylist[key])) raise RuntimeError mongo_doc = { 'name': name, 'userid': userid, 'user': user, 'password': password, 'netchannel': netchannel, 'mgmtchannel': mgmtchannel } self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) self.link(cluster) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id) self._logger = logging.getLogger(__name__ + '.' + self._name)
def create_torrent(self): # TODO check if root tarball_uid = self.get('tarball') if not tarball_uid: self.log.error("No tarball in DB.") return False cluster = Cluster(mongo_db=self._mongo_db) tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz" if not os.path.exists(tarball): self.log.error("Wrong path in DB.") return False tracker_address = cluster.get('frontend_address') if tracker_address == '': self.log.error("Tracker address needs to be configured.") return False tracker_port = cluster.get('frontend_port') if tracker_port == 0: self.log.error("Tracker port needs to be configured.") return False user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid old_cwd = os.getcwd() os.chdir(os.path.dirname(tarball)) uid = str(uuid.uuid4()) torrentfile = cluster.get('path') + "/torrents/" + uid + ".torrent" fs = libtorrent.file_storage() libtorrent.add_files(fs, os.path.basename(tarball)) t = libtorrent.create_torrent(fs) if cluster.get('frontend_https'): proto = 'https' else: proto = 'http' t.add_tracker((proto + "://" + str(tracker_address) + ":" + str(tracker_port) + "/announce")) t.set_creator(torrent_key) t.set_comment(uid) libtorrent.set_piece_hashes(t, ".") f = open(torrentfile, 'w') f.write(libtorrent.bencode(t.generate())) f.close() os.chown(torrentfile, user_id, grp_id) self.set('torrent', str(uid)) os.chdir(old_cwd) return True
def render_script(self, name): scripts = ['boot', 'install'] if name not in scripts: self.log.error( "'{}' is not correct script. Valid options are: '{}'" .format(name, scripts) ) return None cluster = Cluster(mongo_db=self._mongo_db) self._get_group() path = cluster.get('path') tloader = template.Loader(path + '/templates') if cluster.get('frontend_https'): protocol = 'https' else: protocol = 'http' server_ip = cluster.get('frontend_address') server_port = cluster.get('frontend_port') if name == 'boot': p = self.boot_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port return tloader.load('templ_nodeboot.cfg').generate(p=p) if name == 'install': p = self.install_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port res = tloader.load('templ_install.cfg').generate(p=p) return res
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db = self._mongo_db) path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None tracker_address = cluster.get('fronend_address') if tracker_address == '': self._logger.error("Tracker address needs to be configured.") return None tracker_port = cluster.get('frontend_port') if tracker_port == 0: self._logger.error("Tracker port needs to be configured.") return None user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None path_to_store = path + "/torrents" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) uid = str(uuid.uuid4()) tarfile_path = path_to_store + "/" + uid + ".tgz" image_path = self.get('path') try: tar_out = subprocess.Popen(['/usr/bin/tar', '-C', image_path + '/.', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '-c', '-z', '-f', tarfile_path, '.'], stderr=subprocess.PIPE) # dirty, but 4 times faster stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: os.remove(tarfile_path) sys.stdout.write('\r') return None os.chown(tarfile_path, user_id, grp_id) os.chmod(tarfile_path, 0644) self.set('tarball', str(uid)) return True
def create_torrent(self): # TODO check if root tarball_uid = self.get('tarball') cluster = Cluster(mongo_db = self._mongo_db) if not bool(tarball_uid): self._logger.error("No tarball in DB.") return None tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz" if not os.path.exists(tarball): self._logger.error("Wrong path in DB.") return None tracker_address = cluster.get('frontend_address') if tracker_address == '': self._logger.error("Tracker address needs to be configured.") return None tracker_port = cluster.get('frontend_port') if tracker_port == 0: self._logger.error("Tracker port needs to be configured.") return None user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None #group = cluster.get('group') #if not group: # self._logger.error("Group needs to be configured.") # return None user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid old_cwd = os.getcwd() os.chdir(os.path.dirname(tarball)) uid = str(uuid.uuid4()) torrentfile = str(cluster.get('path')) + "/torrents/" + uid fs = libtorrent.file_storage() libtorrent.add_files(fs, os.path.basename(tarball)) t = libtorrent.create_torrent(fs) t.add_tracker("http://" + str(tracker_address) + ":" + str(tracker_port) + "/announce") t.set_creator(torrent_key) t.set_comment(uid) libtorrent.set_piece_hashes(t, ".") f = open(torrentfile, 'w') f.write(libtorrent.bencode(t.generate())) f.close() self.set('torrent', str(uid)) os.chown(torrentfile, user_id, grp_id) shutil.move(torrentfile, torrentfile + ".torrent") os.chdir(old_cwd) return True
def __init__(self, name = None, mongo_db = None, create = False, id = None, network = None, ip = None): """ netwwork - network device connected ip - ip of the switch """ self._logger.debug("Arguments to function '{}".format(self._debug_function())) self._collection_name = 'otherdev' mongo_doc = self._check_name(name, mongo_db, create, id) self._keylist = {} if create: cluster = Cluster(mongo_db = self._mongo_db) passed_vars = inspect.currentframe().f_locals for key in self._keylist: if type(passed_vars[key]) is not self._keylist[key]: self._logger.error("Argument '{}' should be '{}'".format(key, self._keylist[key])) raise RuntimeError if not bool(network): connected = {} else: if not bool(ip): self._logger.error("IP needs to be specified") raise RuntimeError net = Network(name = network, mongo_db = self._mongo_db) ip = net.reserve_ip(ip, ignore_errors = False) if not bool(ip): raise RuntimeError connected = {str(net.DBRef.id): ip} mongo_doc = { 'name': name, 'connected': connected} self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) self.link(cluster) if bool(connected): self.link(net) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id)
def __init__(self, name = None, mongo_db = None, create = False, id = None, network = None, ip = None, read = 'public', rw = 'private', oid = None): """ ip - ip of the switch read - read community rw - rw community oid - could be, for instance .1.3.6.1.2.1.17.7.1.2.2.1.2 .1.3.6.1.2.1.17.4.3.1.2 .1.3.6.1.2.1.17.7.1.2.2 .1.3.6.1.2.1.17.4.3.1.2 """ self._logger.debug("Arguments to function '{}".format(self._debug_function())) self._collection_name = 'switch' mongo_doc = self._check_name(name, mongo_db, create, id) self._keylist = { 'ip': type(''), 'read': type(''), 'rw': type(''), 'oid': type(''), 'network': type('')} if create: cluster = Cluster(mongo_db = self._mongo_db) passed_vars = inspect.currentframe().f_locals for key in self._keylist: if type(passed_vars[key]) is not self._keylist[key]: self._logger.error("Argument '{}' should be '{}'".format(key, self._keylist[key])) raise RuntimeError net = Network(name = network, mongo_db = self._mongo_db) ip = net.reserve_ip(ip) if not bool(ip): self._logger.error("Could not acquire ip for switch.") raise RuntimeError mongo_doc = { 'name': name, 'network': net.DBRef, 'ip': ip, 'read': read, 'rw': rw, 'oid': oid} self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) self.link(cluster) self.link(net) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id)
def render_script(self, name): scripts = ['boot', 'install'] if name not in scripts: self.log.error( "'{}' is not correct script. Valid options are: '{}'".format( name, scripts)) return None cluster = Cluster(mongo_db=self._mongo_db) self._get_group() path = cluster.get('path') tloader = template.Loader(path + '/templates') if cluster.get('frontend_https'): protocol = 'https' else: protocol = 'http' server_ip = cluster.get('frontend_address') server_port = cluster.get('frontend_port') if name == 'boot': p = self.boot_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port return tloader.load('templ_nodeboot.cfg').generate(p=p) if name == 'install': p = self.install_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port res = tloader.load('templ_install.cfg').generate(p=p) return res
def __init__(self, name=None, mongo_db=None, create=False, id=None, group=None, localboot=False, setupbmc=True, service=False, comment=''): """ name - optional group - the group the node belongs to; required FLAGS: localboot - boot from localdisk setupbmc - whether we setup ipmi on install service - do not install, boot into installer (dracut environment) """ self.log.debug("function {} args".format(self._debug_function())) # Define the schema used to represent node objects self._collection_name = 'node' self._keylist = { 'port': type(''), 'localboot': type(True), 'setupbmc': type(True), 'service': type(True), 'mac': type(''), 'comment': type(''), } # Check if this node is already present in the datastore # Read it if that is the case node = self._get_object(name, mongo_db, create, id) self.group = None if group: if type(group) == Group: self.group = group if type(group) == str: self.group = Group(group, mongo_db=self._mongo_db) if create: if not group: err_msg = "Group needs to be specified when creating node." self.log.error(err_msg) raise RuntimeError, err_msg cluster = Cluster(mongo_db=self._mongo_db) # If a name is not provided, generate one if not bool(name): name = self._generate_name(cluster, mongo_db=mongo_db) # Store the new node in the datastore node = { 'name': name, 'group': self.group.DBRef, 'interfaces': {}, 'mac': None, 'switch': None, 'port': None, 'localboot': localboot, 'setupbmc': setupbmc, 'service': service, 'comment': comment } self.log.debug("Saving node '{}' to the datastore".format(node)) self.store(node) # check if we are able to allocate IPs for all interfaces error_on_alloc = False configured_interfaces = [] for interface_name in self.group.list_ifs().keys(): configured_interfaces.append(interface_name) if not self.add_ip(interface_name): error_on_alloc = True break if error_on_alloc: # roll back allocations and throw exception for interface_name in configured_interfaces[:-1]: self.del_ip(interface_name) err_msg = ("Unable to allocate IP address " + "for interface '{}'").format( configured_interfaces[-1]) self.log.error(err_msg) raise RuntimeError, err_msg # Link this node to its group and the current cluster self.link(self.group) self.link(cluster) if group: # check if group specified is the group node belongs to if self.group.DBRef != self._json['group']: err_msg = ("DBref of the group is not the same " + "the node node belongs to") self.log.error(err_msg) raise RuntimeError, err_msg self.log = logging.getLogger(__name__ + '.' + self._json['name'])
def __init__(self, name=None, mongo_db=None, create=False, id=None, NETWORK=None, PREFIX=None, ns_hostname=None, ns_ip=None): """ create - should be True if we need create a network NETWORK - network PREFIX - the prefix in a networks CIDR format """ self.log.debug("function {} args".format(self._debug_function())) # Define the schema used to represent network objects self._collection_name = 'network' self._keylist = {'NETWORK': long, 'PREFIX': type(''), 'ns_hostname': type(''), 'ns_ip': type('')} # Check if this network is already present in the datastore # Read it if that is the case net = self._check_name(name, mongo_db, create, id) if create: cluster = Cluster(mongo_db=self._mongo_db) num_subnet = utils.ip.get_num_subnet(NETWORK, PREFIX) flist = [{'start': 1, 'end': (1 << (32 - int(PREFIX))) - 2}] # Try to guess the nameserver hostname if none provided if not ns_hostname: ns_hostname = utils.ip.guess_ns_hostname() # Define a new mongo document net = {'name': name, 'NETWORK': num_subnet, 'PREFIX': PREFIX, 'freelist': flist, 'ns_hostname': ns_hostname, 'ns_ip': None} # Store the new network in the datastore self.log.debug("Saving net '{}' to the datastore".format(net)) self._name = name self._id = self._mongo_collection.insert(net) self._DBRef = DBRef(self._collection_name, self._id) # Link this network to the current cluster self.link(cluster) # If no IP address is provided for the nameserver, default to # the cluster's frontend address if ns_ip is None: ns_ip = utils.ip.reltoa(num_subnet, flist[0]['end']) self.set('ns_ip', ns_ip) else: self._name = net['name'] self._id = net['_id'] self._DBRef = DBRef(self._collection_name, self._id) self.log = logging.getLogger(__name__ + '.' + self._name)
def __init__(self, name=None, mongo_db=None, create=False, id=None, prescript='', bmcsetup=None, partscript='', osimage=None, interfaces=[], postscript='', torrent_if=None, domain=None, comment=''): """ prescript - preinstall script bmcsetup - bmcsetup options partscript - parition script osimage - osimage interfaces - list of the newtork interfaces postscript - postinstall script """ self.log.debug("function args {}".format(self._debug_function())) # Define the schema used to represent group objects self._collection_name = 'group' self._keylist = { 'prescript': type(''), 'partscript': type(''), 'postscript': type(''), 'torrent_if': type(''), 'comment': type(''), } # Check if this group is already present in the datastore # Read it if that is the case group = self._get_object(name, mongo_db, create, id) if create: cluster = Cluster(mongo_db=self._mongo_db) osimageobj = OsImage(osimage, mongo_db=self._mongo_db) (bmcobj, domainobj) = (None, None) if bmcsetup: bmcobj = BMCSetup(bmcsetup, mongo_db=self._mongo_db).DBRef if domain: domainobj = Network(domain, mongo_db=self._mongo_db).DBRef if interfaces and type(interfaces) is not list: err_msg = "'interfaces' should be list" self.log.error(err_msg) raise RuntimeError, err_msg if not interfaces: interfaces = [] if_dict = {} for interface in interfaces: if_dict[uuid.uuid4().hex] = { 'name': interface, 'network': { '4': None, '6': None }, 'params': '' } if not partscript: partscript = "mount -t tmpfs tmpfs /sysroot" if not postscript: postscript = ("cat << EOF >> /sysroot/etc/fstab\n" "tmpfs / tmpfs defaults 0 0\n" "EOF") # Store the new group in the datastore group = { 'name': name, 'prescript': prescript, 'bmcsetup': bmcobj, 'partscript': partscript, 'osimage': osimageobj.DBRef, 'interfaces': if_dict, 'postscript': postscript, 'domain': domainobj, 'torrent_if': torrent_if, 'comment': comment, } self.log.debug("Saving group '{}' to the datastore".format(group)) self.store(group) # Link this group to its dependencies and the current cluster self.link(cluster) if bmcobj: self.link(bmcobj) if domainobj: self.link(domainobj) self.link(osimageobj) self.log = logging.getLogger('group.' + self._name) self._networks = {}
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid path_to_store = path + "/torrents" if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) image_path = self.get('path') real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) uid = str(uuid.uuid4()) tarfile = uid + ".tgz" try: # dirty, but 4 times faster tar_out = subprocess.Popen([ '/usr/bin/tar', '-C', '/', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '--exclude=./tmp/' + tarfile, '--use-compress-program=/usr/bin/pigz', '-c', '-f', '/tmp/' + tarfile, '.' ], stderr=subprocess.PIPE) stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == exceptions.KeyboardInterrupt: self.log.error('Keyboard interrupt.') else: self.log.error(exc_value) self.log.debug(traceback.format_exc()) if os.path.isfile('/tmp/' + tarfile): os.remove('/tmp/' + tarfile) sys.stdout.write('\r') os.fchdir(real_root) os.chroot(".") os.close(real_root) return False os.fchdir(real_root) os.chroot(".") os.close(real_root) # copy image, so permissions and selinux contexts # will be inherited from parent folder shutil.copy(image_path + '/tmp/' + tarfile, path_to_store) os.remove(image_path + '/tmp/' + tarfile) os.chown(path_to_store + '/' + tarfile, user_id, grp_id) os.chmod(path_to_store + '/' + tarfile, 0644) self.set('tarball', str(uid)) return True
def __init__(self, name=None, mongo_db=None, create=False, id=None, path='', kernver='', kernopts='', comment='', grab_list='grab_default_centos.lst'): """ path - path to / of the image (will be converted to absolute) kernver - kernel version (will be checked on creation) kernopt - kernel options grab_list - rsync exclude list for grabbing live node to image """ self.log.debug("function args {}".format(self._debug_function())) # Define the schema used to represent osimage objects self._collection_name = 'osimage' self._keylist = {'path': type(''), 'kernver': type(''), 'kernopts': type(''), 'kernmodules': type(''), 'dracutmodules': type(''), 'tarball': type(''), 'torrent': type(''), 'kernfile': type(''), 'initrdfile': type(''), 'grab_exclude_list': type(''), 'grab_filesystems': type(''), 'comment': type('')} # Check if this osimage is already present in the datastore # Read it if that is the case osimage = self._get_object(name, mongo_db, create, id) if bool(kernopts) and type(kernopts) is not str: err_msg = "Kernel options should be 'str' type" self.log.error(err_msg) raise RuntimeError, err_msg if create: cluster = Cluster(mongo_db=self._mongo_db) path = os.path.abspath(path) duplicate = self._mongo_collection.find_one({'path': path}) if duplicate: err_msg = ("Path belongs to osimage '{}'" .format(duplicate['name'])) self.log.error(err_msg) raise RuntimeError, err_msg if not os.path.isdir(path): err_msg = "'{}' is not a valid directory".format(path) self.log.error(err_msg) raise RuntimeError, err_msg kernels = self.get_package_ver(path, 'kernel') if not kernels: err_msg = "No kernels installed in '{}'".format(path) self.log.error(err_msg) raise RuntimeError, err_msg elif not kernver: kernver = kernels[0] elif kernver not in kernels: err_msg = "Available kernels are '{}'".format(kernels) self.log.error(err_msg) raise RuntimeError, err_msg grab_list_path = cluster.get('path') + '/templates/' + grab_list if not os.path.isfile(grab_list_path): err_msg = "'{}' is not a file.".format(grab_list_path) self.log.error(err_msg) raise RuntimeError, err_msg with open(grab_list_path) as lst: grab_list_content = lst.read() # Store the new osimage in the datastore osimage = {'name': name, 'path': path, 'kernver': kernver, 'kernopts': kernopts, 'kernfile': '', 'initrdfile': '', 'dracutmodules': 'luna,-i18n,-plymouth', 'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler', 'grab_exclude_list': grab_list_content, 'grab_filesystems': '/,/boot', 'comment': comment} self.log.debug("Saving osimage '{}' to the datastore" .format(osimage)) self.store(osimage) # Link this osimage to its dependencies and the current cluster self.link(cluster) self.log = logging.getLogger(__name__ + '.' + self._name)
def pack_boot(self): def mount(source, target, fs): subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target]) def umount(source): subprocess.Popen(['/usr/bin/umount', source]) def prepare_mounts(path): mount('devtmpfs', path + '/dev', 'devtmpfs') mount('proc', path + '/proc', 'proc') mount('sysfs', path + '/sys', 'sysfs') def cleanup_mounts(path): umount(path + '/dev') umount(path + '/proc') umount(path + '/sys') tmp_path = '/tmp' # in chroot env image_path = self.get('path') kernver = self.get('kernver') kernfile = self.name + '-vmlinuz-' + kernver initrdfile = self.name + '-initramfs-' + kernver cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) modules_add = [] modules_remove = [] drivers_add = [] drivers_remove = [] dracutmodules = self.get('dracutmodules') if dracutmodules: for i in dracutmodules.split(','): if i[0] != '-': modules_add.extend(['--add', i]) else: modules_remove.extend(['--omit', i[1:]]) kernmodules = self.get('kernmodules') if kernmodules: for i in kernmodules.split(','): if i[0] != '-': drivers_add.extend(['--add-drivers', i]) else: drivers_remove.extend(['--omit-drivers', i[1:]]) prepare_mounts(image_path) real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) chroot_path = os.open("/", os.O_DIRECTORY) os.fchdir(chroot_path) dracut_succeed = True create = None try: dracut_modules = subprocess.Popen( ['/usr/sbin/dracut', '--kver', kernver, '--list-modules'], stdout=subprocess.PIPE) luna_exists = False while dracut_modules.poll() is None: line = dracut_modules.stdout.readline() if line.strip() == 'luna': luna_exists = True break if not luna_exists: err_msg = ("No luna dracut module in osimage '{}'".format( self.name)) self.log.error(err_msg) raise RuntimeError, err_msg dracut_cmd = (['/usr/sbin/dracut', '--force', '--kver', kernver] + modules_add + modules_remove + drivers_add + drivers_remove + [tmp_path + '/' + initrdfile]) create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE) while create.poll() is None: line = create.stdout.readline() except: dracut_succeed = False if create and create.returncode: dracut_succeed = False if not create: dracut_succeed = False os.fchdir(real_root) os.chroot(".") os.close(real_root) os.close(chroot_path) cleanup_mounts(image_path) if not dracut_succeed: self.log.error("Error while building initrd.") return False initrd_path = image_path + tmp_path + '/' + initrdfile kernel_path = image_path + '/boot/vmlinuz-' + kernver if not os.path.isfile(kernel_path): self.log.error("Unable to find kernel in {}".format(kernel_path)) return False if not os.path.isfile(initrd_path): self.log.error("Unable to find initrd in {}".format(initrd_path)) return False # copy initrd file to inherit perms from parent folder shutil.copy(initrd_path, path_to_store + '/' + initrdfile) os.remove(initrd_path) shutil.copy(kernel_path, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) return True
def pack_boot(self): def mount(source, target, fs): subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target]) def umount(source): subprocess.Popen(['/usr/bin/umount', source]) def prepare_mounts(path): mount('devtmpfs', path + '/dev', 'devtmpfs') mount('proc', path + '/proc', 'proc') mount('sysfs', path + '/sys', 'sysfs') def cleanup_mounts(path): umount(path + '/dev') umount(path + '/proc') umount(path + '/sys') tmp_path = '/tmp' # in chroot env image_path = self.get('path') kernver = self.get('kernver') kernfile = self.name + '-vmlinuz-' + kernver initrdfile = self.name + '-initramfs-' + kernver cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) modules_add = [] modules_remove = [] drivers_add = [] drivers_remove = [] dracutmodules = self.get('dracutmodules') if dracutmodules: for i in dracutmodules.split(','): if i[0] != '-': modules_add.extend(['--add', i]) else: modules_remove.extend(['--omit', i[1:]]) kernmodules = self.get('kernmodules') if kernmodules: for i in kernmodules.split(','): if i[0] != '-': drivers_add.extend(['--add-drivers', i]) else: drivers_remove.extend(['--omit-drivers', i[1:]]) prepare_mounts(image_path) real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) chroot_path = os.open("/", os.O_DIRECTORY) os.fchdir(chroot_path) dracut_succeed = True create = None try: dracut_modules = subprocess.Popen(['/usr/sbin/dracut', '--kver', kernver, '--list-modules'], stdout=subprocess.PIPE) luna_exists = False while dracut_modules.poll() is None: line = dracut_modules.stdout.readline() if line.strip() == 'luna': luna_exists = True break if not luna_exists: err_msg = ("No luna dracut module in osimage '{}'" .format(self.name)) self.log.error(err_msg) raise RuntimeError, err_msg dracut_cmd = (['/usr/sbin/dracut', '--force', '--kver', kernver] + modules_add + modules_remove + drivers_add + drivers_remove + [tmp_path + '/' + initrdfile]) create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE) while create.poll() is None: line = create.stdout.readline() except: dracut_succeed = False if create and create.returncode: dracut_succeed = False if not create: dracut_succeed = False os.fchdir(real_root) os.chroot(".") os.close(real_root) os.close(chroot_path) cleanup_mounts(image_path) if not dracut_succeed: self.log.error("Error while building initrd.") return False initrd_path = image_path + tmp_path + '/' + initrdfile kernel_path = image_path + '/boot/vmlinuz-' + kernver if not os.path.isfile(kernel_path): self.log.error("Unable to find kernel in {}".format(kernel_path)) return False if not os.path.isfile(initrd_path): self.log.error("Unable to find initrd in {}".format(initrd_path)) return False # copy initrd file to inherit perms from parent folder shutil.copy(initrd_path, path_to_store + '/' + initrdfile) os.remove(initrd_path) shutil.copy(kernel_path, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) return True
def __init__(self, name=None, mongo_db=None, create=False, id=None, userid=3, user='******', password='******', netchannel=1, mgmtchannel=1, comment=''): """ userid - default user id user - username password - pasword netchannel - network channel mgmtchannel - management channel """ self.log.debug("function args '{}".format(self._debug_function())) # Define the schema used to represent BMC configuration objects self._collection_name = 'bmcsetup' self._keylist = { 'userid': type(0), 'user': type(''), 'password': type(''), 'netchannel': type(0), 'mgmtchannel': type(0), 'comment': type(''), } # Check if this BMC config is already present in the datastore # Read it if that is the case bmc = self._get_object(name, mongo_db, create, id) if create: cluster = Cluster(mongo_db=self._mongo_db) # Verify that all the keywords arguments have the correct types # as specified in the self._keylist args = locals() for key in self._keylist: if type(args[key]) is not self._keylist[key]: err_msg = ("Argument '{}' should be '{}'".format( key, self._keylist[key])) self.log.error(err_msg) raise RuntimeError, err_msg # Store the new BMC config in the datastore bmc = { 'name': name, 'userid': userid, 'user': user, 'password': password, 'netchannel': netchannel, 'mgmtchannel': mgmtchannel, 'comment': comment } self.log.debug("Saving BMC conf '{}' to the datastore".format(bmc)) self.store(bmc) # Link this BMC config to the current cluster self.link(cluster) self.log = logging.getLogger(__name__ + '.' + self._name)
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db = self._mongo_db) path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None tracker_address = cluster.get('fronend_address') if tracker_address == '': self._logger.error("Tracker address needs to be configured.") return None tracker_port = cluster.get('frontend_port') if tracker_port == 0: self._logger.error("Tracker port needs to be configured.") return None user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None #group = cluster.get('group') #if not group: # self._logger.error("Group needs to be configured.") # return None path_to_store = path + "/torrents" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) uid = str(uuid.uuid4()) tarfile_path = path_to_store + "/" + uid + ".tgz" image_path = self.get('path') #tarball = tarfile.open(tarfile_path, "w:gz") #tarball.add(image_path, arcname=os.path.basename(image_path + "/.")) #tarball.close() try: tar_out = subprocess.Popen(['/usr/bin/tar', '-C', image_path + '/.', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '-c', '-z', '-f', tarfile_path, '.'], stderr=subprocess.PIPE) # dirty, but 4 times faster stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: os.remove(tarfile_path) sys.stdout.write('\r') return None os.chown(tarfile_path, user_id, grp_id) os.chmod(tarfile_path, 0644) self.set('tarball', str(uid)) return True
def __init__(self, name=None, mongo_db=None, create=False, id=None, path='', kernver='', kernopts='', comment='', grab_list='grab_default_centos.lst'): """ path - path to / of the image (will be converted to absolute) kernver - kernel version (will be checked on creation) kernopt - kernel options grab_list - rsync exclude list for grabbing live node to image """ self.log.debug("function args {}".format(self._debug_function())) # Define the schema used to represent osimage objects self._collection_name = 'osimage' self._keylist = { 'path': type(''), 'kernver': type(''), 'kernopts': type(''), 'kernmodules': type(''), 'dracutmodules': type(''), 'tarball': type(''), 'torrent': type(''), 'kernfile': type(''), 'initrdfile': type(''), 'grab_exclude_list': type(''), 'grab_filesystems': type(''), 'comment': type('') } # Check if this osimage is already present in the datastore # Read it if that is the case osimage = self._get_object(name, mongo_db, create, id) if bool(kernopts) and type(kernopts) is not str: err_msg = "Kernel options should be 'str' type" self.log.error(err_msg) raise RuntimeError, err_msg if create: cluster = Cluster(mongo_db=self._mongo_db) path = os.path.abspath(path) duplicate = self._mongo_collection.find_one({'path': path}) if duplicate: err_msg = ("Path belongs to osimage '{}'".format( duplicate['name'])) self.log.error(err_msg) raise RuntimeError, err_msg if not os.path.isdir(path): err_msg = "'{}' is not a valid directory".format(path) self.log.error(err_msg) raise RuntimeError, err_msg kernels = self.get_package_ver(path, 'kernel') if not kernels: err_msg = "No kernels installed in '{}'".format(path) self.log.error(err_msg) raise RuntimeError, err_msg elif not kernver: kernver = kernels[0] elif kernver not in kernels: err_msg = "Available kernels are '{}'".format(kernels) self.log.error(err_msg) raise RuntimeError, err_msg grab_list_path = cluster.get('path') + '/templates/' + grab_list if not os.path.isfile(grab_list_path): err_msg = "'{}' is not a file.".format(grab_list_path) self.log.error(err_msg) raise RuntimeError, err_msg with open(grab_list_path) as lst: grab_list_content = lst.read() # Store the new osimage in the datastore osimage = { 'name': name, 'path': path, 'kernver': kernver, 'kernopts': kernopts, 'kernfile': '', 'initrdfile': '', 'dracutmodules': 'luna,-i18n,-plymouth', 'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler', 'grab_exclude_list': grab_list_content, 'grab_filesystems': '/,/boot', 'comment': comment } self.log.debug( "Saving osimage '{}' to the datastore".format(osimage)) self.store(osimage) # Link this osimage to its dependencies and the current cluster self.link(cluster) self.log = logging.getLogger(__name__ + '.' + self._name)
def pack_boot(self): def mount(source, target, fs): subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target]) #ret = ctypes.CDLL('libc.so.6', use_errno=True).mount(source, target, fs, 0, options) #if ret < 0: # errno = ctypes.get_errno() # raise RuntimeError("Error mounting {} ({}) on {} with options '{}': {}". # format(source, fs, target, options, os.strerror(errno))) def umount(source): subprocess.Popen(['/usr/bin/umount', source]) #ret = ctypes.CDLL('libc.so.6', use_errno=True).umount(source) #if ret < 0: # errno = ctypes.get_errno() # raise RuntimeError("Error umounting {}: .". # format(source, os.strerror(errno))) def prepare_mounts(path): mount('devtmpfs', path + '/dev', 'devtmpfs') mount('proc', path + '/proc', 'proc') mount('sysfs', path + '/sys', 'sysfs') def cleanup_mounts(path): umount(path + '/dev') umount(path + '/proc') umount(path + '/sys') cluster = Cluster(mongo_db = self._mongo_db) #boot_prefix = '/boot' image_path = str(self.get('path')) kernver = str(self.get('kernver')) tmp_path = '/tmp' # in chroot env initrdfile = str(self.name) + '-initramfs-' + kernver kernfile = str(self.name) + '-vmlinuz-' + kernver #kernel_image = kernel_name + '-' + kernver #kernel_path = image_path + boot_prefix + '/' + kernel_image path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None path = str(path) user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None #group = cluster.get('group') #if not group: # self._logger.error("Group needs to be configured.") # return None path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) modules_add = [] modules_remove = [] drivers_add = [] drivers_remove = [] dracutmodules = self.get('dracutmodules') if dracutmodules: dracutmodules = str(dracutmodules) modules_add = sum([['--add', i] for i in dracutmodules.split(',') if i[0] != '-'], []) modules_remove = sum([['--omit', i[1:]] for i in dracutmodules.split(',') if i[0] == '-'], []) kernmodules = self.get('kernmodules') if kernmodules: kernmodules = str(kernmodules) drivers_add = sum([['--add-drivers', i] for i in kernmodules.split(',') if i[0] != '-'], []) drivers_remove = sum([['--omit-drivers', i[1:]] for i in kernmodules.split(',') if i[0] == '-'], []) prepare_mounts(image_path) real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) try: dracut_modules = subprocess.Popen(['/usr/sbin/dracut', '--kver', kernver, '--list-modules'], stdout=subprocess.PIPE) luna_exists = False while dracut_modules.poll() is None: line = dracut_modules.stdout.readline() if line.strip() == 'luna': luna_exists = True if not luna_exists: self._logger.error("No luna dracut module in osimage '{}'".format(self.name)) raise RuntimeError dracut_cmd = ['/usr/sbin/dracut', '--force', '--kver', kernver] + modules_add + modules_remove + drivers_add + drivers_remove + [tmp_path + '/' + initrdfile] dracut_create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE) while dracut_create.poll() is None: line = dracut_create.stdout.readline() except: os.fchdir(real_root) os.chroot(".") os.close(real_root) cleanup_mounts(image_path) try: pass #os.remove(image_path + '/' + tmp_path + '/' + initrdfile) except: pass return None os.fchdir(real_root) os.chroot(".") os.close(real_root) cleanup_mounts(image_path) shutil.copy(image_path + tmp_path + '/' + initrdfile, path_to_store) shutil.copy(image_path + '/boot/vmlinuz-' + kernver, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile)
def __init__(self, name=None, mongo_db=None, create=False, id=None, version=None, NETWORK=None, PREFIX=None, ns_hostname=None, ns_ip=None, comment=''): """ create - should be True if we need create a network NETWORK - network PREFIX - the prefix in a networks CIDR format """ self.log.debug("function {} args".format(self._debug_function())) # Define the schema used to represent network objects self._collection_name = 'network' self._keylist = { 'PREFIX': type(0), 'ns_hostname': type(''), 'include': type(''), 'rev_include': type(''), 'comment': type(''), } # Check if this network is already present in the datastore # Read it if that is the case net = self._get_object(name, mongo_db, create, id) if create: if not version: version = utils.ip.get_ip_version(NETWORK) if version == 0: err_msg = ("Unable to determine protocol version " + "for given network") self.log.error(err_msg) raise RuntimeError, err_msg if version not in [4, 6]: err_msg = "IP version should be 4 or 6" self.log.error(err_msg) raise RuntimeError, err_msg maxbits = 32 if version == 6: maxbits = 128 self.version = version self.maxbits = maxbits cluster = Cluster(mongo_db=self._mongo_db) num_subnet = utils.ip.get_num_subnet(NETWORK, PREFIX, self.version) flist = self._flist_to_str([{ 'start': 1, 'end': (1 << (self.maxbits - int(PREFIX))) - 2 }]) # Try to guess the nameserver hostname if none provided if not ns_hostname: ns_hostname = utils.ip.guess_ns_hostname() # Store the new network in the datastore if self.version == 6: num_subnet = str(num_subnet) # Try to find duplicate net if utils.helpers.find_duplicate_net(num_subnet, mongo_db=self._mongo_db): err_msg = ("Network {}/{} is defined already".format( NETWORK, PREFIX)) self.log.error(err_msg) raise RuntimeError, err_msg net = { 'name': name, 'NETWORK': num_subnet, 'PREFIX': PREFIX, 'freelist': flist, 'ns_hostname': ns_hostname, 'ns_ip': None, 'version': version, 'include': None, 'rev_include': None, 'comment': comment } self.log.debug("Saving net '{}' to the datastore".format(net)) self.store(net) self._convert_to_int() # Link this network to the current cluster self.link(cluster) # If no IP address is provided for the nameserver, default to # the cluster's frontend address if ns_ip is None: ns_ip = utils.ip.reltoa(num_subnet, int(flist[0]['end']), self.version) self.set('ns_ip', ns_ip) self.version = self._json['version'] self.maxbits = 32 if self.version == 6: self.maxbits = 128 self._convert_to_int() self.log = logging.getLogger(__name__ + '.' + self._name)
def __init__(self, name=None, mongo_db=None, create=False, id=None, network=None, ip=None, read='public', rw='private', oid=None, comment=''): """ ip - ip of the switch read - read community rw - rw community oid - could be, for instance .1.3.6.1.2.1.17.7.1.2.2.1.2 .1.3.6.1.2.1.17.4.3.1.2 .1.3.6.1.2.1.17.7.1.2.2 .1.3.6.1.2.1.17.4.3.1.2 """ self.log.debug("function args {}".format(self._debug_function())) # Define the schema used to represent switch objects self._collection_name = 'switch' self._keylist = { 'read': type(''), 'rw': type(''), 'oid': type(''), 'comment': type(''), } # Check if this switch is already present in the datastore # Read it if that is the case switch = self._get_object(name, mongo_db, create, id) if create: cluster = Cluster(mongo_db=self._mongo_db) if not network: err_msg = "Network must be provided" self.log.error(err_msg) raise RuntimeError, err_msg if not name: err_msg = "Name must be provided" self.log.error(err_msg) raise RuntimeError, err_msg net = Network(name=network, mongo_db=self._mongo_db) ip = net.reserve_ip(ip) if not ip: err_msg = "Could not acquire ip for switch" self.log.error(err_msg) raise RuntimeError, err_msg # Store the new switch in the datastore switch = { 'name': name, 'network': net.DBRef, 'ip': ip, 'read': read, 'rw': rw, 'oid': oid, 'comment': comment } self.log.debug( "Saving switch '{}' to the datastore".format(switch)) self.store(switch) # Link this switch to its dependencies and the current cluster self.link(cluster) self.link(net) self.log = logging.getLogger('switch.' + self._name)
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid path_to_store = path + "/torrents" if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) image_path = self.get('path') real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) uid = str(uuid.uuid4()) tarfile = uid + ".tgz" try: # dirty, but 4 times faster tar_out = subprocess.Popen( [ '/usr/bin/tar', '-C', '/', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '--exclude=./tmp/' + tarfile, '--use-compress-program=/usr/bin/pigz', '-c', '-f', '/tmp/' + tarfile, '.' ], stderr=subprocess.PIPE ) stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == exceptions.KeyboardInterrupt: self.log.error('Keyboard interrupt.') else: self.log.error(exc_value) self.log.debug(traceback.format_exc()) if os.path.isfile('/tmp/' + tarfile): os.remove('/tmp/' + tarfile) sys.stdout.write('\r') os.fchdir(real_root) os.chroot(".") os.close(real_root) return False os.fchdir(real_root) os.chroot(".") os.close(real_root) # copy image, so permissions and selinux contexts # will be inherited from parent folder shutil.copy(image_path + '/tmp/' + tarfile, path_to_store) os.remove(image_path + '/tmp/' + tarfile) os.chown(path_to_store + '/' + tarfile, user_id, grp_id) os.chmod(path_to_store + '/' + tarfile, 0644) self.set('tarball', str(uid)) return True