def copy_boot(self): cluster = Cluster(mongo_db = self._mongo_db) image_path = str(self.get('path')) kernver = str(self.get('kernver')) tmp_path = '/tmp' # in chroot env initrdfile = str(self.name) + '-initramfs-' + kernver kernfile = str(self.name) + '-vmlinuz-' + kernver path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None path = str(path) user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) shutil.copy(image_path + '/boot/initramfs-' + kernver + '.img', path_to_store + '/' + initrdfile) shutil.copy(image_path + '/boot/vmlinuz-' + kernver, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) self._logger.warning("Boot files was copied, but luna module might not being added to initrd. Please check /etc/dracut.conf.d in image") return True
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db = self._mongo_db) path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None tracker_address = cluster.get('fronend_address') if tracker_address == '': self._logger.error("Tracker address needs to be configured.") return None tracker_port = cluster.get('frontend_port') if tracker_port == 0: self._logger.error("Tracker port needs to be configured.") return None user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None path_to_store = path + "/torrents" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) uid = str(uuid.uuid4()) tarfile_path = path_to_store + "/" + uid + ".tgz" image_path = self.get('path') try: tar_out = subprocess.Popen(['/usr/bin/tar', '-C', image_path + '/.', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '-c', '-z', '-f', tarfile_path, '.'], stderr=subprocess.PIPE) # dirty, but 4 times faster stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: os.remove(tarfile_path) sys.stdout.write('\r') return None os.chown(tarfile_path, user_id, grp_id) os.chmod(tarfile_path, 0644) self.set('tarball', str(uid)) return True
def create_torrent(self): # TODO check if root tarball_uid = self.get('tarball') cluster = Cluster(mongo_db = self._mongo_db) if not bool(tarball_uid): self._logger.error("No tarball in DB.") return None tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz" if not os.path.exists(tarball): self._logger.error("Wrong path in DB.") return None tracker_address = cluster.get('frontend_address') if tracker_address == '': self._logger.error("Tracker address needs to be configured.") return None tracker_port = cluster.get('frontend_port') if tracker_port == 0: self._logger.error("Tracker port needs to be configured.") return None user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None #group = cluster.get('group') #if not group: # self._logger.error("Group needs to be configured.") # return None user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid old_cwd = os.getcwd() os.chdir(os.path.dirname(tarball)) uid = str(uuid.uuid4()) torrentfile = str(cluster.get('path')) + "/torrents/" + uid fs = libtorrent.file_storage() libtorrent.add_files(fs, os.path.basename(tarball)) t = libtorrent.create_torrent(fs) t.add_tracker("http://" + str(tracker_address) + ":" + str(tracker_port) + "/announce") t.set_creator(torrent_key) t.set_comment(uid) libtorrent.set_piece_hashes(t, ".") f = open(torrentfile, 'w') f.write(libtorrent.bencode(t.generate())) f.close() self.set('torrent', str(uid)) os.chown(torrentfile, user_id, grp_id) shutil.move(torrentfile, torrentfile + ".torrent") os.chdir(old_cwd) return True
def copy_boot(self): image_path = self.get('path') kernver = self.get('kernver') initrdfile = self.name + '-initramfs-' + kernver kernfile = self.name + '-vmlinuz-' + kernver cluster = Cluster(mongo_db=self._mongo_db) user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid path = cluster.get('path') path_to_store = path + "/boot" initrd_path = image_path + '/boot/initramfs-' + kernver + '.img' kernel_path = image_path + '/boot/vmlinuz-' + kernver if not os.path.isfile(kernel_path): self.log.error("Unable to find kernel in {}".format(kernel_path)) return False if not os.path.isfile(initrd_path): self.log.error("Unable to find initrd in {}".format(initrd_path)) return False if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) shutil.copy(initrd_path, path_to_store + '/' + initrdfile) shutil.copy(kernel_path, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) self.log.warning(("Boot files were copied, but luna modules might not " "have been added to initrd. Please check " "/etc/dracut.conf.d in the image")) return True
def __init__(self, name = None, mongo_db = None, create = False, id = None, path = '', kernver = '', kernopts = '', grab_list = 'grab_default_centos.lst'): """ create - shoulld be True if we need create osimage path - path to / of the image/ can be ralative, if needed (will be converted to absolute) kernver - kernel version (will be checked on creation) kernopt - kernel options grab_list - rsync exclude list for grabbing live node to image """ self._logger.debug("Arguments to function '{}".format(self._debug_function())) self._collection_name = 'osimage' mongo_doc = self._check_name(name, mongo_db, create, id) if bool(kernopts) and type(kernopts) is not str: self._logger.error("Kernel options should be 'str' type") raise RuntimeError self._keylist = {'path': type(''), 'kernver': type(''), 'kernopts': type(''), 'kernmodules': type(''), 'dracutmodules': type(''), 'tarball': type(''), 'torrent': type(''), 'kernfile': type(''), 'initrdfile': type(''), 'grab_exclude_list': type(''), 'grab_filesystems': type('')} if create: cluster = Cluster(mongo_db = self._mongo_db) path = os.path.abspath(path) path_suspected_doc = self._mongo_collection.find_one({'path': path}) if path_suspected_doc and path_suspected_doc['path'] == path: self._logger.error("Cannot create 'osimage' with the same 'path' as name='{}' has".format(path_suspected_doc['name'])) raise RuntimeError if kernver == 'ANY': try: kernver = self.get_package_ver(path, 'kernel')[0] except: pass if not self._check_kernel(path, kernver): raise RuntimeError grab_list_path = cluster.get('path') + '/templates/' + grab_list if not os.path.isfile(grab_list_path): self._logger.error("'{}' is not a file.".format(grab_list_path)) raise RuntimeError with open(grab_list_path) as lst: grab_list_content = lst.read() mongo_doc = {'name': name, 'path': path, 'kernver': kernver, 'kernopts': kernopts, 'dracutmodules': 'luna,-i18n,-plymouth', 'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler', 'grab_exclude_list': grab_list_content, 'grab_filesystems': '/,/boot'} self._logger.debug("mongo_doc: '{}'".format(mongo_doc)) self._name = name self._id = self._mongo_collection.insert(mongo_doc) self._DBRef = DBRef(self._collection_name, self._id) self.link(cluster) else: self._name = mongo_doc['name'] self._id = mongo_doc['_id'] self._DBRef = DBRef(self._collection_name, self._id) self._logger = logging.getLogger(__name__ + '.' + self._name)
def render_script(self, name): scripts = ['boot', 'install'] if name not in scripts: self.log.error( "'{}' is not correct script. Valid options are: '{}'" .format(name, scripts) ) return None cluster = Cluster(mongo_db=self._mongo_db) self._get_group() path = cluster.get('path') tloader = template.Loader(path + '/templates') if cluster.get('frontend_https'): protocol = 'https' else: protocol = 'http' server_ip = cluster.get('frontend_address') server_port = cluster.get('frontend_port') if name == 'boot': p = self.boot_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port return tloader.load('templ_nodeboot.cfg').generate(p=p) if name == 'install': p = self.install_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port res = tloader.load('templ_install.cfg').generate(p=p) return res
def render_script(self, name): scripts = ['boot', 'install'] if name not in scripts: self.log.error( "'{}' is not correct script. Valid options are: '{}'".format( name, scripts)) return None cluster = Cluster(mongo_db=self._mongo_db) self._get_group() path = cluster.get('path') tloader = template.Loader(path + '/templates') if cluster.get('frontend_https'): protocol = 'https' else: protocol = 'http' server_ip = cluster.get('frontend_address') server_port = cluster.get('frontend_port') if name == 'boot': p = self.boot_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port return tloader.load('templ_nodeboot.cfg').generate(p=p) if name == 'install': p = self.install_params p['protocol'] = protocol p['server_ip'] = server_ip p['server_port'] = server_port res = tloader.load('templ_install.cfg').generate(p=p) return res
def create_torrent(self): # TODO check if root tarball_uid = self.get('tarball') if not tarball_uid: self.log.error("No tarball in DB.") return False cluster = Cluster(mongo_db=self._mongo_db) tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz" if not os.path.exists(tarball): self.log.error("Wrong path in DB.") return False tracker_address = cluster.get('frontend_address') if tracker_address == '': self.log.error("Tracker address needs to be configured.") return False tracker_port = cluster.get('frontend_port') if tracker_port == 0: self.log.error("Tracker port needs to be configured.") return False user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid old_cwd = os.getcwd() os.chdir(os.path.dirname(tarball)) uid = str(uuid.uuid4()) torrentfile = cluster.get('path') + "/torrents/" + uid + ".torrent" fs = libtorrent.file_storage() libtorrent.add_files(fs, os.path.basename(tarball)) t = libtorrent.create_torrent(fs) if cluster.get('frontend_https'): proto = 'https' else: proto = 'http' t.add_tracker((proto + "://" + str(tracker_address) + ":" + str(tracker_port) + "/announce")) t.set_creator(torrent_key) t.set_comment(uid) libtorrent.set_piece_hashes(t, ".") f = open(torrentfile, 'w') f.write(libtorrent.bencode(t.generate())) f.close() os.chown(torrentfile, user_id, grp_id) self.set('torrent', str(uid)) os.chdir(old_cwd) return True
def _generate_name(self, mongo_db): cluster = Cluster(mongo_db) prefix = cluster.get('nodeprefix') digits = cluster.get('nodedigits') back_links = cluster.get_back_links() max_num = 0 for link in back_links: if not link['collection'] == self._collection_name: continue node = Node(id = link['DBRef'].id, mongo_db = mongo_db) name = node.name try: nnode = int(name.lstrip(prefix)) except ValueError: continue if nnode > max_num: max_num = nnode ret_name = prefix + str(max_num + 1).zfill(digits) return ret_name
def pack_boot(self): def mount(source, target, fs): subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target]) #ret = ctypes.CDLL('libc.so.6', use_errno=True).mount(source, target, fs, 0, options) #if ret < 0: # errno = ctypes.get_errno() # raise RuntimeError("Error mounting {} ({}) on {} with options '{}': {}". # format(source, fs, target, options, os.strerror(errno))) def umount(source): subprocess.Popen(['/usr/bin/umount', source]) #ret = ctypes.CDLL('libc.so.6', use_errno=True).umount(source) #if ret < 0: # errno = ctypes.get_errno() # raise RuntimeError("Error umounting {}: .". # format(source, os.strerror(errno))) def prepare_mounts(path): mount('devtmpfs', path + '/dev', 'devtmpfs') mount('proc', path + '/proc', 'proc') mount('sysfs', path + '/sys', 'sysfs') def cleanup_mounts(path): umount(path + '/dev') umount(path + '/proc') umount(path + '/sys') cluster = Cluster(mongo_db = self._mongo_db) #boot_prefix = '/boot' image_path = str(self.get('path')) kernver = str(self.get('kernver')) tmp_path = '/tmp' # in chroot env initrdfile = str(self.name) + '-initramfs-' + kernver kernfile = str(self.name) + '-vmlinuz-' + kernver #kernel_image = kernel_name + '-' + kernver #kernel_path = image_path + boot_prefix + '/' + kernel_image path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None path = str(path) user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None #group = cluster.get('group') #if not group: # self._logger.error("Group needs to be configured.") # return None path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) modules_add = [] modules_remove = [] drivers_add = [] drivers_remove = [] dracutmodules = self.get('dracutmodules') if dracutmodules: dracutmodules = str(dracutmodules) modules_add = sum([['--add', i] for i in dracutmodules.split(',') if i[0] != '-'], []) modules_remove = sum([['--omit', i[1:]] for i in dracutmodules.split(',') if i[0] == '-'], []) kernmodules = self.get('kernmodules') if kernmodules: kernmodules = str(kernmodules) drivers_add = sum([['--add-drivers', i] for i in kernmodules.split(',') if i[0] != '-'], []) drivers_remove = sum([['--omit-drivers', i[1:]] for i in kernmodules.split(',') if i[0] == '-'], []) prepare_mounts(image_path) real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) try: dracut_modules = subprocess.Popen(['/usr/sbin/dracut', '--kver', kernver, '--list-modules'], stdout=subprocess.PIPE) luna_exists = False while dracut_modules.poll() is None: line = dracut_modules.stdout.readline() if line.strip() == 'luna': luna_exists = True if not luna_exists: self._logger.error("No luna dracut module in osimage '{}'".format(self.name)) raise RuntimeError dracut_cmd = ['/usr/sbin/dracut', '--force', '--kver', kernver] + modules_add + modules_remove + drivers_add + drivers_remove + [tmp_path + '/' + initrdfile] dracut_create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE) while dracut_create.poll() is None: line = dracut_create.stdout.readline() except: os.fchdir(real_root) os.chroot(".") os.close(real_root) cleanup_mounts(image_path) try: pass #os.remove(image_path + '/' + tmp_path + '/' + initrdfile) except: pass return None os.fchdir(real_root) os.chroot(".") os.close(real_root) cleanup_mounts(image_path) shutil.copy(image_path + tmp_path + '/' + initrdfile, path_to_store) shutil.copy(image_path + '/boot/vmlinuz-' + kernver, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile)
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db = self._mongo_db) path = cluster.get('path') if not path: self._logger.error("Path needs to be configured.") return None tracker_address = cluster.get('fronend_address') if tracker_address == '': self._logger.error("Tracker address needs to be configured.") return None tracker_port = cluster.get('frontend_port') if tracker_port == 0: self._logger.error("Tracker port needs to be configured.") return None user = cluster.get('user') if not user: self._logger.error("User needs to be configured.") return None #group = cluster.get('group') #if not group: # self._logger.error("Group needs to be configured.") # return None path_to_store = path + "/torrents" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) uid = str(uuid.uuid4()) tarfile_path = path_to_store + "/" + uid + ".tgz" image_path = self.get('path') #tarball = tarfile.open(tarfile_path, "w:gz") #tarball.add(image_path, arcname=os.path.basename(image_path + "/.")) #tarball.close() try: tar_out = subprocess.Popen(['/usr/bin/tar', '-C', image_path + '/.', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '-c', '-z', '-f', tarfile_path, '.'], stderr=subprocess.PIPE) # dirty, but 4 times faster stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: os.remove(tarfile_path) sys.stdout.write('\r') return None os.chown(tarfile_path, user_id, grp_id) os.chmod(tarfile_path, 0644) self.set('tarball', str(uid)) return True
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid path_to_store = path + "/torrents" if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) image_path = self.get('path') real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) uid = str(uuid.uuid4()) tarfile = uid + ".tgz" try: # dirty, but 4 times faster tar_out = subprocess.Popen([ '/usr/bin/tar', '-C', '/', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '--exclude=./tmp/' + tarfile, '--use-compress-program=/usr/bin/pigz', '-c', '-f', '/tmp/' + tarfile, '.' ], stderr=subprocess.PIPE) stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == exceptions.KeyboardInterrupt: self.log.error('Keyboard interrupt.') else: self.log.error(exc_value) self.log.debug(traceback.format_exc()) if os.path.isfile('/tmp/' + tarfile): os.remove('/tmp/' + tarfile) sys.stdout.write('\r') os.fchdir(real_root) os.chroot(".") os.close(real_root) return False os.fchdir(real_root) os.chroot(".") os.close(real_root) # copy image, so permissions and selinux contexts # will be inherited from parent folder shutil.copy(image_path + '/tmp/' + tarfile, path_to_store) os.remove(image_path + '/tmp/' + tarfile) os.chown(path_to_store + '/' + tarfile, user_id, grp_id) os.chmod(path_to_store + '/' + tarfile, 0644) self.set('tarball', str(uid)) return True
def pack_boot(self): def mount(source, target, fs): subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target]) def umount(source): subprocess.Popen(['/usr/bin/umount', source]) def prepare_mounts(path): mount('devtmpfs', path + '/dev', 'devtmpfs') mount('proc', path + '/proc', 'proc') mount('sysfs', path + '/sys', 'sysfs') def cleanup_mounts(path): umount(path + '/dev') umount(path + '/proc') umount(path + '/sys') tmp_path = '/tmp' # in chroot env image_path = self.get('path') kernver = self.get('kernver') kernfile = self.name + '-vmlinuz-' + kernver initrdfile = self.name + '-initramfs-' + kernver cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) modules_add = [] modules_remove = [] drivers_add = [] drivers_remove = [] dracutmodules = self.get('dracutmodules') if dracutmodules: for i in dracutmodules.split(','): if i[0] != '-': modules_add.extend(['--add', i]) else: modules_remove.extend(['--omit', i[1:]]) kernmodules = self.get('kernmodules') if kernmodules: for i in kernmodules.split(','): if i[0] != '-': drivers_add.extend(['--add-drivers', i]) else: drivers_remove.extend(['--omit-drivers', i[1:]]) prepare_mounts(image_path) real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) chroot_path = os.open("/", os.O_DIRECTORY) os.fchdir(chroot_path) dracut_succeed = True create = None try: dracut_modules = subprocess.Popen( ['/usr/sbin/dracut', '--kver', kernver, '--list-modules'], stdout=subprocess.PIPE) luna_exists = False while dracut_modules.poll() is None: line = dracut_modules.stdout.readline() if line.strip() == 'luna': luna_exists = True break if not luna_exists: err_msg = ("No luna dracut module in osimage '{}'".format( self.name)) self.log.error(err_msg) raise RuntimeError, err_msg dracut_cmd = (['/usr/sbin/dracut', '--force', '--kver', kernver] + modules_add + modules_remove + drivers_add + drivers_remove + [tmp_path + '/' + initrdfile]) create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE) while create.poll() is None: line = create.stdout.readline() except: dracut_succeed = False if create and create.returncode: dracut_succeed = False if not create: dracut_succeed = False os.fchdir(real_root) os.chroot(".") os.close(real_root) os.close(chroot_path) cleanup_mounts(image_path) if not dracut_succeed: self.log.error("Error while building initrd.") return False initrd_path = image_path + tmp_path + '/' + initrdfile kernel_path = image_path + '/boot/vmlinuz-' + kernver if not os.path.isfile(kernel_path): self.log.error("Unable to find kernel in {}".format(kernel_path)) return False if not os.path.isfile(initrd_path): self.log.error("Unable to find initrd in {}".format(initrd_path)) return False # copy initrd file to inherit perms from parent folder shutil.copy(initrd_path, path_to_store + '/' + initrdfile) os.remove(initrd_path) shutil.copy(kernel_path, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) return True
def __init__(self, name=None, mongo_db=None, create=False, id=None, path='', kernver='', kernopts='', comment='', grab_list='grab_default_centos.lst'): """ path - path to / of the image (will be converted to absolute) kernver - kernel version (will be checked on creation) kernopt - kernel options grab_list - rsync exclude list for grabbing live node to image """ self.log.debug("function args {}".format(self._debug_function())) # Define the schema used to represent osimage objects self._collection_name = 'osimage' self._keylist = {'path': type(''), 'kernver': type(''), 'kernopts': type(''), 'kernmodules': type(''), 'dracutmodules': type(''), 'tarball': type(''), 'torrent': type(''), 'kernfile': type(''), 'initrdfile': type(''), 'grab_exclude_list': type(''), 'grab_filesystems': type(''), 'comment': type('')} # Check if this osimage is already present in the datastore # Read it if that is the case osimage = self._get_object(name, mongo_db, create, id) if bool(kernopts) and type(kernopts) is not str: err_msg = "Kernel options should be 'str' type" self.log.error(err_msg) raise RuntimeError, err_msg if create: cluster = Cluster(mongo_db=self._mongo_db) path = os.path.abspath(path) duplicate = self._mongo_collection.find_one({'path': path}) if duplicate: err_msg = ("Path belongs to osimage '{}'" .format(duplicate['name'])) self.log.error(err_msg) raise RuntimeError, err_msg if not os.path.isdir(path): err_msg = "'{}' is not a valid directory".format(path) self.log.error(err_msg) raise RuntimeError, err_msg kernels = self.get_package_ver(path, 'kernel') if not kernels: err_msg = "No kernels installed in '{}'".format(path) self.log.error(err_msg) raise RuntimeError, err_msg elif not kernver: kernver = kernels[0] elif kernver not in kernels: err_msg = "Available kernels are '{}'".format(kernels) self.log.error(err_msg) raise RuntimeError, err_msg grab_list_path = cluster.get('path') + '/templates/' + grab_list if not os.path.isfile(grab_list_path): err_msg = "'{}' is not a file.".format(grab_list_path) self.log.error(err_msg) raise RuntimeError, err_msg with open(grab_list_path) as lst: grab_list_content = lst.read() # Store the new osimage in the datastore osimage = {'name': name, 'path': path, 'kernver': kernver, 'kernopts': kernopts, 'kernfile': '', 'initrdfile': '', 'dracutmodules': 'luna,-i18n,-plymouth', 'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler', 'grab_exclude_list': grab_list_content, 'grab_filesystems': '/,/boot', 'comment': comment} self.log.debug("Saving osimage '{}' to the datastore" .format(osimage)) self.store(osimage) # Link this osimage to its dependencies and the current cluster self.link(cluster) self.log = logging.getLogger(__name__ + '.' + self._name)
def __init__(self, name=None, mongo_db=None, create=False, id=None, path='', kernver='', kernopts='', comment='', grab_list='grab_default_centos.lst'): """ path - path to / of the image (will be converted to absolute) kernver - kernel version (will be checked on creation) kernopt - kernel options grab_list - rsync exclude list for grabbing live node to image """ self.log.debug("function args {}".format(self._debug_function())) # Define the schema used to represent osimage objects self._collection_name = 'osimage' self._keylist = { 'path': type(''), 'kernver': type(''), 'kernopts': type(''), 'kernmodules': type(''), 'dracutmodules': type(''), 'tarball': type(''), 'torrent': type(''), 'kernfile': type(''), 'initrdfile': type(''), 'grab_exclude_list': type(''), 'grab_filesystems': type(''), 'comment': type('') } # Check if this osimage is already present in the datastore # Read it if that is the case osimage = self._get_object(name, mongo_db, create, id) if bool(kernopts) and type(kernopts) is not str: err_msg = "Kernel options should be 'str' type" self.log.error(err_msg) raise RuntimeError, err_msg if create: cluster = Cluster(mongo_db=self._mongo_db) path = os.path.abspath(path) duplicate = self._mongo_collection.find_one({'path': path}) if duplicate: err_msg = ("Path belongs to osimage '{}'".format( duplicate['name'])) self.log.error(err_msg) raise RuntimeError, err_msg if not os.path.isdir(path): err_msg = "'{}' is not a valid directory".format(path) self.log.error(err_msg) raise RuntimeError, err_msg kernels = self.get_package_ver(path, 'kernel') if not kernels: err_msg = "No kernels installed in '{}'".format(path) self.log.error(err_msg) raise RuntimeError, err_msg elif not kernver: kernver = kernels[0] elif kernver not in kernels: err_msg = "Available kernels are '{}'".format(kernels) self.log.error(err_msg) raise RuntimeError, err_msg grab_list_path = cluster.get('path') + '/templates/' + grab_list if not os.path.isfile(grab_list_path): err_msg = "'{}' is not a file.".format(grab_list_path) self.log.error(err_msg) raise RuntimeError, err_msg with open(grab_list_path) as lst: grab_list_content = lst.read() # Store the new osimage in the datastore osimage = { 'name': name, 'path': path, 'kernver': kernver, 'kernopts': kernopts, 'kernfile': '', 'initrdfile': '', 'dracutmodules': 'luna,-i18n,-plymouth', 'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler', 'grab_exclude_list': grab_list_content, 'grab_filesystems': '/,/boot', 'comment': comment } self.log.debug( "Saving osimage '{}' to the datastore".format(osimage)) self.store(osimage) # Link this osimage to its dependencies and the current cluster self.link(cluster) self.log = logging.getLogger(__name__ + '.' + self._name)
def pack_boot(self): def mount(source, target, fs): subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target]) def umount(source): subprocess.Popen(['/usr/bin/umount', source]) def prepare_mounts(path): mount('devtmpfs', path + '/dev', 'devtmpfs') mount('proc', path + '/proc', 'proc') mount('sysfs', path + '/sys', 'sysfs') def cleanup_mounts(path): umount(path + '/dev') umount(path + '/proc') umount(path + '/sys') tmp_path = '/tmp' # in chroot env image_path = self.get('path') kernver = self.get('kernver') kernfile = self.name + '-vmlinuz-' + kernver initrdfile = self.name + '-initramfs-' + kernver cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') path_to_store = path + "/boot" user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) modules_add = [] modules_remove = [] drivers_add = [] drivers_remove = [] dracutmodules = self.get('dracutmodules') if dracutmodules: for i in dracutmodules.split(','): if i[0] != '-': modules_add.extend(['--add', i]) else: modules_remove.extend(['--omit', i[1:]]) kernmodules = self.get('kernmodules') if kernmodules: for i in kernmodules.split(','): if i[0] != '-': drivers_add.extend(['--add-drivers', i]) else: drivers_remove.extend(['--omit-drivers', i[1:]]) prepare_mounts(image_path) real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) chroot_path = os.open("/", os.O_DIRECTORY) os.fchdir(chroot_path) dracut_succeed = True create = None try: dracut_modules = subprocess.Popen(['/usr/sbin/dracut', '--kver', kernver, '--list-modules'], stdout=subprocess.PIPE) luna_exists = False while dracut_modules.poll() is None: line = dracut_modules.stdout.readline() if line.strip() == 'luna': luna_exists = True break if not luna_exists: err_msg = ("No luna dracut module in osimage '{}'" .format(self.name)) self.log.error(err_msg) raise RuntimeError, err_msg dracut_cmd = (['/usr/sbin/dracut', '--force', '--kver', kernver] + modules_add + modules_remove + drivers_add + drivers_remove + [tmp_path + '/' + initrdfile]) create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE) while create.poll() is None: line = create.stdout.readline() except: dracut_succeed = False if create and create.returncode: dracut_succeed = False if not create: dracut_succeed = False os.fchdir(real_root) os.chroot(".") os.close(real_root) os.close(chroot_path) cleanup_mounts(image_path) if not dracut_succeed: self.log.error("Error while building initrd.") return False initrd_path = image_path + tmp_path + '/' + initrdfile kernel_path = image_path + '/boot/vmlinuz-' + kernver if not os.path.isfile(kernel_path): self.log.error("Unable to find kernel in {}".format(kernel_path)) return False if not os.path.isfile(initrd_path): self.log.error("Unable to find initrd in {}".format(initrd_path)) return False # copy initrd file to inherit perms from parent folder shutil.copy(initrd_path, path_to_store + '/' + initrdfile) os.remove(initrd_path) shutil.copy(kernel_path, path_to_store + '/' + kernfile) os.chown(path_to_store + '/' + initrdfile, user_id, grp_id) os.chmod(path_to_store + '/' + initrdfile, 0644) os.chown(path_to_store + '/' + kernfile, user_id, grp_id) os.chmod(path_to_store + '/' + kernfile, 0644) self.set('kernfile', kernfile) self.set('initrdfile', initrdfile) return True
def create_tarball(self): # TODO check if root cluster = Cluster(mongo_db=self._mongo_db) path = cluster.get('path') user = cluster.get('user') user_id = pwd.getpwnam(user).pw_uid grp_id = pwd.getpwnam(user).pw_gid path_to_store = path + "/torrents" if not os.path.exists(path_to_store): os.makedirs(path_to_store) os.chown(path_to_store, user_id, grp_id) os.chmod(path_to_store, 0644) image_path = self.get('path') real_root = os.open("/", os.O_RDONLY) os.chroot(image_path) uid = str(uuid.uuid4()) tarfile = uid + ".tgz" try: # dirty, but 4 times faster tar_out = subprocess.Popen( [ '/usr/bin/tar', '-C', '/', '--one-file-system', '--xattrs', '--selinux', '--acls', '--checkpoint=100', '--exclude=./tmp/' + tarfile, '--use-compress-program=/usr/bin/pigz', '-c', '-f', '/tmp/' + tarfile, '.' ], stderr=subprocess.PIPE ) stat_symb = ['\\', '|', '/', '-'] i = 0 while True: line = tar_out.stderr.readline() if line == '': break i = i + 1 sys.stdout.write(stat_symb[i % len(stat_symb)]) sys.stdout.write('\r') except: exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == exceptions.KeyboardInterrupt: self.log.error('Keyboard interrupt.') else: self.log.error(exc_value) self.log.debug(traceback.format_exc()) if os.path.isfile('/tmp/' + tarfile): os.remove('/tmp/' + tarfile) sys.stdout.write('\r') os.fchdir(real_root) os.chroot(".") os.close(real_root) return False os.fchdir(real_root) os.chroot(".") os.close(real_root) # copy image, so permissions and selinux contexts # will be inherited from parent folder shutil.copy(image_path + '/tmp/' + tarfile, path_to_store) os.remove(image_path + '/tmp/' + tarfile) os.chown(path_to_store + '/' + tarfile, user_id, grp_id) os.chmod(path_to_store + '/' + tarfile, 0644) self.set('tarball', str(uid)) return True