def copy_boot(self):
cluster = Cluster(mongo_db = self._mongo_db)
image_path = str(self.get('path'))
kernver = str(self.get('kernver'))
tmp_path = '/tmp' # in chroot env
initrdfile = str(self.name) + '-initramfs-' + kernver
kernfile = str(self.name) + '-vmlinuz-' + kernver
path = cluster.get('path')
if not path:
self._logger.error("Path needs to be configured.")
return None
path = str(path)
user = cluster.get('user')
if not user:
self._logger.error("User needs to be configured.")
return None
path_to_store = path + "/boot"
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
shutil.copy(image_path + '/boot/initramfs-' + kernver + '.img', path_to_store + '/' + initrdfile)
shutil.copy(image_path + '/boot/vmlinuz-' + kernver, path_to_store + '/' + kernfile)
os.chown(path_to_store + '/' + initrdfile, user_id, grp_id)
os.chmod(path_to_store + '/' + initrdfile, 0644)
os.chown(path_to_store + '/' + kernfile, user_id, grp_id)
os.chmod(path_to_store + '/' + kernfile, 0644)
self.set('kernfile', kernfile)
self.set('initrdfile', initrdfile)
self._logger.warning("Boot files was copied, but luna module might not being added to initrd. Please check /etc/dracut.conf.d in image")
return True
def create_tarball(self):
# TODO check if root
cluster = Cluster(mongo_db = self._mongo_db)
path = cluster.get('path')
if not path:
self._logger.error("Path needs to be configured.")
return None
tracker_address = cluster.get('fronend_address')
if tracker_address == '':
self._logger.error("Tracker address needs to be configured.")
return None
tracker_port = cluster.get('frontend_port')
if tracker_port == 0:
self._logger.error("Tracker port needs to be configured.")
return None
user = cluster.get('user')
if not user:
self._logger.error("User needs to be configured.")
return None
path_to_store = path + "/torrents"
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
os.chmod(path_to_store, 0644)
uid = str(uuid.uuid4())
tarfile_path = path_to_store + "/" + uid + ".tgz"
image_path = self.get('path')
try:
tar_out = subprocess.Popen(['/usr/bin/tar',
'-C', image_path + '/.',
'--one-file-system',
'--xattrs',
'--selinux',
'--acls',
'--checkpoint=100',
'-c', '-z', '-f', tarfile_path, '.'], stderr=subprocess.PIPE) # dirty, but 4 times faster
stat_symb = ['\\', '|', '/', '-']
i = 0
while True:
line = tar_out.stderr.readline()
if line == '':
break
i = i + 1
sys.stdout.write(stat_symb[i % len(stat_symb)])
sys.stdout.write('\r')
except:
os.remove(tarfile_path)
sys.stdout.write('\r')
return None
os.chown(tarfile_path, user_id, grp_id)
os.chmod(tarfile_path, 0644)
self.set('tarball', str(uid))
return True
def create_torrent(self):
# TODO check if root
tarball_uid = self.get('tarball')
cluster = Cluster(mongo_db = self._mongo_db)
if not bool(tarball_uid):
self._logger.error("No tarball in DB.")
return None
tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz"
if not os.path.exists(tarball):
self._logger.error("Wrong path in DB.")
return None
tracker_address = cluster.get('frontend_address')
if tracker_address == '':
self._logger.error("Tracker address needs to be configured.")
return None
tracker_port = cluster.get('frontend_port')
if tracker_port == 0:
self._logger.error("Tracker port needs to be configured.")
return None
user = cluster.get('user')
if not user:
self._logger.error("User needs to be configured.")
return None
#group = cluster.get('group')
#if not group:
# self._logger.error("Group needs to be configured.")
# return None
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
old_cwd = os.getcwd()
os.chdir(os.path.dirname(tarball))
uid = str(uuid.uuid4())
torrentfile = str(cluster.get('path')) + "/torrents/" + uid
fs = libtorrent.file_storage()
libtorrent.add_files(fs, os.path.basename(tarball))
t = libtorrent.create_torrent(fs)
t.add_tracker("http://" + str(tracker_address) + ":" + str(tracker_port) + "/announce")
t.set_creator(torrent_key)
t.set_comment(uid)
libtorrent.set_piece_hashes(t, ".")
f = open(torrentfile, 'w')
f.write(libtorrent.bencode(t.generate()))
f.close()
self.set('torrent', str(uid))
os.chown(torrentfile, user_id, grp_id)
shutil.move(torrentfile, torrentfile + ".torrent")
os.chdir(old_cwd)
return True
def copy_boot(self):
image_path = self.get('path')
kernver = self.get('kernver')
initrdfile = self.name + '-initramfs-' + kernver
kernfile = self.name + '-vmlinuz-' + kernver
cluster = Cluster(mongo_db=self._mongo_db)
user = cluster.get('user')
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
path = cluster.get('path')
path_to_store = path + "/boot"
initrd_path = image_path + '/boot/initramfs-' + kernver + '.img'
kernel_path = image_path + '/boot/vmlinuz-' + kernver
if not os.path.isfile(kernel_path):
self.log.error("Unable to find kernel in {}".format(kernel_path))
return False
if not os.path.isfile(initrd_path):
self.log.error("Unable to find initrd in {}".format(initrd_path))
return False
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
shutil.copy(initrd_path, path_to_store + '/' + initrdfile)
shutil.copy(kernel_path, path_to_store + '/' + kernfile)
os.chown(path_to_store + '/' + initrdfile, user_id, grp_id)
os.chmod(path_to_store + '/' + initrdfile, 0644)
os.chown(path_to_store + '/' + kernfile, user_id, grp_id)
os.chmod(path_to_store + '/' + kernfile, 0644)
self.set('kernfile', kernfile)
self.set('initrdfile', initrdfile)
self.log.warning(("Boot files were copied, but luna modules might not "
"have been added to initrd. Please check "
"/etc/dracut.conf.d in the image"))
return True
def copy_boot(self):
image_path = self.get('path')
kernver = self.get('kernver')
initrdfile = self.name + '-initramfs-' + kernver
kernfile = self.name + '-vmlinuz-' + kernver
cluster = Cluster(mongo_db=self._mongo_db)
user = cluster.get('user')
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
path = cluster.get('path')
path_to_store = path + "/boot"
initrd_path = image_path + '/boot/initramfs-' + kernver + '.img'
kernel_path = image_path + '/boot/vmlinuz-' + kernver
if not os.path.isfile(kernel_path):
self.log.error("Unable to find kernel in {}".format(kernel_path))
return False
if not os.path.isfile(initrd_path):
self.log.error("Unable to find initrd in {}".format(initrd_path))
return False
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
shutil.copy(initrd_path, path_to_store + '/' + initrdfile)
shutil.copy(kernel_path, path_to_store + '/' + kernfile)
os.chown(path_to_store + '/' + initrdfile, user_id, grp_id)
os.chmod(path_to_store + '/' + initrdfile, 0644)
os.chown(path_to_store + '/' + kernfile, user_id, grp_id)
os.chmod(path_to_store + '/' + kernfile, 0644)
self.set('kernfile', kernfile)
self.set('initrdfile', initrdfile)
self.log.warning(("Boot files were copied, but luna modules might not "
"have been added to initrd. Please check "
"/etc/dracut.conf.d in the image"))
return True
def __init__(self, name = None, mongo_db = None, create = False, id = None, path = '', kernver = '', kernopts = '', grab_list = 'grab_default_centos.lst'):
"""
create - shoulld be True if we need create osimage
path - path to / of the image/ can be ralative, if needed (will be converted to absolute)
kernver - kernel version (will be checked on creation)
kernopt - kernel options
grab_list - rsync exclude list for grabbing live node to image
"""
self._logger.debug("Arguments to function '{}".format(self._debug_function()))
self._collection_name = 'osimage'
mongo_doc = self._check_name(name, mongo_db, create, id)
if bool(kernopts) and type(kernopts) is not str:
self._logger.error("Kernel options should be 'str' type")
raise RuntimeError
self._keylist = {'path': type(''), 'kernver': type(''), 'kernopts': type(''),
'kernmodules': type(''), 'dracutmodules': type(''), 'tarball': type(''),
'torrent': type(''), 'kernfile': type(''), 'initrdfile': type(''),
'grab_exclude_list': type(''), 'grab_filesystems': type('')}
if create:
cluster = Cluster(mongo_db = self._mongo_db)
path = os.path.abspath(path)
path_suspected_doc = self._mongo_collection.find_one({'path': path})
if path_suspected_doc and path_suspected_doc['path'] == path:
self._logger.error("Cannot create 'osimage' with the same 'path' as name='{}' has".format(path_suspected_doc['name']))
raise RuntimeError
if kernver == 'ANY':
try:
kernver = self.get_package_ver(path, 'kernel')[0]
except:
pass
if not self._check_kernel(path, kernver):
raise RuntimeError
grab_list_path = cluster.get('path') + '/templates/' + grab_list
if not os.path.isfile(grab_list_path):
self._logger.error("'{}' is not a file.".format(grab_list_path))
raise RuntimeError
with open(grab_list_path) as lst:
grab_list_content = lst.read()
mongo_doc = {'name': name, 'path': path,
'kernver': kernver, 'kernopts': kernopts,
'dracutmodules': 'luna,-i18n,-plymouth',
'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler',
'grab_exclude_list': grab_list_content,
'grab_filesystems': '/,/boot'}
self._logger.debug("mongo_doc: '{}'".format(mongo_doc))
self._name = name
self._id = self._mongo_collection.insert(mongo_doc)
self._DBRef = DBRef(self._collection_name, self._id)
self.link(cluster)
else:
self._name = mongo_doc['name']
self._id = mongo_doc['_id']
self._DBRef = DBRef(self._collection_name, self._id)
self._logger = logging.getLogger(__name__ + '.' + self._name)
def render_script(self, name):
scripts = ['boot', 'install']
if name not in scripts:
self.log.error(
"'{}' is not correct script. Valid options are: '{}'"
.format(name, scripts)
)
return None
cluster = Cluster(mongo_db=self._mongo_db)
self._get_group()
path = cluster.get('path')
tloader = template.Loader(path + '/templates')
if cluster.get('frontend_https'):
protocol = 'https'
else:
protocol = 'http'
server_ip = cluster.get('frontend_address')
server_port = cluster.get('frontend_port')
if name == 'boot':
p = self.boot_params
p['protocol'] = protocol
p['server_ip'] = server_ip
p['server_port'] = server_port
return tloader.load('templ_nodeboot.cfg').generate(p=p)
if name == 'install':
p = self.install_params
p['protocol'] = protocol
p['server_ip'] = server_ip
p['server_port'] = server_port
res = tloader.load('templ_install.cfg').generate(p=p)
return res
def render_script(self, name):
scripts = ['boot', 'install']
if name not in scripts:
self.log.error(
"'{}' is not correct script. Valid options are: '{}'".format(
name, scripts))
return None
cluster = Cluster(mongo_db=self._mongo_db)
self._get_group()
path = cluster.get('path')
tloader = template.Loader(path + '/templates')
if cluster.get('frontend_https'):
protocol = 'https'
else:
protocol = 'http'
server_ip = cluster.get('frontend_address')
server_port = cluster.get('frontend_port')
if name == 'boot':
p = self.boot_params
p['protocol'] = protocol
p['server_ip'] = server_ip
p['server_port'] = server_port
return tloader.load('templ_nodeboot.cfg').generate(p=p)
if name == 'install':
p = self.install_params
p['protocol'] = protocol
p['server_ip'] = server_ip
p['server_port'] = server_port
res = tloader.load('templ_install.cfg').generate(p=p)
return res
def create_torrent(self):
# TODO check if root
tarball_uid = self.get('tarball')
if not tarball_uid:
self.log.error("No tarball in DB.")
return False
cluster = Cluster(mongo_db=self._mongo_db)
tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz"
if not os.path.exists(tarball):
self.log.error("Wrong path in DB.")
return False
tracker_address = cluster.get('frontend_address')
if tracker_address == '':
self.log.error("Tracker address needs to be configured.")
return False
tracker_port = cluster.get('frontend_port')
if tracker_port == 0:
self.log.error("Tracker port needs to be configured.")
return False
user = cluster.get('user')
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
old_cwd = os.getcwd()
os.chdir(os.path.dirname(tarball))
uid = str(uuid.uuid4())
torrentfile = cluster.get('path') + "/torrents/" + uid + ".torrent"
fs = libtorrent.file_storage()
libtorrent.add_files(fs, os.path.basename(tarball))
t = libtorrent.create_torrent(fs)
if cluster.get('frontend_https'):
proto = 'https'
else:
proto = 'http'
t.add_tracker((proto + "://" + str(tracker_address) + ":" +
str(tracker_port) + "/announce"))
t.set_creator(torrent_key)
t.set_comment(uid)
libtorrent.set_piece_hashes(t, ".")
f = open(torrentfile, 'w')
f.write(libtorrent.bencode(t.generate()))
f.close()
os.chown(torrentfile, user_id, grp_id)
self.set('torrent', str(uid))
os.chdir(old_cwd)
return True
def create_torrent(self):
# TODO check if root
tarball_uid = self.get('tarball')
if not tarball_uid:
self.log.error("No tarball in DB.")
return False
cluster = Cluster(mongo_db=self._mongo_db)
tarball = cluster.get('path') + "/torrents/" + tarball_uid + ".tgz"
if not os.path.exists(tarball):
self.log.error("Wrong path in DB.")
return False
tracker_address = cluster.get('frontend_address')
if tracker_address == '':
self.log.error("Tracker address needs to be configured.")
return False
tracker_port = cluster.get('frontend_port')
if tracker_port == 0:
self.log.error("Tracker port needs to be configured.")
return False
user = cluster.get('user')
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
old_cwd = os.getcwd()
os.chdir(os.path.dirname(tarball))
uid = str(uuid.uuid4())
torrentfile = cluster.get('path') + "/torrents/" + uid + ".torrent"
fs = libtorrent.file_storage()
libtorrent.add_files(fs, os.path.basename(tarball))
t = libtorrent.create_torrent(fs)
if cluster.get('frontend_https'):
proto = 'https'
else:
proto = 'http'
t.add_tracker((proto + "://" + str(tracker_address) +
":" + str(tracker_port) + "/announce"))
t.set_creator(torrent_key)
t.set_comment(uid)
libtorrent.set_piece_hashes(t, ".")
f = open(torrentfile, 'w')
f.write(libtorrent.bencode(t.generate()))
f.close()
os.chown(torrentfile, user_id, grp_id)
self.set('torrent', str(uid))
os.chdir(old_cwd)
return True
def _generate_name(self, mongo_db):
cluster = Cluster(mongo_db)
prefix = cluster.get('nodeprefix')
digits = cluster.get('nodedigits')
back_links = cluster.get_back_links()
max_num = 0
for link in back_links:
if not link['collection'] == self._collection_name:
continue
node = Node(id = link['DBRef'].id, mongo_db = mongo_db)
name = node.name
try:
nnode = int(name.lstrip(prefix))
except ValueError:
continue
if nnode > max_num:
max_num = nnode
ret_name = prefix + str(max_num + 1).zfill(digits)
return ret_name
def _generate_name(self, mongo_db):
cluster = Cluster(mongo_db)
prefix = cluster.get('nodeprefix')
digits = cluster.get('nodedigits')
back_links = cluster.get_back_links()
max_num = 0
for link in back_links:
if not link['collection'] == self._collection_name:
continue
node = Node(id = link['DBRef'].id, mongo_db = mongo_db)
name = node.name
try:
nnode = int(name.lstrip(prefix))
except ValueError:
continue
if nnode > max_num:
max_num = nnode
ret_name = prefix + str(max_num + 1).zfill(digits)
return ret_name
def pack_boot(self):
def mount(source, target, fs):
subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target])
#ret = ctypes.CDLL('libc.so.6', use_errno=True).mount(source, target, fs, 0, options)
#if ret < 0:
# errno = ctypes.get_errno()
# raise RuntimeError("Error mounting {} ({}) on {} with options '{}': {}".
# format(source, fs, target, options, os.strerror(errno)))
def umount(source):
subprocess.Popen(['/usr/bin/umount', source])
#ret = ctypes.CDLL('libc.so.6', use_errno=True).umount(source)
#if ret < 0:
# errno = ctypes.get_errno()
# raise RuntimeError("Error umounting {}: .".
# format(source, os.strerror(errno)))
def prepare_mounts(path):
mount('devtmpfs', path + '/dev', 'devtmpfs')
mount('proc', path + '/proc', 'proc')
mount('sysfs', path + '/sys', 'sysfs')
def cleanup_mounts(path):
umount(path + '/dev')
umount(path + '/proc')
umount(path + '/sys')
cluster = Cluster(mongo_db = self._mongo_db)
#boot_prefix = '/boot'
image_path = str(self.get('path'))
kernver = str(self.get('kernver'))
tmp_path = '/tmp' # in chroot env
initrdfile = str(self.name) + '-initramfs-' + kernver
kernfile = str(self.name) + '-vmlinuz-' + kernver
#kernel_image = kernel_name + '-' + kernver
#kernel_path = image_path + boot_prefix + '/' + kernel_image
path = cluster.get('path')
if not path:
self._logger.error("Path needs to be configured.")
return None
path = str(path)
user = cluster.get('user')
if not user:
self._logger.error("User needs to be configured.")
return None
#group = cluster.get('group')
#if not group:
# self._logger.error("Group needs to be configured.")
# return None
path_to_store = path + "/boot"
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
modules_add = []
modules_remove = []
drivers_add = []
drivers_remove = []
dracutmodules = self.get('dracutmodules')
if dracutmodules:
dracutmodules = str(dracutmodules)
modules_add = sum([['--add', i] for i in dracutmodules.split(',') if i[0] != '-'], [])
modules_remove = sum([['--omit', i[1:]] for i in dracutmodules.split(',') if i[0] == '-'], [])
kernmodules = self.get('kernmodules')
if kernmodules:
kernmodules = str(kernmodules)
drivers_add = sum([['--add-drivers', i] for i in kernmodules.split(',') if i[0] != '-'], [])
drivers_remove = sum([['--omit-drivers', i[1:]] for i in kernmodules.split(',') if i[0] == '-'], [])
prepare_mounts(image_path)
real_root = os.open("/", os.O_RDONLY)
os.chroot(image_path)
try:
dracut_modules = subprocess.Popen(['/usr/sbin/dracut', '--kver', kernver, '--list-modules'], stdout=subprocess.PIPE)
luna_exists = False
while dracut_modules.poll() is None:
line = dracut_modules.stdout.readline()
if line.strip() == 'luna':
luna_exists = True
if not luna_exists:
self._logger.error("No luna dracut module in osimage '{}'".format(self.name))
raise RuntimeError
dracut_cmd = ['/usr/sbin/dracut', '--force', '--kver', kernver] + modules_add + modules_remove + drivers_add + drivers_remove + [tmp_path + '/' + initrdfile]
dracut_create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE)
while dracut_create.poll() is None:
line = dracut_create.stdout.readline()
except:
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
cleanup_mounts(image_path)
try:
pass
#os.remove(image_path + '/' + tmp_path + '/' + initrdfile)
except:
pass
return None
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
cleanup_mounts(image_path)
shutil.copy(image_path + tmp_path + '/' + initrdfile, path_to_store)
shutil.copy(image_path + '/boot/vmlinuz-' + kernver, path_to_store + '/' + kernfile)
os.chown(path_to_store + '/' + initrdfile, user_id, grp_id)
os.chmod(path_to_store + '/' + initrdfile, 0644)
os.chown(path_to_store + '/' + kernfile, user_id, grp_id)
os.chmod(path_to_store + '/' + kernfile, 0644)
self.set('kernfile', kernfile)
self.set('initrdfile', initrdfile)
def create_tarball(self):
# TODO check if root
cluster = Cluster(mongo_db = self._mongo_db)
path = cluster.get('path')
if not path:
self._logger.error("Path needs to be configured.")
return None
tracker_address = cluster.get('fronend_address')
if tracker_address == '':
self._logger.error("Tracker address needs to be configured.")
return None
tracker_port = cluster.get('frontend_port')
if tracker_port == 0:
self._logger.error("Tracker port needs to be configured.")
return None
user = cluster.get('user')
if not user:
self._logger.error("User needs to be configured.")
return None
#group = cluster.get('group')
#if not group:
# self._logger.error("Group needs to be configured.")
# return None
path_to_store = path + "/torrents"
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
os.chmod(path_to_store, 0644)
uid = str(uuid.uuid4())
tarfile_path = path_to_store + "/" + uid + ".tgz"
image_path = self.get('path')
#tarball = tarfile.open(tarfile_path, "w:gz")
#tarball.add(image_path, arcname=os.path.basename(image_path + "/."))
#tarball.close()
try:
tar_out = subprocess.Popen(['/usr/bin/tar',
'-C', image_path + '/.',
'--one-file-system',
'--xattrs',
'--selinux',
'--acls',
'--checkpoint=100',
'-c', '-z', '-f', tarfile_path, '.'], stderr=subprocess.PIPE) # dirty, but 4 times faster
stat_symb = ['\\', '|', '/', '-']
i = 0
while True:
line = tar_out.stderr.readline()
if line == '':
break
i = i + 1
sys.stdout.write(stat_symb[i % len(stat_symb)])
sys.stdout.write('\r')
except:
os.remove(tarfile_path)
sys.stdout.write('\r')
return None
os.chown(tarfile_path, user_id, grp_id)
os.chmod(tarfile_path, 0644)
self.set('tarball', str(uid))
return True
def create_tarball(self):
# TODO check if root
cluster = Cluster(mongo_db=self._mongo_db)
path = cluster.get('path')
user = cluster.get('user')
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
path_to_store = path + "/torrents"
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
os.chmod(path_to_store, 0644)
image_path = self.get('path')
real_root = os.open("/", os.O_RDONLY)
os.chroot(image_path)
uid = str(uuid.uuid4())
tarfile = uid + ".tgz"
try:
# dirty, but 4 times faster
tar_out = subprocess.Popen([
'/usr/bin/tar', '-C', '/', '--one-file-system', '--xattrs',
'--selinux', '--acls', '--checkpoint=100', '--exclude=./tmp/' +
tarfile, '--use-compress-program=/usr/bin/pigz', '-c', '-f',
'/tmp/' + tarfile, '.'
],
stderr=subprocess.PIPE)
stat_symb = ['\\', '|', '/', '-']
i = 0
while True:
line = tar_out.stderr.readline()
if line == '':
break
i = i + 1
sys.stdout.write(stat_symb[i % len(stat_symb)])
sys.stdout.write('\r')
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == exceptions.KeyboardInterrupt:
self.log.error('Keyboard interrupt.')
else:
self.log.error(exc_value)
self.log.debug(traceback.format_exc())
if os.path.isfile('/tmp/' + tarfile):
os.remove('/tmp/' + tarfile)
sys.stdout.write('\r')
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
return False
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
# copy image, so permissions and selinux contexts
# will be inherited from parent folder
shutil.copy(image_path + '/tmp/' + tarfile, path_to_store)
os.remove(image_path + '/tmp/' + tarfile)
os.chown(path_to_store + '/' + tarfile, user_id, grp_id)
os.chmod(path_to_store + '/' + tarfile, 0644)
self.set('tarball', str(uid))
return True
def pack_boot(self):
def mount(source, target, fs):
subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target])
def umount(source):
subprocess.Popen(['/usr/bin/umount', source])
def prepare_mounts(path):
mount('devtmpfs', path + '/dev', 'devtmpfs')
mount('proc', path + '/proc', 'proc')
mount('sysfs', path + '/sys', 'sysfs')
def cleanup_mounts(path):
umount(path + '/dev')
umount(path + '/proc')
umount(path + '/sys')
tmp_path = '/tmp' # in chroot env
image_path = self.get('path')
kernver = self.get('kernver')
kernfile = self.name + '-vmlinuz-' + kernver
initrdfile = self.name + '-initramfs-' + kernver
cluster = Cluster(mongo_db=self._mongo_db)
path = cluster.get('path')
user = cluster.get('user')
path_to_store = path + "/boot"
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
modules_add = []
modules_remove = []
drivers_add = []
drivers_remove = []
dracutmodules = self.get('dracutmodules')
if dracutmodules:
for i in dracutmodules.split(','):
if i[0] != '-':
modules_add.extend(['--add', i])
else:
modules_remove.extend(['--omit', i[1:]])
kernmodules = self.get('kernmodules')
if kernmodules:
for i in kernmodules.split(','):
if i[0] != '-':
drivers_add.extend(['--add-drivers', i])
else:
drivers_remove.extend(['--omit-drivers', i[1:]])
prepare_mounts(image_path)
real_root = os.open("/", os.O_RDONLY)
os.chroot(image_path)
chroot_path = os.open("/", os.O_DIRECTORY)
os.fchdir(chroot_path)
dracut_succeed = True
create = None
try:
dracut_modules = subprocess.Popen(
['/usr/sbin/dracut', '--kver', kernver, '--list-modules'],
stdout=subprocess.PIPE)
luna_exists = False
while dracut_modules.poll() is None:
line = dracut_modules.stdout.readline()
if line.strip() == 'luna':
luna_exists = True
break
if not luna_exists:
err_msg = ("No luna dracut module in osimage '{}'".format(
self.name))
self.log.error(err_msg)
raise RuntimeError, err_msg
dracut_cmd = (['/usr/sbin/dracut', '--force', '--kver', kernver] +
modules_add + modules_remove + drivers_add +
drivers_remove + [tmp_path + '/' + initrdfile])
create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE)
while create.poll() is None:
line = create.stdout.readline()
except:
dracut_succeed = False
if create and create.returncode:
dracut_succeed = False
if not create:
dracut_succeed = False
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
os.close(chroot_path)
cleanup_mounts(image_path)
if not dracut_succeed:
self.log.error("Error while building initrd.")
return False
initrd_path = image_path + tmp_path + '/' + initrdfile
kernel_path = image_path + '/boot/vmlinuz-' + kernver
if not os.path.isfile(kernel_path):
self.log.error("Unable to find kernel in {}".format(kernel_path))
return False
if not os.path.isfile(initrd_path):
self.log.error("Unable to find initrd in {}".format(initrd_path))
return False
# copy initrd file to inherit perms from parent folder
shutil.copy(initrd_path, path_to_store + '/' + initrdfile)
os.remove(initrd_path)
shutil.copy(kernel_path, path_to_store + '/' + kernfile)
os.chown(path_to_store + '/' + initrdfile, user_id, grp_id)
os.chmod(path_to_store + '/' + initrdfile, 0644)
os.chown(path_to_store + '/' + kernfile, user_id, grp_id)
os.chmod(path_to_store + '/' + kernfile, 0644)
self.set('kernfile', kernfile)
self.set('initrdfile', initrdfile)
return True
def __init__(self, name=None, mongo_db=None, create=False, id=None,
path='', kernver='', kernopts='', comment='',
grab_list='grab_default_centos.lst'):
"""
path - path to / of the image (will be converted to absolute)
kernver - kernel version (will be checked on creation)
kernopt - kernel options
grab_list - rsync exclude list for grabbing live node to image
"""
self.log.debug("function args {}".format(self._debug_function()))
# Define the schema used to represent osimage objects
self._collection_name = 'osimage'
self._keylist = {'path': type(''), 'kernver': type(''),
'kernopts': type(''), 'kernmodules': type(''),
'dracutmodules': type(''), 'tarball': type(''),
'torrent': type(''), 'kernfile': type(''),
'initrdfile': type(''), 'grab_exclude_list': type(''),
'grab_filesystems': type(''), 'comment': type('')}
# Check if this osimage is already present in the datastore
# Read it if that is the case
osimage = self._get_object(name, mongo_db, create, id)
if bool(kernopts) and type(kernopts) is not str:
err_msg = "Kernel options should be 'str' type"
self.log.error(err_msg)
raise RuntimeError, err_msg
if create:
cluster = Cluster(mongo_db=self._mongo_db)
path = os.path.abspath(path)
duplicate = self._mongo_collection.find_one({'path': path})
if duplicate:
err_msg = ("Path belongs to osimage '{}'"
.format(duplicate['name']))
self.log.error(err_msg)
raise RuntimeError, err_msg
if not os.path.isdir(path):
err_msg = "'{}' is not a valid directory".format(path)
self.log.error(err_msg)
raise RuntimeError, err_msg
kernels = self.get_package_ver(path, 'kernel')
if not kernels:
err_msg = "No kernels installed in '{}'".format(path)
self.log.error(err_msg)
raise RuntimeError, err_msg
elif not kernver:
kernver = kernels[0]
elif kernver not in kernels:
err_msg = "Available kernels are '{}'".format(kernels)
self.log.error(err_msg)
raise RuntimeError, err_msg
grab_list_path = cluster.get('path') + '/templates/' + grab_list
if not os.path.isfile(grab_list_path):
err_msg = "'{}' is not a file.".format(grab_list_path)
self.log.error(err_msg)
raise RuntimeError, err_msg
with open(grab_list_path) as lst:
grab_list_content = lst.read()
# Store the new osimage in the datastore
osimage = {'name': name, 'path': path,
'kernver': kernver, 'kernopts': kernopts,
'kernfile': '', 'initrdfile': '',
'dracutmodules': 'luna,-i18n,-plymouth',
'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler',
'grab_exclude_list': grab_list_content,
'grab_filesystems': '/,/boot', 'comment': comment}
self.log.debug("Saving osimage '{}' to the datastore"
.format(osimage))
self.store(osimage)
# Link this osimage to its dependencies and the current cluster
self.link(cluster)
self.log = logging.getLogger(__name__ + '.' + self._name)
def __init__(self,
name=None,
mongo_db=None,
create=False,
id=None,
path='',
kernver='',
kernopts='',
comment='',
grab_list='grab_default_centos.lst'):
"""
path - path to / of the image (will be converted to absolute)
kernver - kernel version (will be checked on creation)
kernopt - kernel options
grab_list - rsync exclude list for grabbing live node to image
"""
self.log.debug("function args {}".format(self._debug_function()))
# Define the schema used to represent osimage objects
self._collection_name = 'osimage'
self._keylist = {
'path': type(''),
'kernver': type(''),
'kernopts': type(''),
'kernmodules': type(''),
'dracutmodules': type(''),
'tarball': type(''),
'torrent': type(''),
'kernfile': type(''),
'initrdfile': type(''),
'grab_exclude_list': type(''),
'grab_filesystems': type(''),
'comment': type('')
}
# Check if this osimage is already present in the datastore
# Read it if that is the case
osimage = self._get_object(name, mongo_db, create, id)
if bool(kernopts) and type(kernopts) is not str:
err_msg = "Kernel options should be 'str' type"
self.log.error(err_msg)
raise RuntimeError, err_msg
if create:
cluster = Cluster(mongo_db=self._mongo_db)
path = os.path.abspath(path)
duplicate = self._mongo_collection.find_one({'path': path})
if duplicate:
err_msg = ("Path belongs to osimage '{}'".format(
duplicate['name']))
self.log.error(err_msg)
raise RuntimeError, err_msg
if not os.path.isdir(path):
err_msg = "'{}' is not a valid directory".format(path)
self.log.error(err_msg)
raise RuntimeError, err_msg
kernels = self.get_package_ver(path, 'kernel')
if not kernels:
err_msg = "No kernels installed in '{}'".format(path)
self.log.error(err_msg)
raise RuntimeError, err_msg
elif not kernver:
kernver = kernels[0]
elif kernver not in kernels:
err_msg = "Available kernels are '{}'".format(kernels)
self.log.error(err_msg)
raise RuntimeError, err_msg
grab_list_path = cluster.get('path') + '/templates/' + grab_list
if not os.path.isfile(grab_list_path):
err_msg = "'{}' is not a file.".format(grab_list_path)
self.log.error(err_msg)
raise RuntimeError, err_msg
with open(grab_list_path) as lst:
grab_list_content = lst.read()
# Store the new osimage in the datastore
osimage = {
'name': name,
'path': path,
'kernver': kernver,
'kernopts': kernopts,
'kernfile': '',
'initrdfile': '',
'dracutmodules': 'luna,-i18n,-plymouth',
'kernmodules': 'ipmi_devintf,ipmi_si,ipmi_msghandler',
'grab_exclude_list': grab_list_content,
'grab_filesystems': '/,/boot',
'comment': comment
}
self.log.debug(
"Saving osimage '{}' to the datastore".format(osimage))
self.store(osimage)
# Link this osimage to its dependencies and the current cluster
self.link(cluster)
self.log = logging.getLogger(__name__ + '.' + self._name)
def pack_boot(self):
def mount(source, target, fs):
subprocess.Popen(['/usr/bin/mount', '-t', fs, source, target])
def umount(source):
subprocess.Popen(['/usr/bin/umount', source])
def prepare_mounts(path):
mount('devtmpfs', path + '/dev', 'devtmpfs')
mount('proc', path + '/proc', 'proc')
mount('sysfs', path + '/sys', 'sysfs')
def cleanup_mounts(path):
umount(path + '/dev')
umount(path + '/proc')
umount(path + '/sys')
tmp_path = '/tmp' # in chroot env
image_path = self.get('path')
kernver = self.get('kernver')
kernfile = self.name + '-vmlinuz-' + kernver
initrdfile = self.name + '-initramfs-' + kernver
cluster = Cluster(mongo_db=self._mongo_db)
path = cluster.get('path')
user = cluster.get('user')
path_to_store = path + "/boot"
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
modules_add = []
modules_remove = []
drivers_add = []
drivers_remove = []
dracutmodules = self.get('dracutmodules')
if dracutmodules:
for i in dracutmodules.split(','):
if i[0] != '-':
modules_add.extend(['--add', i])
else:
modules_remove.extend(['--omit', i[1:]])
kernmodules = self.get('kernmodules')
if kernmodules:
for i in kernmodules.split(','):
if i[0] != '-':
drivers_add.extend(['--add-drivers', i])
else:
drivers_remove.extend(['--omit-drivers', i[1:]])
prepare_mounts(image_path)
real_root = os.open("/", os.O_RDONLY)
os.chroot(image_path)
chroot_path = os.open("/", os.O_DIRECTORY)
os.fchdir(chroot_path)
dracut_succeed = True
create = None
try:
dracut_modules = subprocess.Popen(['/usr/sbin/dracut', '--kver',
kernver, '--list-modules'],
stdout=subprocess.PIPE)
luna_exists = False
while dracut_modules.poll() is None:
line = dracut_modules.stdout.readline()
if line.strip() == 'luna':
luna_exists = True
break
if not luna_exists:
err_msg = ("No luna dracut module in osimage '{}'"
.format(self.name))
self.log.error(err_msg)
raise RuntimeError, err_msg
dracut_cmd = (['/usr/sbin/dracut', '--force', '--kver', kernver] +
modules_add + modules_remove + drivers_add +
drivers_remove + [tmp_path + '/' + initrdfile])
create = subprocess.Popen(dracut_cmd, stdout=subprocess.PIPE)
while create.poll() is None:
line = create.stdout.readline()
except:
dracut_succeed = False
if create and create.returncode:
dracut_succeed = False
if not create:
dracut_succeed = False
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
os.close(chroot_path)
cleanup_mounts(image_path)
if not dracut_succeed:
self.log.error("Error while building initrd.")
return False
initrd_path = image_path + tmp_path + '/' + initrdfile
kernel_path = image_path + '/boot/vmlinuz-' + kernver
if not os.path.isfile(kernel_path):
self.log.error("Unable to find kernel in {}".format(kernel_path))
return False
if not os.path.isfile(initrd_path):
self.log.error("Unable to find initrd in {}".format(initrd_path))
return False
# copy initrd file to inherit perms from parent folder
shutil.copy(initrd_path, path_to_store + '/' + initrdfile)
os.remove(initrd_path)
shutil.copy(kernel_path, path_to_store + '/' + kernfile)
os.chown(path_to_store + '/' + initrdfile, user_id, grp_id)
os.chmod(path_to_store + '/' + initrdfile, 0644)
os.chown(path_to_store + '/' + kernfile, user_id, grp_id)
os.chmod(path_to_store + '/' + kernfile, 0644)
self.set('kernfile', kernfile)
self.set('initrdfile', initrdfile)
return True
def create_tarball(self):
# TODO check if root
cluster = Cluster(mongo_db=self._mongo_db)
path = cluster.get('path')
user = cluster.get('user')
user_id = pwd.getpwnam(user).pw_uid
grp_id = pwd.getpwnam(user).pw_gid
path_to_store = path + "/torrents"
if not os.path.exists(path_to_store):
os.makedirs(path_to_store)
os.chown(path_to_store, user_id, grp_id)
os.chmod(path_to_store, 0644)
image_path = self.get('path')
real_root = os.open("/", os.O_RDONLY)
os.chroot(image_path)
uid = str(uuid.uuid4())
tarfile = uid + ".tgz"
try:
# dirty, but 4 times faster
tar_out = subprocess.Popen(
[
'/usr/bin/tar',
'-C', '/',
'--one-file-system',
'--xattrs',
'--selinux',
'--acls',
'--checkpoint=100',
'--exclude=./tmp/' + tarfile,
'--use-compress-program=/usr/bin/pigz',
'-c', '-f', '/tmp/' + tarfile, '.'
],
stderr=subprocess.PIPE
)
stat_symb = ['\\', '|', '/', '-']
i = 0
while True:
line = tar_out.stderr.readline()
if line == '':
break
i = i + 1
sys.stdout.write(stat_symb[i % len(stat_symb)])
sys.stdout.write('\r')
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == exceptions.KeyboardInterrupt:
self.log.error('Keyboard interrupt.')
else:
self.log.error(exc_value)
self.log.debug(traceback.format_exc())
if os.path.isfile('/tmp/' + tarfile):
os.remove('/tmp/' + tarfile)
sys.stdout.write('\r')
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
return False
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
# copy image, so permissions and selinux contexts
# will be inherited from parent folder
shutil.copy(image_path + '/tmp/' + tarfile, path_to_store)
os.remove(image_path + '/tmp/' + tarfile)
os.chown(path_to_store + '/' + tarfile, user_id, grp_id)
os.chmod(path_to_store + '/' + tarfile, 0644)
self.set('tarball', str(uid))
return True