def _detectedPackageManager(self):
""" set the proper _getPackage*(self, path)
function to handle rpm or dpkg based on /etc/issue content"""
#rpm based OSes
rpmOSs = ["red hat", "fedora", "suse", "centos", "scientific linux"]
#dpkg based OSes
dpkgOSs = ["debian", "ubuntu"]
f=open('/etc/issue')
issues=f.read()
f.close()
if any(os in issues.lower() for os in rpmOSs):
self._getPackage = self._getPackageRpm
if any(os in issues.lower() for os in dpkgOSs):
self._getPackage = self._getPackageDpkg
if not '_getPackage' in dir(self):
#we could not detect the pakcage manager
self._getPackage = lambda p : None
def is_special_file(path):
"""
:type path: string
:param path: a path to a file
:rtype: bool
:return: it returns true if the path points to a file which contains
personal data
"""
return any([ path.endswith(i) for i in specialFile ])
def is_special_file(path):
"""
:type path: string
:param path: a path to a file
:rtype: bool
:return: it returns true if the path points to a file which contains
personal data
"""
return any([path.endswith(i) for i in specialFile])
def _detectedPackageManager(self):
""" set the proper _getPackage*(self, path)
function to handle rpm or dpkg based on /etc/issue content"""
#rpm based OSes
rpmOSs = ["red hat", "fedora", "suse", "centos", "scientific linux"]
#dpkg based OSes
dpkgOSs = ["debian", "ubuntu"]
if os.path.exists('/etc/issue.net'):
f = open('/etc/issue.net')
issues = f.read()
f.close()
if any(os in issues.lower() for os in rpmOSs):
self._getPackage = self._getPackageRpm
return
elif any(os in issues.lower() for os in dpkgOSs):
self._getPackage = self._getPackageDpkg
return
if not '_getPackage' in dir(self):
#we could not detect the pakcage manager
self._getPackage = lambda p: None
def is_special_folder(path):
"""
return true if path is to be considered special, which means it should
be skipped from archivingi, checksumming, etc.
:type path: string
:param path: an absolute path to the file
:rtype: bool
:return: True if the given path is special
"""
return any([ path.startswith(i) for i in specialFolders ])
def is_special_folder(path):
"""
return true if path is to be considered special, which means it should
be skipped from archivingi, checksumming, etc.
:type path: string
:param path: an absolute path to the file
:rtype: bool
:return: True if the given path is special
"""
return any([path.startswith(i) for i in specialFolders])
def _make_rpm(self, base_path, rpm_name):
""" makes an rpm called rpm_name starting from base_path
return False if something went wrong
"""
# rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/
logger.info("RPM " + rpm_name + " root dir " + base_path)
(output, retcode) = utils.getOutputAsList( ["rocks", "create",
"package", base_path + "/*", rpm_name, "prefix=/"])
if any([i for i in output if 'RPM build errors' in i ]):
logger.error(' > ' + '\n > '.join(output))
logger.error("Error building " + rpm_name + " RPM package\n")
return False
logger.debug(' > '+ '\n > '.join(output))
shutil.rmtree(base_path)
return True
def _make_rpm(self, base_path, rpm_name):
""" makes an rpm called rpm_name starting from base_path
return False if something went wrong
"""
# rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/
logger.info("RPM " + rpm_name + " root dir " + base_path)
(output, retcode) = utils.getOutputAsList([
"rocks", "create", "package", base_path + "/*", rpm_name,
"prefix=/"
])
if any([i for i in output if 'RPM build errors' in i]):
logger.error(' > ' + '\n > '.join(output))
logger.error("Error building " + rpm_name + " RPM package\n")
return False
logger.debug(' > ' + '\n > '.join(output))
shutil.rmtree(base_path)
return True
def make_roll(self, fingerprint_base_path, use_remapping = False):
"""
It creates a roll from a swirl archive.
:type fingerprint_base_path: string
:param fingerprint_base_path: a string pointing to the base path of the
fingerprint source code. Used to find the
remapper source code
:type use_remapping: bool
:param use_remapping: if True it will use the remapper technology when
creating the roll
:rtype: bool
:return: it returns false in case of failure
"""
if not os.path.exists(self.archive_filename) :
logger.error("The file " + self.archive_filename + " does not exist" +
" (specify a different one with -f option)")
return False
# this is the list of package we will have to hadd
self.packages = set()
self.skipped_swfs = set()
# this is a list of swirlFile which will need to be installed
# the additional self.files[0].source_path attribute has been added
self.files = []
# keeps track of already processed package in self._resolve_file()
# so we do not process a package twice
self.processed_package = []
# internal swirl package we want to include in the final rpm
self.wanted_pcks = set()
# list of rpm pakcage we have to exclude
self.disable_pcks = set()
#
# ---------------- read the content of the archive
#
temp_workdir = tempfile.mkdtemp()
logger.info("Extracting archive in %s..." % temp_workdir)
tar_tmp_dir = os.path.join(temp_workdir, def_base_dir)
archive_file = tarfile.open(self.archive_filename, 'r:gz')
archive_file.extractall(temp_workdir)
archive_file.close()
# open swirl
logger.info("Reading swirl %s" % os.path.join(temp_workdir, def_swirl_path))
self.swirl = sergeant.readFromPickle(os.path.join(temp_workdir,
def_swirl_path)).swirl
#
# ---------------- recursively resolve all dependencies of the execedFile
#
for swf in self.swirl.execedFiles:
self._resolve_file(swf, use_remapping)
logger.debug("Dependency resolution terminated. Skipped swirl Files:\n - " +
'\n - '.join([i.path for i in self.skipped_swfs]))
#
# ---------------- make rpms with all the files
#
# list of user that should be added
self.users = set()
rpm_tmp_dir = tempfile.mkdtemp()
home_rpm_tmp_dir = tempfile.mkdtemp()
rpm_list = set()
remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path
# laydown the file
for swf in self.files:
source_path = os.path.join(tar_tmp_dir, str(swf.md5sum),
os.path.basename(swf.path))
if not os.path.exists(source_path) :
# if the file is not in the archive do not go on
logger.debug("File " + source_path + " is not present in the archive")
continue
# if use_remapping = true swf must be executable
# if use_remapping = false just follow the first swf.path.startswith("/home/")
if swf.path.startswith("/home/"):
# files in /home need special treatment 1. we need to create a user
# 2 they need to go in /export/home only on the Frontend
rpm_list.add((home_rpm_tmp_dir,self.roll_name + "-home"))
tmp_user = swf.path.split("/home/",1)[1]
self.users.add(tmp_user.split("/",1)[0])
rpm_prefix_dir = home_rpm_tmp_dir + "/export"
else:
rpm_list.add((rpm_tmp_dir,self.roll_name))
rpm_prefix_dir = rpm_tmp_dir
dest_path = rpm_prefix_dir + swf.path
if not os.path.exists( os.path.dirname(dest_path) ):
os.makedirs( os.path.dirname(dest_path) )
if getattr(swf, 'executable', False):
# we need a wrapper script to set the environment
shutil.copy2(source_path, dest_path + ".orig")
f=open(dest_path, 'w')
f.write("#!/bin/bash\n\n")
ldconf_written = False
env = None
if 'ELF' not in swf.type:
# this is not a ELF but is a script so we need to get the
# env from its parent swirl file (the interpreter)
for execswf in self.swirl.execedFiles:
if swf in execswf.openedFiles[execswf.path]:
env = execswf.env
break
else:
env = swf.env
if env == None:
logger.error('Unable to find interpreter for ', swf)
logger.error('Failing on ', swf)
return False
for env_variable in env:
if '=' not in env_variable:
continue
if env_variable.startswith('HYDI'):
# MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate
# subprocesses the control socket
continue
variable_name = env_variable.split('=')[0]
variable_value = env_variable.split('=')[1]
if any([ env_variable.startswith(i) for i in self._append_variables]):
# for these variables we want to add their content to
# the corresponding system variable values
if self.swirl.ldconf_paths and env_variable.startswith('LD_LIBRARY_PATH'):
variable_value = variable_value + ':' + ':'.join(self.swirl.ldconf_paths)
ldconf_written = True
f.write("export " + variable_name + "=\"" + variable_value + ":$" +
variable_name + "\"\n")
else:
# for all the other variables we simply want to define them
# if they are not already defined them
f.write("if [ -z \"$" + variable_name + "\" ]; then export " +
variable_name + "=\"" + variable_value + "\"; fi\n")
if not ldconf_written and self.swirl.ldconf_paths:
f.write("export LD_LIBRARY_PATH=\"" +
':'.join( self.swirl.ldconf_paths ) + ':$LD_LIBRARY_PATH\"\n')
f.write("\n")
if use_remapping and 'ELF' in swf.type:
f.write(self._remapper_executable + " ")
loader = self.swirl.getLoader(swf)
if loader:
f.write(self._remapper_base_path +\
loader.path + " ")
f.write(swf.path + ".orig $@\n")
f.close()
os.chmod(dest_path, 0755)
else:
if use_remapping:
tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path)
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
shutil.copy2(source_path, tmp_path + '/' + os.path.basename(swf.path))
else:
shutil.copy2(source_path, dest_path)
# if use remapping we don't need the symlinks
if use_remapping and not getattr(swf, 'executable', False):
continue
# and the symlinks
for i in swf.links:
dest_link = rpm_prefix_dir + i
# source link must be without the rpm_tmp_dir part
if not os.path.isdir(os.path.dirname(dest_link)):
os.makedirs(os.path.dirname(dest_link))
os.symlink( swf.path, dest_link)
#
# ---------------- create file mapping and include remapper in the RPM
#
if use_remapping :
if not os.path.exists(rpm_tmp_dir + "/etc"):
os.mkdir(rpm_tmp_dir + "/etc")
make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping",
self._remapper_base_path)
build_remapper_path = fingerprint_base_path + '/remapper'
(output, retcode) = utils.getOutputAsList( ["make", "-C",
build_remapper_path] )
if retcode :
logger.error("Unable to built remapper")
logger.error("You need to install make and gcc")
logger.error(" > " + "\n > ".join(output))
return False
logger.debug(' > '+ '\n > '.join(output))
remapper_basedir = rpm_tmp_dir + os.path.dirname(self._remapper_executable)
if not os.path.exists(remapper_basedir):
os.makedirs(remapper_basedir)
shutil.copy2(build_remapper_path + "/remapper", remapper_basedir)
#let's notify we have to build the base RPM
rpm_list.add((rpm_tmp_dir,self.roll_name))
#
# ---------------- files are in place so let's make the RPMs
#
for (base_dir, rpm_name) in rpm_list:
if self._make_rpm(base_dir, rpm_name):
if '-home-' not in rpm_name:
self.packages.add(rpm_name)
else:
return False
shutil.rmtree(temp_workdir)
#
# ---------------- create roll copy files there and build
#
logger.info("Creating roll " + self.roll_name)
(output, retcode) = utils.getOutputAsList( ["rocks", "create",
"new", "roll", self.roll_name] )
if retcode :
logger.error("Unable to create the roll")
if os.path.exists(self.roll_name):
logger.error("Remove the direcotry: rm -rf %s" % (self.roll_name))
logger.error(" > " + "\n > ".join(output))
return False
shutil.rmtree(self.roll_name + "/src/" + self.roll_name)
shutil.rmtree(self.roll_name + "/src/usersguide")
os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml")
dest = self.roll_name + "/RPMS/" + platform.machine()
os.makedirs(dest)
# copying global RPM
source = glob.glob(self.roll_name + "-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the base-nodea.xml
node_base_xml = self._node_base_xml_top
# 1. install packages
for package in self.packages:
node_base_xml += '<package>' + package + '</package>\n'
# 2. remove pakcages
for package in self.disable_pcks:
node_base_xml += '<package>-' + package + '</package>\n'
# 3. set the paths
new_paths = set()
for swf in self.swirl.execedFiles:
new_paths |= set([os.path.dirname(i) for i in swf.getPaths()])
node_base_xml += self._node_base_xml_bottom % (self.roll_name, ' '.join(new_paths))
self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-base.xml",
node_base_xml)
# copying -home- RPM
source = glob.glob(self.roll_name + "-home-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the server-node
self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-server.xml",
self._node_server_xml % (self.roll_name, ' '.join(self.users)))
# create the graph xml
self._write_file(self.roll_name + "/graphs/default/" + self.roll_name + ".xml",
self._graph_node_xml % (self.roll_name, self.roll_name, self.roll_name))
# make the roll
os.chdir(self.roll_name)
(output, retcode) = utils.getOutputAsList(["make", "roll"])
os.chdir("..")
roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso")
if retcode or len(roll_path) < 1:
# error :-(
logger.error("Unable to make the roll")
logger.error(' > ' + '\n > '.join(output))
return False
logger.error("Roll %s succesfully created.\nTo add it to your distribution:" % roll_path[0])
logger.error("rocks add roll " + roll_path[0])
logger.error("rocks enable roll " + self.roll_name)
logger.error("cd /export/rocks/install")
logger.error("rocks create distro")
logger.error("rocks run roll " + self.roll_name + " | bash")
return True
def make_roll(self, fingerprint_base_path, use_remapping=False):
"""
It creates a roll from a swirl archive.
:type fingerprint_base_path: string
:param fingerprint_base_path: a string pointing to the base path of the
fingerprint source code. Used to find the
remapper source code
:type use_remapping: bool
:param use_remapping: if True it will use the remapper technology when
creating the roll
:rtype: bool
:return: it returns false in case of failure
"""
if not os.path.exists(self.archive_filename):
logger.error("The file " + self.archive_filename +
" does not exist" +
" (specify a different one with -f option)")
return False
# this is the list of package we will have to hadd
self.packages = set()
self.skipped_swfs = set()
# this is a list of swirlFile which will need to be installed
# the additional self.files[0].source_path attribute has been added
self.files = []
# keeps track of already processed package in self._resolve_file()
# so we do not process a package twice
self.processed_package = []
# internal swirl package we want to include in the final rpm
self.wanted_pcks = set()
# list of rpm pakcage we have to exclude
self.disable_pcks = set()
#
# ---------------- read the content of the archive
#
temp_workdir = tempfile.mkdtemp()
logger.info("Extracting archive in %s..." % temp_workdir)
tar_tmp_dir = os.path.join(temp_workdir, def_base_dir)
archive_file = tarfile.open(self.archive_filename, 'r:gz')
archive_file.extractall(temp_workdir)
archive_file.close()
# open swirl
logger.info("Reading swirl %s" %
os.path.join(temp_workdir, def_swirl_path))
self.swirl = sergeant.readFromPickle(
os.path.join(temp_workdir, def_swirl_path)).swirl
#
# ---------------- recursively resolve all dependencies of the execedFile
#
for swf in self.swirl.execedFiles:
self._resolve_file(swf, use_remapping)
logger.debug(
"Dependency resolution terminated. Skipped swirl Files:\n - " +
'\n - '.join([i.path for i in self.skipped_swfs]))
#
# ---------------- make rpms with all the files
#
# list of user that should be added
self.users = set()
rpm_tmp_dir = tempfile.mkdtemp()
home_rpm_tmp_dir = tempfile.mkdtemp()
rpm_list = set()
remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path
# laydown the file
for swf in self.files:
source_path = os.path.join(tar_tmp_dir, str(swf.md5sum),
os.path.basename(swf.path))
if not os.path.exists(source_path):
# if the file is not in the archive do not go on
logger.debug("File " + source_path +
" is not present in the archive")
continue
# if use_remapping = true swf must be executable
# if use_remapping = false just follow the first swf.path.startswith("/home/")
if swf.path.startswith("/home/"):
# files in /home need special treatment 1. we need to create a user
# 2 they need to go in /export/home only on the Frontend
rpm_list.add((home_rpm_tmp_dir, self.roll_name + "-home"))
tmp_user = swf.path.split("/home/", 1)[1]
self.users.add(tmp_user.split("/", 1)[0])
rpm_prefix_dir = home_rpm_tmp_dir + "/export"
else:
rpm_list.add((rpm_tmp_dir, self.roll_name))
rpm_prefix_dir = rpm_tmp_dir
dest_path = rpm_prefix_dir + swf.path
if not os.path.exists(os.path.dirname(dest_path)):
os.makedirs(os.path.dirname(dest_path))
if getattr(swf, 'executable', False):
# we need a wrapper script to set the environment
shutil.copy2(source_path, dest_path + ".orig")
f = open(dest_path, 'w')
f.write("#!/bin/bash\n\n")
ldconf_written = False
env = None
if 'ELF' not in swf.type:
# this is not a ELF but is a script so we need to get the
# env from its parent swirl file (the interpreter)
for execswf in self.swirl.execedFiles:
if swf in execswf.openedFiles[execswf.path]:
env = execswf.env
break
else:
env = swf.env
if env == None:
logger.error('Unable to find interpreter for ', swf)
logger.error('Failing on ', swf)
return False
for env_variable in env:
if '=' not in env_variable:
continue
if env_variable.startswith('HYDI'):
# MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate
# subprocesses the control socket
continue
variable_name = env_variable.split('=')[0]
variable_value = env_variable.split('=')[1]
if any([
env_variable.startswith(i)
for i in self._append_variables
]):
# for these variables we want to add their content to
# the corresponding system variable values
if self.swirl.ldconf_paths and env_variable.startswith(
'LD_LIBRARY_PATH'):
variable_value = variable_value + ':' + ':'.join(
self.swirl.ldconf_paths)
ldconf_written = True
f.write("export " + variable_name + "=\"" +
variable_value + ":$" + variable_name + "\"\n")
else:
# for all the other variables we simply want to define them
# if they are not already defined them
f.write("if [ -z \"$" + variable_name +
"\" ]; then export " + variable_name + "=\"" +
variable_value + "\"; fi\n")
if not ldconf_written and self.swirl.ldconf_paths:
f.write("export LD_LIBRARY_PATH=\"" +
':'.join(self.swirl.ldconf_paths) +
':$LD_LIBRARY_PATH\"\n')
f.write("\n")
if use_remapping and 'ELF' in swf.type:
f.write(self._remapper_executable + " ")
loader = self.swirl.getLoader(swf)
if loader:
f.write(self._remapper_base_path +\
loader.path + " ")
f.write(swf.path + ".orig $@\n")
f.close()
os.chmod(dest_path, 0755)
else:
if use_remapping:
tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path)
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
shutil.copy2(source_path,
tmp_path + '/' + os.path.basename(swf.path))
else:
shutil.copy2(source_path, dest_path)
# if use remapping we don't need the symlinks
if use_remapping and not getattr(swf, 'executable', False):
continue
# and the symlinks
for i in swf.links:
dest_link = rpm_prefix_dir + i
# source link must be without the rpm_tmp_dir part
if not os.path.isdir(os.path.dirname(dest_link)):
os.makedirs(os.path.dirname(dest_link))
os.symlink(swf.path, dest_link)
#
# ---------------- create file mapping and include remapper in the RPM
#
if use_remapping:
if not os.path.exists(rpm_tmp_dir + "/etc"):
os.mkdir(rpm_tmp_dir + "/etc")
make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping",
self._remapper_base_path)
build_remapper_path = fingerprint_base_path + '/remapper'
(output, retcode) = utils.getOutputAsList(
["make", "-C", build_remapper_path])
if retcode:
logger.error("Unable to built remapper")
logger.error("You need to install make and gcc")
logger.error(" > " + "\n > ".join(output))
return False
logger.debug(' > ' + '\n > '.join(output))
remapper_basedir = rpm_tmp_dir + os.path.dirname(
self._remapper_executable)
if not os.path.exists(remapper_basedir):
os.makedirs(remapper_basedir)
shutil.copy2(build_remapper_path + "/remapper", remapper_basedir)
#let's notify we have to build the base RPM
rpm_list.add((rpm_tmp_dir, self.roll_name))
#
# ---------------- files are in place so let's make the RPMs
#
for (base_dir, rpm_name) in rpm_list:
if self._make_rpm(base_dir, rpm_name):
if '-home-' not in rpm_name:
self.packages.add(rpm_name)
else:
return False
shutil.rmtree(temp_workdir)
#
# ---------------- create roll copy files there and build
#
logger.info("Creating roll " + self.roll_name)
(output, retcode) = utils.getOutputAsList(
["rocks", "create", "new", "roll", self.roll_name])
if retcode:
logger.error("Unable to create the roll")
if os.path.exists(self.roll_name):
logger.error("Remove the direcotry: rm -rf %s" %
(self.roll_name))
logger.error(" > " + "\n > ".join(output))
return False
shutil.rmtree(self.roll_name + "/src/" + self.roll_name)
shutil.rmtree(self.roll_name + "/src/usersguide")
os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml")
dest = self.roll_name + "/RPMS/" + platform.machine()
os.makedirs(dest)
# copying global RPM
source = glob.glob(self.roll_name + "-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the base-nodea.xml
node_base_xml = self._node_base_xml_top
# 1. install packages
for package in self.packages:
node_base_xml += '<package>' + package + '</package>\n'
# 2. remove pakcages
for package in self.disable_pcks:
node_base_xml += '<package>-' + package + '</package>\n'
# 3. set the paths
new_paths = set()
for swf in self.swirl.execedFiles:
new_paths |= set([os.path.dirname(i) for i in swf.getPaths()])
node_base_xml += self._node_base_xml_bottom % (self.roll_name,
' '.join(new_paths))
self._write_file(
self.roll_name + "/nodes/" + self.roll_name + "-base.xml",
node_base_xml)
# copying -home- RPM
source = glob.glob(self.roll_name + "-home-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the server-node
self._write_file(
self.roll_name + "/nodes/" + self.roll_name + "-server.xml",
self._node_server_xml % (self.roll_name, ' '.join(self.users)))
# create the graph xml
self._write_file(
self.roll_name + "/graphs/default/" + self.roll_name + ".xml",
self._graph_node_xml %
(self.roll_name, self.roll_name, self.roll_name))
# make the roll
os.chdir(self.roll_name)
(output, retcode) = utils.getOutputAsList(["make", "roll"])
os.chdir("..")
roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso")
if retcode or len(roll_path) < 1:
# error :-(
logger.error("Unable to make the roll")
logger.error(' > ' + '\n > '.join(output))
return False
logger.error(
"Roll %s succesfully created.\nTo add it to your distribution:" %
roll_path[0])
logger.error("rocks add roll " + roll_path[0])
logger.error("rocks enable roll " + self.roll_name)
logger.error("cd /export/rocks/install")
logger.error("rocks create distro")
logger.error("rocks run roll " + self.roll_name + " | bash")
return True
