def archive(self):
"""
It triggers the creation of the archive.
:rtype: bool
:return: it returns false in case of failure
"""
#we need a valid swirl
if not self.sergeant.check():
logger.error("The fingerprint " + self.sergeant.filename +
" fails:\n " +
"\n ".join(self.sergeant.getError()) +
"\n\nThe archive creation failed.\n")
return False
# prepare the folders for the tar
base_tar = tempfile.mkdtemp()
base_path = os.path.join(base_tar, def_base_dir)
os.mkdir(base_path)
# copy all the files referenced by this swirl
for swf in self.sergeant.swirl.swirlFiles:
if swf.path[0] == '$' or sergeant.is_special_folder(swf.path) or \
is_special_file(swf.path):
#TODO maybe we could keep user data into a special folder?
# this file belongs to the special folders let's skip it
continue
if os.path.exists(swf.path) and swf.md5sum:
# the file was not a temporary file
dest_path_dir = os.path.join(base_path, swf.md5sum)
dest_path_full = os.path.join(dest_path_dir,
os.path.basename(swf.path))
if not os.path.exists(dest_path_full):
# do not copy twice the same file
if not os.path.exists(dest_path_dir):
os.mkdir(dest_path_dir)
shutil.copy2(swf.path, dest_path_dir)
if sergeant.prelink:
utils.getOutputAsList(
[sergeant.prelink, "-u", dest_path_full])
#for i in swf.links:
# new_link = os.path.join(temp_path, os.path.basename(i))
# if not os.path.exists( new_link ):
# os.symlink( os.path.basename(swf.path), new_link)
# copy the swirl itself
shutil.copy2(self.sergeant.filename,
os.path.join(base_tar, def_swirl_path))
# let's do the tar
tar = tarfile.open(self.archive_filename, "w:gz")
cwd = os.getcwd()
os.chdir(base_tar)
tar.add(".")
tar.close()
os.chdir(cwd)
shutil.rmtree(base_tar)
return True
def searchModules(self):
"""
It searches for missing dependencies using the 'module' command line.
:meth:`check` should be called before this
:rtype: string
:return: with a human readable list of module which can satisfy
missing dependencies
"""
# loop through all the modules
retDict = {}
(output, retval) = utils.getOutputAsList(
["bash", "-c", "module -t avail 2>&1"])
if retval:
print "Unable to run module command, verify it\'s in the path."
return ""
for module in output:
# in the output there are some paths! remove them e.g. "/opt/module/blabla:"
if ':' in module:
continue
# remove (default)
module = module.split("(default)")[0]
(output, retval) = utils.getOutputAsList(
["bash", "-c", "module show " + module + " 2>&1"])
if retval:
#print "Unable to fetch module information: \'module show " + module + "\'"
# all module which depend on another module return 1
pass
for line in output:
if 'LD_LIBRARY_PATH' in line:
# we found another path to scan
path = line.split('LD_LIBRARY_PATH')[1]
path = [i.strip() for i in path.split(":")] #strip
PluginManager.systemPath = path # update path
for dep in self.missingDeps:
if PluginManager.getPathToLibrary(dep, False):
#we found a candidate for this missing dependency
if module not in retDict:
retDict[module] = []
retDict[module].append(dep.getName())
retStr = ""
for mod in retDict:
retStr += " " + mod + " satisfies "
num_deps = len(retDict[mod])
if num_deps == len(self.missingDeps):
retStr += "all "
retStr += "" + str(num_deps) + " dependencies:\n"
# print the deps
retStr += " " + "\n ".join(retDict[mod]) + "\n"
return retStr
def archive(self):
"""
It triggers the creation of the archive.
:rtype: bool
:return: it returns false in case of failure
"""
#we need a valid swirl
if not self.sergeant.check():
logger.error("The fingerprint " + self.sergeant.filename + " fails:\n " +
"\n ".join(self.sergeant.getError()) + "\n\nThe archive creation failed.\n")
return False
# prepare the folders for the tar
base_tar = tempfile.mkdtemp()
base_path = os.path.join(base_tar, def_base_dir)
os.mkdir(base_path)
# copy all the files referenced by this swirl
for swf in self.sergeant.swirl.swirlFiles:
if swf.path[0] == '$' or sergeant.is_special_folder(swf.path) or \
is_special_file(swf.path):
#TODO maybe we could keep user data into a special folder?
# this file belongs to the special folders let's skip it
continue
if os.path.exists(swf.path) and swf.md5sum:
# the file was not a temporary file
dest_path_dir = os.path.join(base_path, swf.md5sum)
dest_path_full = os.path.join(dest_path_dir, os.path.basename(swf.path))
if not os.path.exists(dest_path_full):
# do not copy twice the same file
if not os.path.exists(dest_path_dir):
os.mkdir(dest_path_dir)
shutil.copy2(swf.path, dest_path_dir)
if sergeant.prelink :
utils.getOutputAsList([sergeant.prelink, "-u", dest_path_full])
#for i in swf.links:
# new_link = os.path.join(temp_path, os.path.basename(i))
# if not os.path.exists( new_link ):
# os.symlink( os.path.basename(swf.path), new_link)
# copy the swirl itself
shutil.copy2(self.sergeant.filename, os.path.join(base_tar, def_swirl_path))
# let's do the tar
tar = tarfile.open(self.archive_filename, "w:gz")
cwd = os.getcwd()
os.chdir(base_tar)
tar.add(".")
tar.close()
os.chdir(cwd)
shutil.rmtree(base_tar)
return True
def searchModules(self):
"""
It searches for missing dependencies using the 'module' command line.
:meth:`check` should be called before this
:rtype: string
:return: with a human readable list of module which can satisfy
missing dependencies
"""
# loop through all the modules
retDict = {}
(output, retval) = utils.getOutputAsList(["bash", "-c", "module -t avail 2>&1"])
if retval:
print "Unable to run module command, verify it\'s in the path."
return ""
for module in output :
# in the output there are some paths! remove them e.g. "/opt/module/blabla:"
if ':' in module:
continue
# remove (default)
module = module.split("(default)")[0]
(output, retval) = utils.getOutputAsList(["bash", "-c",
"module show " + module + " 2>&1"])
if retval:
#print "Unable to fetch module information: \'module show " + module + "\'"
# all module which depend on another module return 1
pass
for line in output:
if 'LD_LIBRARY_PATH' in line:
# we found another path to scan
path = line.split('LD_LIBRARY_PATH')[1]
path = [i.strip() for i in path.split(":")] #strip
PluginManager.systemPath = path # update path
for dep in self.missingDeps:
if PluginManager.getPathToLibrary(dep, False):
#we found a candidate for this missing dependency
if module not in retDict:
retDict[module] = []
retDict[module].append(dep.getName())
retStr = ""
for mod in retDict:
retStr += " " + mod + " satisfies "
num_deps = len(retDict[mod])
if num_deps == len(self.missingDeps):
retStr += "all "
retStr += "" + str(num_deps) + " dependencies:\n"
# print the deps
retStr += " " + "\n ".join(retDict[mod]) + "\n"
return retStr
def getHash(fileName, pluginName):
"""Given a valid fileName it returns a string containing a md5sum
of the file content. If we are running on a system which prelink
binaries (aka RedHat based) the command prelink must be on the PATH"""
global _isPrelink
if _isPrelink == None:
#first execution let's check for prelink
_isPrelink = utils.which("prelink")
if _isPrelink == None:
_isPrelink = ""
else:
print "Using: ", _isPrelink
if pluginName == 'ELF' and len(_isPrelink) > 0:
#let's use prelink for the md5sum
#TODO what if isPrelink fails
(temp, returncode) = utils.getOutputAsList([_isPrelink, '-y', '--md5', fileName])
if returncode == 0:
return temp[0].split()[0]
else:
#undoing prelinking failed for some reasons
pass
try:
#ok let's do standard md5sum
fd=open(fileName)
md=md5()
md.update(fd.read())
fd.close()
return md.hexdigest()
except IOError:
#file not found
return None
def _make_rpm(self, base_path, rpm_name):
""" makes an rpm called rpm_name starting from base_path
return False if something went wrong
"""
# rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/
logger.info("RPM " + rpm_name + " root dir " + base_path)
(output, retcode) = utils.getOutputAsList( ["rocks", "create",
"package", base_path + "/*", rpm_name, "prefix=/"])
if any([i for i in output if 'RPM build errors' in i ]):
logger.error(' > ' + '\n > '.join(output))
logger.error("Error building " + rpm_name + " RPM package\n")
return False
logger.debug(' > '+ '\n > '.join(output))
shutil.rmtree(base_path)
return True
def _make_rpm(self, base_path, rpm_name):
""" makes an rpm called rpm_name starting from base_path
return False if something went wrong
"""
# rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/
logger.info("RPM " + rpm_name + " root dir " + base_path)
(output, retcode) = utils.getOutputAsList([
"rocks", "create", "package", base_path + "/*", rpm_name,
"prefix=/"
])
if any([i for i in output if 'RPM build errors' in i]):
logger.error(' > ' + '\n > '.join(output))
logger.error("Error building " + rpm_name + " RPM package\n")
return False
logger.debug(' > ' + '\n > '.join(output))
shutil.rmtree(base_path)
return True
def _get_ldconf_paths(self):
"""return a the list of path used by the dynamic loader
this list is gathered from what you have in /etc/ld.so.conf"""
return_paths = []
ldconf_cmd = "ldconfig"
if not utils.which(ldconf_cmd) :
ldconf_cmd = "/sbin/ldconfig"
if not utils.which(ldconf_cmd) :
logger.error("Unable to find ldconfig. You need ldconfig in you PATH to properly run fingerprint.")
(output, retcode) = utils.getOutputAsList([ldconf_cmd, "-v"])
#default_paths = ["/lib", "/usr/lib"]
# if run as a user it will fail so we don't care for retcode
for line in output:
if line and line[0] == '/':
return_paths.append(line.split(":")[0])
return return_paths
def getHash(fileName, fileType):
"""
It return a md5 checksum of the given file name. If we are running
on a system which prelink binaries (aka RedHat based) the command
prelink must be on the PATH
:type fileName: string
:param fileName: a path to the file which we want to checksum
:type fileType: string
:param fileType: the file type (the only recognized value is EFL for
triggering the prelink on RHEL base system)
:rtype: string
:return: an hexdadeciaml representation of the md5sum checksum
"""
# let's skip weird stuff
if is_special_folder(fileName):
return ""
if not stat.S_ISREG(os.stat(fileName).st_mode):
# probably a socket, fifo, or similar
return ""
if fileType == 'ELF' and prelink:
#let's use prelink for the md5sum
#TODO what if isPrelink fails
(temp, returncode) = utils.getOutputAsList(
[prelink, '-y', '--md5', fileName])
if returncode == 0:
return temp[0].split()[0]
else:
#undoing prelinking failed for some reasons
pass
try:
# ok let's do standard md5sum
fd = open(fileName)
md = md5()
md.update(fd.read())
fd.close()
return md.hexdigest()
except IOError:
#file not found
return None
def getHash(fileName, fileType):
"""
It return a md5 checksum of the given file name. If we are running
on a system which prelink binaries (aka RedHat based) the command
prelink must be on the PATH
:type fileName: string
:param fileName: a path to the file which we want to checksum
:type fileType: string
:param fileType: the file type (the only recognized value is EFL for
triggering the prelink on RHEL base system)
:rtype: string
:return: an hexdadeciaml representation of the md5sum checksum
"""
# let's skip weird stuff
if is_special_folder(fileName):
return ""
if not stat.S_ISREG( os.stat(fileName).st_mode ):
# probably a socket, fifo, or similar
return ""
if fileType == 'ELF' and prelink:
#let's use prelink for the md5sum
#TODO what if isPrelink fails
(temp, returncode) = utils.getOutputAsList([prelink, '-y', '--md5', fileName])
if returncode == 0:
return temp[0].split()[0]
else:
#undoing prelinking failed for some reasons
pass
try:
# ok let's do standard md5sum
fd=open(fileName)
md=md5()
md.update(fd.read())
fd.close()
return md.hexdigest()
except IOError:
#file not found
return None
def _get_ldconf_paths(self):
"""return a the list of path used by the dynamic loader
this list is gathered from what you have in /etc/ld.so.conf"""
return_paths = []
ldconf_cmd = "ldconfig"
if not utils.which(ldconf_cmd):
ldconf_cmd = "/sbin/ldconfig"
if not utils.which(ldconf_cmd):
logger.error(
"Unable to find ldconfig. You need ldconfig in you PATH to properly run fingerprint."
)
(output, retcode) = utils.getOutputAsList([ldconf_cmd, "-v"])
#default_paths = ["/lib", "/usr/lib"]
# if run as a user it will fail so we don't care for retcode
for line in output:
if line and line[0] == '/':
return_paths.append(line.split(":")[0])
return return_paths
def make_roll(self, fingerprint_base_path, use_remapping = False):
"""
It creates a roll from a swirl archive.
:type fingerprint_base_path: string
:param fingerprint_base_path: a string pointing to the base path of the
fingerprint source code. Used to find the
remapper source code
:type use_remapping: bool
:param use_remapping: if True it will use the remapper technology when
creating the roll
:rtype: bool
:return: it returns false in case of failure
"""
if not os.path.exists(self.archive_filename) :
logger.error("The file " + self.archive_filename + " does not exist" +
" (specify a different one with -f option)")
return False
# this is the list of package we will have to hadd
self.packages = set()
self.skipped_swfs = set()
# this is a list of swirlFile which will need to be installed
# the additional self.files[0].source_path attribute has been added
self.files = []
# keeps track of already processed package in self._resolve_file()
# so we do not process a package twice
self.processed_package = []
# internal swirl package we want to include in the final rpm
self.wanted_pcks = set()
# list of rpm pakcage we have to exclude
self.disable_pcks = set()
#
# ---------------- read the content of the archive
#
temp_workdir = tempfile.mkdtemp()
logger.info("Extracting archive in %s..." % temp_workdir)
tar_tmp_dir = os.path.join(temp_workdir, def_base_dir)
archive_file = tarfile.open(self.archive_filename, 'r:gz')
archive_file.extractall(temp_workdir)
archive_file.close()
# open swirl
logger.info("Reading swirl %s" % os.path.join(temp_workdir, def_swirl_path))
self.swirl = sergeant.readFromPickle(os.path.join(temp_workdir,
def_swirl_path)).swirl
#
# ---------------- recursively resolve all dependencies of the execedFile
#
for swf in self.swirl.execedFiles:
self._resolve_file(swf, use_remapping)
logger.debug("Dependency resolution terminated. Skipped swirl Files:\n - " +
'\n - '.join([i.path for i in self.skipped_swfs]))
#
# ---------------- make rpms with all the files
#
# list of user that should be added
self.users = set()
rpm_tmp_dir = tempfile.mkdtemp()
home_rpm_tmp_dir = tempfile.mkdtemp()
rpm_list = set()
remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path
# laydown the file
for swf in self.files:
source_path = os.path.join(tar_tmp_dir, str(swf.md5sum),
os.path.basename(swf.path))
if not os.path.exists(source_path) :
# if the file is not in the archive do not go on
logger.debug("File " + source_path + " is not present in the archive")
continue
# if use_remapping = true swf must be executable
# if use_remapping = false just follow the first swf.path.startswith("/home/")
if swf.path.startswith("/home/"):
# files in /home need special treatment 1. we need to create a user
# 2 they need to go in /export/home only on the Frontend
rpm_list.add((home_rpm_tmp_dir,self.roll_name + "-home"))
tmp_user = swf.path.split("/home/",1)[1]
self.users.add(tmp_user.split("/",1)[0])
rpm_prefix_dir = home_rpm_tmp_dir + "/export"
else:
rpm_list.add((rpm_tmp_dir,self.roll_name))
rpm_prefix_dir = rpm_tmp_dir
dest_path = rpm_prefix_dir + swf.path
if not os.path.exists( os.path.dirname(dest_path) ):
os.makedirs( os.path.dirname(dest_path) )
if getattr(swf, 'executable', False):
# we need a wrapper script to set the environment
shutil.copy2(source_path, dest_path + ".orig")
f=open(dest_path, 'w')
f.write("#!/bin/bash\n\n")
ldconf_written = False
env = None
if 'ELF' not in swf.type:
# this is not a ELF but is a script so we need to get the
# env from its parent swirl file (the interpreter)
for execswf in self.swirl.execedFiles:
if swf in execswf.openedFiles[execswf.path]:
env = execswf.env
break
else:
env = swf.env
if env == None:
logger.error('Unable to find interpreter for ', swf)
logger.error('Failing on ', swf)
return False
for env_variable in env:
if '=' not in env_variable:
continue
if env_variable.startswith('HYDI'):
# MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate
# subprocesses the control socket
continue
variable_name = env_variable.split('=')[0]
variable_value = env_variable.split('=')[1]
if any([ env_variable.startswith(i) for i in self._append_variables]):
# for these variables we want to add their content to
# the corresponding system variable values
if self.swirl.ldconf_paths and env_variable.startswith('LD_LIBRARY_PATH'):
variable_value = variable_value + ':' + ':'.join(self.swirl.ldconf_paths)
ldconf_written = True
f.write("export " + variable_name + "=\"" + variable_value + ":$" +
variable_name + "\"\n")
else:
# for all the other variables we simply want to define them
# if they are not already defined them
f.write("if [ -z \"$" + variable_name + "\" ]; then export " +
variable_name + "=\"" + variable_value + "\"; fi\n")
if not ldconf_written and self.swirl.ldconf_paths:
f.write("export LD_LIBRARY_PATH=\"" +
':'.join( self.swirl.ldconf_paths ) + ':$LD_LIBRARY_PATH\"\n')
f.write("\n")
if use_remapping and 'ELF' in swf.type:
f.write(self._remapper_executable + " ")
loader = self.swirl.getLoader(swf)
if loader:
f.write(self._remapper_base_path +\
loader.path + " ")
f.write(swf.path + ".orig $@\n")
f.close()
os.chmod(dest_path, 0755)
else:
if use_remapping:
tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path)
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
shutil.copy2(source_path, tmp_path + '/' + os.path.basename(swf.path))
else:
shutil.copy2(source_path, dest_path)
# if use remapping we don't need the symlinks
if use_remapping and not getattr(swf, 'executable', False):
continue
# and the symlinks
for i in swf.links:
dest_link = rpm_prefix_dir + i
# source link must be without the rpm_tmp_dir part
if not os.path.isdir(os.path.dirname(dest_link)):
os.makedirs(os.path.dirname(dest_link))
os.symlink( swf.path, dest_link)
#
# ---------------- create file mapping and include remapper in the RPM
#
if use_remapping :
if not os.path.exists(rpm_tmp_dir + "/etc"):
os.mkdir(rpm_tmp_dir + "/etc")
make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping",
self._remapper_base_path)
build_remapper_path = fingerprint_base_path + '/remapper'
(output, retcode) = utils.getOutputAsList( ["make", "-C",
build_remapper_path] )
if retcode :
logger.error("Unable to built remapper")
logger.error("You need to install make and gcc")
logger.error(" > " + "\n > ".join(output))
return False
logger.debug(' > '+ '\n > '.join(output))
remapper_basedir = rpm_tmp_dir + os.path.dirname(self._remapper_executable)
if not os.path.exists(remapper_basedir):
os.makedirs(remapper_basedir)
shutil.copy2(build_remapper_path + "/remapper", remapper_basedir)
#let's notify we have to build the base RPM
rpm_list.add((rpm_tmp_dir,self.roll_name))
#
# ---------------- files are in place so let's make the RPMs
#
for (base_dir, rpm_name) in rpm_list:
if self._make_rpm(base_dir, rpm_name):
if '-home-' not in rpm_name:
self.packages.add(rpm_name)
else:
return False
shutil.rmtree(temp_workdir)
#
# ---------------- create roll copy files there and build
#
logger.info("Creating roll " + self.roll_name)
(output, retcode) = utils.getOutputAsList( ["rocks", "create",
"new", "roll", self.roll_name] )
if retcode :
logger.error("Unable to create the roll")
if os.path.exists(self.roll_name):
logger.error("Remove the direcotry: rm -rf %s" % (self.roll_name))
logger.error(" > " + "\n > ".join(output))
return False
shutil.rmtree(self.roll_name + "/src/" + self.roll_name)
shutil.rmtree(self.roll_name + "/src/usersguide")
os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml")
dest = self.roll_name + "/RPMS/" + platform.machine()
os.makedirs(dest)
# copying global RPM
source = glob.glob(self.roll_name + "-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the base-nodea.xml
node_base_xml = self._node_base_xml_top
# 1. install packages
for package in self.packages:
node_base_xml += '<package>' + package + '</package>\n'
# 2. remove pakcages
for package in self.disable_pcks:
node_base_xml += '<package>-' + package + '</package>\n'
# 3. set the paths
new_paths = set()
for swf in self.swirl.execedFiles:
new_paths |= set([os.path.dirname(i) for i in swf.getPaths()])
node_base_xml += self._node_base_xml_bottom % (self.roll_name, ' '.join(new_paths))
self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-base.xml",
node_base_xml)
# copying -home- RPM
source = glob.glob(self.roll_name + "-home-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the server-node
self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-server.xml",
self._node_server_xml % (self.roll_name, ' '.join(self.users)))
# create the graph xml
self._write_file(self.roll_name + "/graphs/default/" + self.roll_name + ".xml",
self._graph_node_xml % (self.roll_name, self.roll_name, self.roll_name))
# make the roll
os.chdir(self.roll_name)
(output, retcode) = utils.getOutputAsList(["make", "roll"])
os.chdir("..")
roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso")
if retcode or len(roll_path) < 1:
# error :-(
logger.error("Unable to make the roll")
logger.error(' > ' + '\n > '.join(output))
return False
logger.error("Roll %s succesfully created.\nTo add it to your distribution:" % roll_path[0])
logger.error("rocks add roll " + roll_path[0])
logger.error("rocks enable roll " + self.roll_name)
logger.error("cd /export/rocks/install")
logger.error("rocks create distro")
logger.error("rocks run roll " + self.roll_name + " | bash")
return True
def make_roll(self, fingerprint_base_path, use_remapping=False):
"""
It creates a roll from a swirl archive.
:type fingerprint_base_path: string
:param fingerprint_base_path: a string pointing to the base path of the
fingerprint source code. Used to find the
remapper source code
:type use_remapping: bool
:param use_remapping: if True it will use the remapper technology when
creating the roll
:rtype: bool
:return: it returns false in case of failure
"""
if not os.path.exists(self.archive_filename):
logger.error("The file " + self.archive_filename +
" does not exist" +
" (specify a different one with -f option)")
return False
# this is the list of package we will have to hadd
self.packages = set()
self.skipped_swfs = set()
# this is a list of swirlFile which will need to be installed
# the additional self.files[0].source_path attribute has been added
self.files = []
# keeps track of already processed package in self._resolve_file()
# so we do not process a package twice
self.processed_package = []
# internal swirl package we want to include in the final rpm
self.wanted_pcks = set()
# list of rpm pakcage we have to exclude
self.disable_pcks = set()
#
# ---------------- read the content of the archive
#
temp_workdir = tempfile.mkdtemp()
logger.info("Extracting archive in %s..." % temp_workdir)
tar_tmp_dir = os.path.join(temp_workdir, def_base_dir)
archive_file = tarfile.open(self.archive_filename, 'r:gz')
archive_file.extractall(temp_workdir)
archive_file.close()
# open swirl
logger.info("Reading swirl %s" %
os.path.join(temp_workdir, def_swirl_path))
self.swirl = sergeant.readFromPickle(
os.path.join(temp_workdir, def_swirl_path)).swirl
#
# ---------------- recursively resolve all dependencies of the execedFile
#
for swf in self.swirl.execedFiles:
self._resolve_file(swf, use_remapping)
logger.debug(
"Dependency resolution terminated. Skipped swirl Files:\n - " +
'\n - '.join([i.path for i in self.skipped_swfs]))
#
# ---------------- make rpms with all the files
#
# list of user that should be added
self.users = set()
rpm_tmp_dir = tempfile.mkdtemp()
home_rpm_tmp_dir = tempfile.mkdtemp()
rpm_list = set()
remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path
# laydown the file
for swf in self.files:
source_path = os.path.join(tar_tmp_dir, str(swf.md5sum),
os.path.basename(swf.path))
if not os.path.exists(source_path):
# if the file is not in the archive do not go on
logger.debug("File " + source_path +
" is not present in the archive")
continue
# if use_remapping = true swf must be executable
# if use_remapping = false just follow the first swf.path.startswith("/home/")
if swf.path.startswith("/home/"):
# files in /home need special treatment 1. we need to create a user
# 2 they need to go in /export/home only on the Frontend
rpm_list.add((home_rpm_tmp_dir, self.roll_name + "-home"))
tmp_user = swf.path.split("/home/", 1)[1]
self.users.add(tmp_user.split("/", 1)[0])
rpm_prefix_dir = home_rpm_tmp_dir + "/export"
else:
rpm_list.add((rpm_tmp_dir, self.roll_name))
rpm_prefix_dir = rpm_tmp_dir
dest_path = rpm_prefix_dir + swf.path
if not os.path.exists(os.path.dirname(dest_path)):
os.makedirs(os.path.dirname(dest_path))
if getattr(swf, 'executable', False):
# we need a wrapper script to set the environment
shutil.copy2(source_path, dest_path + ".orig")
f = open(dest_path, 'w')
f.write("#!/bin/bash\n\n")
ldconf_written = False
env = None
if 'ELF' not in swf.type:
# this is not a ELF but is a script so we need to get the
# env from its parent swirl file (the interpreter)
for execswf in self.swirl.execedFiles:
if swf in execswf.openedFiles[execswf.path]:
env = execswf.env
break
else:
env = swf.env
if env == None:
logger.error('Unable to find interpreter for ', swf)
logger.error('Failing on ', swf)
return False
for env_variable in env:
if '=' not in env_variable:
continue
if env_variable.startswith('HYDI'):
# MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate
# subprocesses the control socket
continue
variable_name = env_variable.split('=')[0]
variable_value = env_variable.split('=')[1]
if any([
env_variable.startswith(i)
for i in self._append_variables
]):
# for these variables we want to add their content to
# the corresponding system variable values
if self.swirl.ldconf_paths and env_variable.startswith(
'LD_LIBRARY_PATH'):
variable_value = variable_value + ':' + ':'.join(
self.swirl.ldconf_paths)
ldconf_written = True
f.write("export " + variable_name + "=\"" +
variable_value + ":$" + variable_name + "\"\n")
else:
# for all the other variables we simply want to define them
# if they are not already defined them
f.write("if [ -z \"$" + variable_name +
"\" ]; then export " + variable_name + "=\"" +
variable_value + "\"; fi\n")
if not ldconf_written and self.swirl.ldconf_paths:
f.write("export LD_LIBRARY_PATH=\"" +
':'.join(self.swirl.ldconf_paths) +
':$LD_LIBRARY_PATH\"\n')
f.write("\n")
if use_remapping and 'ELF' in swf.type:
f.write(self._remapper_executable + " ")
loader = self.swirl.getLoader(swf)
if loader:
f.write(self._remapper_base_path +\
loader.path + " ")
f.write(swf.path + ".orig $@\n")
f.close()
os.chmod(dest_path, 0755)
else:
if use_remapping:
tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path)
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
shutil.copy2(source_path,
tmp_path + '/' + os.path.basename(swf.path))
else:
shutil.copy2(source_path, dest_path)
# if use remapping we don't need the symlinks
if use_remapping and not getattr(swf, 'executable', False):
continue
# and the symlinks
for i in swf.links:
dest_link = rpm_prefix_dir + i
# source link must be without the rpm_tmp_dir part
if not os.path.isdir(os.path.dirname(dest_link)):
os.makedirs(os.path.dirname(dest_link))
os.symlink(swf.path, dest_link)
#
# ---------------- create file mapping and include remapper in the RPM
#
if use_remapping:
if not os.path.exists(rpm_tmp_dir + "/etc"):
os.mkdir(rpm_tmp_dir + "/etc")
make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping",
self._remapper_base_path)
build_remapper_path = fingerprint_base_path + '/remapper'
(output, retcode) = utils.getOutputAsList(
["make", "-C", build_remapper_path])
if retcode:
logger.error("Unable to built remapper")
logger.error("You need to install make and gcc")
logger.error(" > " + "\n > ".join(output))
return False
logger.debug(' > ' + '\n > '.join(output))
remapper_basedir = rpm_tmp_dir + os.path.dirname(
self._remapper_executable)
if not os.path.exists(remapper_basedir):
os.makedirs(remapper_basedir)
shutil.copy2(build_remapper_path + "/remapper", remapper_basedir)
#let's notify we have to build the base RPM
rpm_list.add((rpm_tmp_dir, self.roll_name))
#
# ---------------- files are in place so let's make the RPMs
#
for (base_dir, rpm_name) in rpm_list:
if self._make_rpm(base_dir, rpm_name):
if '-home-' not in rpm_name:
self.packages.add(rpm_name)
else:
return False
shutil.rmtree(temp_workdir)
#
# ---------------- create roll copy files there and build
#
logger.info("Creating roll " + self.roll_name)
(output, retcode) = utils.getOutputAsList(
["rocks", "create", "new", "roll", self.roll_name])
if retcode:
logger.error("Unable to create the roll")
if os.path.exists(self.roll_name):
logger.error("Remove the direcotry: rm -rf %s" %
(self.roll_name))
logger.error(" > " + "\n > ".join(output))
return False
shutil.rmtree(self.roll_name + "/src/" + self.roll_name)
shutil.rmtree(self.roll_name + "/src/usersguide")
os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml")
dest = self.roll_name + "/RPMS/" + platform.machine()
os.makedirs(dest)
# copying global RPM
source = glob.glob(self.roll_name + "-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the base-nodea.xml
node_base_xml = self._node_base_xml_top
# 1. install packages
for package in self.packages:
node_base_xml += '<package>' + package + '</package>\n'
# 2. remove pakcages
for package in self.disable_pcks:
node_base_xml += '<package>-' + package + '</package>\n'
# 3. set the paths
new_paths = set()
for swf in self.swirl.execedFiles:
new_paths |= set([os.path.dirname(i) for i in swf.getPaths()])
node_base_xml += self._node_base_xml_bottom % (self.roll_name,
' '.join(new_paths))
self._write_file(
self.roll_name + "/nodes/" + self.roll_name + "-base.xml",
node_base_xml)
# copying -home- RPM
source = glob.glob(self.roll_name + "-home-1.0-*.rpm")
if len(source) == 1:
logger.info("Coping RPM in: " + dest + "/" + source[0])
shutil.copy2(source[0], dest)
# create the server-node
self._write_file(
self.roll_name + "/nodes/" + self.roll_name + "-server.xml",
self._node_server_xml % (self.roll_name, ' '.join(self.users)))
# create the graph xml
self._write_file(
self.roll_name + "/graphs/default/" + self.roll_name + ".xml",
self._graph_node_xml %
(self.roll_name, self.roll_name, self.roll_name))
# make the roll
os.chdir(self.roll_name)
(output, retcode) = utils.getOutputAsList(["make", "roll"])
os.chdir("..")
roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso")
if retcode or len(roll_path) < 1:
# error :-(
logger.error("Unable to make the roll")
logger.error(' > ' + '\n > '.join(output))
return False
logger.error(
"Roll %s succesfully created.\nTo add it to your distribution:" %
roll_path[0])
logger.error("rocks add roll " + roll_path[0])
logger.error("rocks enable roll " + self.roll_name)
logger.error("cd /export/rocks/install")
logger.error("rocks create distro")
logger.error("rocks run roll " + self.roll_name + " | bash")
return True