示例#1
0
    def archive(self):
        """
        It triggers the creation of the archive.

        :rtype: bool
        :return: it returns false in case of failure
        """
        #we need a valid swirl
        if not self.sergeant.check():
            logger.error("The fingerprint " + self.sergeant.filename +
                         " fails:\n  " +
                         "\n  ".join(self.sergeant.getError()) +
                         "\n\nThe archive creation failed.\n")
            return False
        # prepare the folders for the tar
        base_tar = tempfile.mkdtemp()
        base_path = os.path.join(base_tar, def_base_dir)
        os.mkdir(base_path)
        # copy all the files referenced by this swirl
        for swf in self.sergeant.swirl.swirlFiles:
            if swf.path[0] == '$' or sergeant.is_special_folder(swf.path) or \
                            is_special_file(swf.path):
                #TODO maybe we could keep user data into a special folder?
                # this file belongs to the special folders let's skip it
                continue
            if os.path.exists(swf.path) and swf.md5sum:
                # the file was not a temporary file
                dest_path_dir = os.path.join(base_path, swf.md5sum)
                dest_path_full = os.path.join(dest_path_dir,
                                              os.path.basename(swf.path))
                if not os.path.exists(dest_path_full):
                    # do not copy twice the same file
                    if not os.path.exists(dest_path_dir):
                        os.mkdir(dest_path_dir)
                    shutil.copy2(swf.path, dest_path_dir)
                    if sergeant.prelink:
                        utils.getOutputAsList(
                            [sergeant.prelink, "-u", dest_path_full])
            #for i in swf.links:
            #    new_link = os.path.join(temp_path, os.path.basename(i))
            #    if not os.path.exists( new_link ):
            #        os.symlink( os.path.basename(swf.path), new_link)
        # copy the swirl itself
        shutil.copy2(self.sergeant.filename,
                     os.path.join(base_tar, def_swirl_path))
        # let's do the tar
        tar = tarfile.open(self.archive_filename, "w:gz")
        cwd = os.getcwd()
        os.chdir(base_tar)
        tar.add(".")
        tar.close()
        os.chdir(cwd)
        shutil.rmtree(base_tar)
        return True
示例#2
0
    def searchModules(self):
        """
        It searches for missing dependencies using the 'module' command line.
        :meth:`check` should be called before this

        :rtype: string
        :return: with a human readable list of module which can satisfy
                 missing dependencies
        """
        # loop through all the modules
        retDict = {}
        (output, retval) = utils.getOutputAsList(
            ["bash", "-c", "module -t avail 2>&1"])
        if retval:
            print "Unable to run module command, verify it\'s in the path."
            return ""
        for module in output:
            # in the output there are some paths! remove them e.g. "/opt/module/blabla:"
            if ':' in module:
                continue
            # remove (default)
            module = module.split("(default)")[0]
            (output, retval) = utils.getOutputAsList(
                ["bash", "-c", "module show " + module + " 2>&1"])
            if retval:
                #print "Unable to fetch module information: \'module show " + module + "\'"
                # all module which depend on another module return 1
                pass
            for line in output:
                if 'LD_LIBRARY_PATH' in line:
                    # we found another path to scan
                    path = line.split('LD_LIBRARY_PATH')[1]
                    path = [i.strip() for i in path.split(":")]  #strip
                    PluginManager.systemPath = path  # update path
                    for dep in self.missingDeps:
                        if PluginManager.getPathToLibrary(dep, False):
                            #we found a candidate for this missing dependency
                            if module not in retDict:
                                retDict[module] = []
                            retDict[module].append(dep.getName())
        retStr = ""
        for mod in retDict:
            retStr += "  " + mod + " satisfies "
            num_deps = len(retDict[mod])
            if num_deps == len(self.missingDeps):
                retStr += "all "
            retStr += "" + str(num_deps) + " dependencies:\n"
            # print the deps
            retStr += "    " + "\n    ".join(retDict[mod]) + "\n"
        return retStr
示例#3
0
    def archive(self):
        """
        It triggers the creation of the archive.

        :rtype: bool
        :return: it returns false in case of failure
        """
        #we need a valid swirl
        if not self.sergeant.check():
            logger.error("The fingerprint " + self.sergeant.filename + " fails:\n  " +
                "\n  ".join(self.sergeant.getError()) + "\n\nThe archive creation failed.\n")
            return False
        # prepare the folders for the tar
        base_tar = tempfile.mkdtemp()
        base_path = os.path.join(base_tar, def_base_dir)
        os.mkdir(base_path)
        # copy all the files referenced by this swirl
        for swf in self.sergeant.swirl.swirlFiles:
            if swf.path[0] == '$' or sergeant.is_special_folder(swf.path) or \
                            is_special_file(swf.path):
                #TODO maybe we could keep user data into a special folder?
                # this file belongs to the special folders let's skip it
                continue
            if os.path.exists(swf.path) and swf.md5sum:
		# the file was not a temporary file
                dest_path_dir = os.path.join(base_path, swf.md5sum)
                dest_path_full = os.path.join(dest_path_dir, os.path.basename(swf.path))
                if not os.path.exists(dest_path_full):
                    # do not copy twice the same file
                    if not os.path.exists(dest_path_dir):
                        os.mkdir(dest_path_dir)
                    shutil.copy2(swf.path, dest_path_dir)
                    if sergeant.prelink :
                        utils.getOutputAsList([sergeant.prelink, "-u", dest_path_full])
            #for i in swf.links:
            #    new_link = os.path.join(temp_path, os.path.basename(i))
            #    if not os.path.exists( new_link ):
            #        os.symlink( os.path.basename(swf.path), new_link)
        # copy the swirl itself
        shutil.copy2(self.sergeant.filename, os.path.join(base_tar, def_swirl_path))
        # let's do the tar
        tar = tarfile.open(self.archive_filename, "w:gz")
        cwd = os.getcwd()
        os.chdir(base_tar)
        tar.add(".")
        tar.close()
        os.chdir(cwd)
        shutil.rmtree(base_tar)
        return True
示例#4
0
    def searchModules(self):
        """
        It searches for missing dependencies using the 'module' command line.
        :meth:`check` should be called before this

        :rtype: string
        :return: with a human readable list of module which can satisfy
                 missing dependencies
        """
        # loop through all the modules
        retDict = {}
        (output, retval) = utils.getOutputAsList(["bash", "-c", "module -t avail 2>&1"])
        if retval:
            print "Unable to run module command, verify it\'s in the path."
            return ""
        for module in output :
            # in the output there are some paths! remove them e.g. "/opt/module/blabla:"
            if ':' in module:
                continue
            # remove (default)
            module = module.split("(default)")[0]
            (output, retval) = utils.getOutputAsList(["bash", "-c",
                                                "module show " + module + " 2>&1"])
            if retval:
                #print "Unable to fetch module information: \'module show " + module + "\'"
                # all module which depend on another module return 1
                pass
            for line in output:
                if 'LD_LIBRARY_PATH' in line:
                    # we found another path to scan
                    path = line.split('LD_LIBRARY_PATH')[1]
                    path = [i.strip() for i in path.split(":")] #strip
                    PluginManager.systemPath = path # update path
                    for dep in self.missingDeps:
                        if PluginManager.getPathToLibrary(dep, False):
                            #we found a candidate for this missing dependency
                            if module not in retDict:
                                retDict[module] = []
                            retDict[module].append(dep.getName())
        retStr = ""
        for mod in retDict:
            retStr += "  " + mod + " satisfies "
            num_deps = len(retDict[mod])
            if num_deps == len(self.missingDeps):
                retStr += "all "
            retStr += "" + str(num_deps) + " dependencies:\n"
            # print the deps
            retStr += "    " + "\n    ".join(retDict[mod]) + "\n"
        return retStr
示例#5
0
def getHash(fileName, pluginName):
    """Given a valid fileName it returns a string containing a md5sum
    of the file content. If we are running on a system which prelink
    binaries (aka RedHat based) the command prelink must be on the PATH"""
    global _isPrelink
    if _isPrelink == None:
        #first execution let's check for prelink
        _isPrelink = utils.which("prelink")
        if _isPrelink == None:
            _isPrelink = ""
        else:
            print "Using: ", _isPrelink
    if pluginName == 'ELF' and len(_isPrelink) > 0:
        #let's use prelink for the md5sum
        #TODO what if isPrelink fails
        (temp, returncode) = utils.getOutputAsList([_isPrelink, '-y', '--md5', fileName])
        if returncode == 0:
            return temp[0].split()[0]
        else:
            #undoing prelinking failed for some reasons
            pass
    try:
        #ok let's do standard md5sum
        fd=open(fileName)
        md=md5()
        md.update(fd.read())
        fd.close()
        return md.hexdigest()
    except IOError:
        #file not found
        return None
示例#6
0
    def _make_rpm(self, base_path, rpm_name):
        """ makes an rpm called rpm_name starting from base_path

        return False if something went wrong
        """
        # rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/
        logger.info("RPM " + rpm_name + " root dir " + base_path)
        (output, retcode) = utils.getOutputAsList( ["rocks", "create",
                    "package", base_path + "/*", rpm_name, "prefix=/"])
        if any([i for i in output if 'RPM build errors' in i ]):
            logger.error(' > ' + '\n > '.join(output))
            logger.error("Error building " + rpm_name + " RPM package\n")
            return False
        logger.debug(' > '+ '\n > '.join(output))
        shutil.rmtree(base_path)
        return True
示例#7
0
    def _make_rpm(self, base_path, rpm_name):
        """ makes an rpm called rpm_name starting from base_path

        return False if something went wrong
        """
        # rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/
        logger.info("RPM " + rpm_name + " root dir " + base_path)
        (output, retcode) = utils.getOutputAsList([
            "rocks", "create", "package", base_path + "/*", rpm_name,
            "prefix=/"
        ])
        if any([i for i in output if 'RPM build errors' in i]):
            logger.error(' > ' + '\n > '.join(output))
            logger.error("Error building " + rpm_name + " RPM package\n")
            return False
        logger.debug(' > ' + '\n > '.join(output))
        shutil.rmtree(base_path)
        return True
示例#8
0
    def _get_ldconf_paths(self):
        """return a the list of path used by the dynamic loader

        this list is gathered from what you have in /etc/ld.so.conf"""
        return_paths = []
	ldconf_cmd = "ldconfig"
	if not utils.which(ldconf_cmd) :
            ldconf_cmd = "/sbin/ldconfig"
	if not utils.which(ldconf_cmd) :
	    logger.error("Unable to find ldconfig. You need ldconfig in you PATH to properly run fingerprint.")
        (output, retcode) = utils.getOutputAsList([ldconf_cmd, "-v"])
        #default_paths = ["/lib", "/usr/lib"]

        # if run as a user it will fail so we don't care for retcode
        for line in output:
            if line and line[0] == '/':
                return_paths.append(line.split(":")[0])
        return return_paths
示例#9
0
def getHash(fileName, fileType):
    """
    It return a md5 checksum of the given file name. If we are running
    on a system which prelink binaries (aka RedHat based) the command
    prelink must be on the PATH

    :type fileName: string
    :param fileName: a path to the file which we want to checksum

    :type fileType: string
    :param fileType: the file type (the only recognized value is EFL for
                     triggering the prelink on RHEL base system)

    :rtype: string
    :return: an hexdadeciaml representation of the md5sum checksum
    """
    # let's skip weird stuff
    if is_special_folder(fileName):
        return ""
    if not stat.S_ISREG(os.stat(fileName).st_mode):
        # probably a socket, fifo, or similar
        return ""

    if fileType == 'ELF' and prelink:
        #let's use prelink for the md5sum
        #TODO what if isPrelink fails
        (temp, returncode) = utils.getOutputAsList(
            [prelink, '-y', '--md5', fileName])
        if returncode == 0:
            return temp[0].split()[0]
        else:
            #undoing prelinking failed for some reasons
            pass
    try:
        # ok let's do standard md5sum
        fd = open(fileName)
        md = md5()
        md.update(fd.read())
        fd.close()
        return md.hexdigest()
    except IOError:
        #file not found
        return None
示例#10
0
def getHash(fileName, fileType):
    """
    It return a md5 checksum of the given file name. If we are running
    on a system which prelink binaries (aka RedHat based) the command
    prelink must be on the PATH

    :type fileName: string
    :param fileName: a path to the file which we want to checksum

    :type fileType: string
    :param fileType: the file type (the only recognized value is EFL for
                     triggering the prelink on RHEL base system)

    :rtype: string
    :return: an hexdadeciaml representation of the md5sum checksum
    """
    # let's skip weird stuff
    if is_special_folder(fileName):
        return ""
    if not stat.S_ISREG( os.stat(fileName).st_mode  ):
        # probably a socket, fifo, or similar
        return ""

    if fileType == 'ELF' and prelink:
        #let's use prelink for the md5sum
        #TODO what if isPrelink fails
        (temp, returncode) = utils.getOutputAsList([prelink, '-y', '--md5', fileName])
        if returncode == 0:
            return temp[0].split()[0]
        else:
            #undoing prelinking failed for some reasons
            pass
    try:
        # ok let's do standard md5sum
        fd=open(fileName)
        md=md5()
        md.update(fd.read())
        fd.close()
        return md.hexdigest()
    except IOError:
        #file not found
        return None
示例#11
0
    def _get_ldconf_paths(self):
        """return a the list of path used by the dynamic loader

        this list is gathered from what you have in /etc/ld.so.conf"""
        return_paths = []
        ldconf_cmd = "ldconfig"
        if not utils.which(ldconf_cmd):
            ldconf_cmd = "/sbin/ldconfig"
        if not utils.which(ldconf_cmd):
            logger.error(
                "Unable to find ldconfig. You need ldconfig in you PATH to properly run fingerprint."
            )
        (output, retcode) = utils.getOutputAsList([ldconf_cmd, "-v"])
        #default_paths = ["/lib", "/usr/lib"]

        # if run as a user it will fail so we don't care for retcode
        for line in output:
            if line and line[0] == '/':
                return_paths.append(line.split(":")[0])
        return return_paths
示例#12
0
    def make_roll(self, fingerprint_base_path, use_remapping = False):
        """
        It creates a roll from a swirl archive.


        :type fingerprint_base_path: string
        :param fingerprint_base_path: a string pointing to the base path of the
                                      fingerprint source code. Used to find the
                                      remapper source code
        :type use_remapping: bool
        :param use_remapping: if True it will use the remapper technology when 
                              creating the roll

        :rtype: bool
        :return: it returns false in case of failure
        """
        if not os.path.exists(self.archive_filename) :
            logger.error("The file " + self.archive_filename + " does not exist" +
                " (specify a different one with -f option)")
            return False
        # this is the list of package we will have to hadd
        self.packages = set()
        self.skipped_swfs = set()
        # this is a list of swirlFile which will need to be installed
        # the additional self.files[0].source_path attribute has been added
        self.files = []
        # keeps track of already processed package in self._resolve_file()
        # so we do not process a package twice
        self.processed_package = []
        # internal swirl package we want to include in the final rpm
        self.wanted_pcks = set()
        # list of rpm pakcage we have to exclude
        self.disable_pcks = set()

        #
        #    ----------------      read the content of the archive
        #
        temp_workdir = tempfile.mkdtemp()
        logger.info("Extracting archive in %s..." % temp_workdir)
        tar_tmp_dir = os.path.join(temp_workdir, def_base_dir)
        archive_file = tarfile.open(self.archive_filename, 'r:gz')
        archive_file.extractall(temp_workdir)
        archive_file.close()
        # open swirl
        logger.info("Reading swirl %s" % os.path.join(temp_workdir, def_swirl_path))
        self.swirl = sergeant.readFromPickle(os.path.join(temp_workdir,
                        def_swirl_path)).swirl

        #
        #    ----------------      recursively resolve all dependencies of the execedFile
        #
        for swf in self.swirl.execedFiles:
            self._resolve_file(swf, use_remapping)
        logger.debug("Dependency resolution terminated. Skipped swirl Files:\n - " +
                                '\n - '.join([i.path for i in self.skipped_swfs]))
        #
        #    ----------------      make rpms with all the files
        #
        # list of user that should be added
        self.users = set()
        rpm_tmp_dir = tempfile.mkdtemp()
        home_rpm_tmp_dir = tempfile.mkdtemp()
        rpm_list = set()
        remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path
        # laydown the file
        for swf in self.files:
            source_path = os.path.join(tar_tmp_dir, str(swf.md5sum),
                            os.path.basename(swf.path))
            if not os.path.exists(source_path) :
		# if the file is not in the archive do not go on
                logger.debug("File " + source_path + " is not present in the archive")
                continue
            # if use_remapping = true swf must be executable 
            # if use_remapping = false just follow the first swf.path.startswith("/home/")
            if swf.path.startswith("/home/"):
                # files in /home need special treatment 1. we need to create a user
                # 2 they need to go in /export/home only on the Frontend
                rpm_list.add((home_rpm_tmp_dir,self.roll_name + "-home"))
                tmp_user = swf.path.split("/home/",1)[1]
                self.users.add(tmp_user.split("/",1)[0])
                rpm_prefix_dir = home_rpm_tmp_dir + "/export"
            else:
                rpm_list.add((rpm_tmp_dir,self.roll_name))
                rpm_prefix_dir = rpm_tmp_dir
            dest_path = rpm_prefix_dir + swf.path
            if not os.path.exists( os.path.dirname(dest_path) ):
                os.makedirs( os.path.dirname(dest_path) )
            if getattr(swf, 'executable', False):
                # we need a wrapper script to set the environment
                shutil.copy2(source_path, dest_path + ".orig")
                f=open(dest_path, 'w')
                f.write("#!/bin/bash\n\n")
                ldconf_written = False
                env = None
                if 'ELF' not in swf.type:
                    # this is not a ELF but is a script so we need to get the
                    # env from its parent swirl file (the interpreter)
                    for execswf in self.swirl.execedFiles:
                        if swf in execswf.openedFiles[execswf.path]:
                            env = execswf.env
                            break
                else:
                    env = swf.env
                if env == None:
                    logger.error('Unable to find interpreter for ', swf)
                    logger.error('Failing on ', swf)
                    return False
                for env_variable in env:
                    if '=' not in env_variable:
                        continue
                    if env_variable.startswith('HYDI'):
                        # MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate
                        # subprocesses the control socket
                        continue
                    variable_name = env_variable.split('=')[0]
                    variable_value = env_variable.split('=')[1]
                    if any([ env_variable.startswith(i) for i in self._append_variables]):
                        # for these variables we want to add their content to
                        # the corresponding system variable values
                        if self.swirl.ldconf_paths and env_variable.startswith('LD_LIBRARY_PATH'):
                            variable_value = variable_value + ':' + ':'.join(self.swirl.ldconf_paths)
                            ldconf_written = True
                        f.write("export " + variable_name + "=\"" + variable_value + ":$" +
                            variable_name + "\"\n")
                    else:
                        # for all the other variables we simply want to define them
                        # if they are not already defined them
                        f.write("if [ -z \"$" + variable_name + "\" ]; then export " +
                            variable_name + "=\"" + variable_value + "\"; fi\n")
                if not ldconf_written and self.swirl.ldconf_paths:
                    f.write("export LD_LIBRARY_PATH=\"" +
                            ':'.join( self.swirl.ldconf_paths ) + ':$LD_LIBRARY_PATH\"\n')
                f.write("\n")
                if use_remapping and 'ELF' in swf.type:
                    f.write(self._remapper_executable + " ")
                    loader = self.swirl.getLoader(swf)
                    if loader:
                        f.write(self._remapper_base_path +\
					loader.path + " ")
                f.write(swf.path + ".orig $@\n")
                f.close()
                os.chmod(dest_path, 0755)
            else:
                if use_remapping:
                    tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path)
                    if not os.path.exists(tmp_path):
                        os.makedirs(tmp_path)
                    shutil.copy2(source_path, tmp_path + '/' + os.path.basename(swf.path))
                else:
                    shutil.copy2(source_path, dest_path)
            # if use remapping we don't need the symlinks
            if use_remapping and not getattr(swf, 'executable', False):
                continue
            # and the symlinks
            for i in swf.links:
                dest_link = rpm_prefix_dir + i
                # source link must be without the rpm_tmp_dir part
                if not os.path.isdir(os.path.dirname(dest_link)):
                    os.makedirs(os.path.dirname(dest_link))
                os.symlink( swf.path, dest_link)
        #
        #    ----------------      create file mapping and include remapper in the RPM
        #
        if use_remapping :
            if not os.path.exists(rpm_tmp_dir + "/etc"):
                os.mkdir(rpm_tmp_dir + "/etc")
            make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping",
                    self._remapper_base_path)
            build_remapper_path = fingerprint_base_path + '/remapper'
            (output, retcode) = utils.getOutputAsList( ["make", "-C",
                    build_remapper_path] )
            if retcode :
                logger.error("Unable to built remapper")
                logger.error("You need to install make and gcc")
                logger.error(" > " + "\n > ".join(output))
                return False
            logger.debug(' > '+ '\n > '.join(output))
            remapper_basedir = rpm_tmp_dir + os.path.dirname(self._remapper_executable)
            if not os.path.exists(remapper_basedir):
                os.makedirs(remapper_basedir)
            shutil.copy2(build_remapper_path + "/remapper", remapper_basedir)
            #let's notify we have to build the base RPM
            rpm_list.add((rpm_tmp_dir,self.roll_name))
        #
        #    ----------------      files are in place so let's make the RPMs
        #
        for (base_dir, rpm_name) in rpm_list:
            if self._make_rpm(base_dir, rpm_name):
                if '-home-' not in rpm_name:
                    self.packages.add(rpm_name)
            else:
                return False
        shutil.rmtree(temp_workdir)
        #
        #    ----------------      create roll copy files there and build
        #
        logger.info("Creating roll " + self.roll_name)
        (output, retcode) = utils.getOutputAsList( ["rocks", "create",
                "new", "roll", self.roll_name] )
        if retcode :
            logger.error("Unable to create the roll")
            if os.path.exists(self.roll_name):
                logger.error("Remove the direcotry: rm -rf %s" % (self.roll_name))
            logger.error(" > " + "\n > ".join(output))
            return False
        shutil.rmtree(self.roll_name + "/src/" + self.roll_name)
        shutil.rmtree(self.roll_name + "/src/usersguide")
        os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml")
        dest = self.roll_name + "/RPMS/" + platform.machine()
        os.makedirs(dest)
        # copying global RPM
        source = glob.glob(self.roll_name + "-1.0-*.rpm")
        if len(source) == 1:
            logger.info("Coping RPM in: " + dest + "/" + source[0])
            shutil.copy2(source[0], dest)
            # create the base-nodea.xml
            node_base_xml = self._node_base_xml_top
            #   1. install packages
            for package in self.packages:
                node_base_xml += '<package>' + package + '</package>\n'
            #   2. remove pakcages
            for package in self.disable_pcks:
                node_base_xml += '<package>-' + package + '</package>\n'
            #   3. set the paths
            new_paths = set()
            for swf in self.swirl.execedFiles:
                new_paths |= set([os.path.dirname(i) for i in swf.getPaths()])
            node_base_xml += self._node_base_xml_bottom % (self.roll_name, ' '.join(new_paths))
            self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-base.xml",
                    node_base_xml)
        # copying -home- RPM
        source = glob.glob(self.roll_name + "-home-1.0-*.rpm")
        if len(source) == 1:
            logger.info("Coping RPM in: " + dest + "/" + source[0])
            shutil.copy2(source[0], dest)
            # create the server-node
            self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-server.xml",
                    self._node_server_xml % (self.roll_name, ' '.join(self.users)))
        # create the graph xml
        self._write_file(self.roll_name + "/graphs/default/" + self.roll_name + ".xml",
                self._graph_node_xml % (self.roll_name, self.roll_name, self.roll_name))
        # make the roll
        os.chdir(self.roll_name)
        (output, retcode) = utils.getOutputAsList(["make", "roll"])
        os.chdir("..")
        roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso")
        if retcode or len(roll_path) < 1:
            # error :-(
            logger.error("Unable to make the roll")
            logger.error(' > ' + '\n > '.join(output))
            return False
        logger.error("Roll %s succesfully created.\nTo add it to your distribution:" % roll_path[0])
        logger.error("rocks add roll " + roll_path[0])
        logger.error("rocks enable roll " + self.roll_name)
        logger.error("cd /export/rocks/install")
        logger.error("rocks create distro")
        logger.error("rocks run roll " + self.roll_name + " | bash")
        return True
示例#13
0
    def make_roll(self, fingerprint_base_path, use_remapping=False):
        """
        It creates a roll from a swirl archive.


        :type fingerprint_base_path: string
        :param fingerprint_base_path: a string pointing to the base path of the
                                      fingerprint source code. Used to find the
                                      remapper source code
        :type use_remapping: bool
        :param use_remapping: if True it will use the remapper technology when 
                              creating the roll

        :rtype: bool
        :return: it returns false in case of failure
        """
        if not os.path.exists(self.archive_filename):
            logger.error("The file " + self.archive_filename +
                         " does not exist" +
                         " (specify a different one with -f option)")
            return False
        # this is the list of package we will have to hadd
        self.packages = set()
        self.skipped_swfs = set()
        # this is a list of swirlFile which will need to be installed
        # the additional self.files[0].source_path attribute has been added
        self.files = []
        # keeps track of already processed package in self._resolve_file()
        # so we do not process a package twice
        self.processed_package = []
        # internal swirl package we want to include in the final rpm
        self.wanted_pcks = set()
        # list of rpm pakcage we have to exclude
        self.disable_pcks = set()

        #
        #    ----------------      read the content of the archive
        #
        temp_workdir = tempfile.mkdtemp()
        logger.info("Extracting archive in %s..." % temp_workdir)
        tar_tmp_dir = os.path.join(temp_workdir, def_base_dir)
        archive_file = tarfile.open(self.archive_filename, 'r:gz')
        archive_file.extractall(temp_workdir)
        archive_file.close()
        # open swirl
        logger.info("Reading swirl %s" %
                    os.path.join(temp_workdir, def_swirl_path))
        self.swirl = sergeant.readFromPickle(
            os.path.join(temp_workdir, def_swirl_path)).swirl

        #
        #    ----------------      recursively resolve all dependencies of the execedFile
        #
        for swf in self.swirl.execedFiles:
            self._resolve_file(swf, use_remapping)
        logger.debug(
            "Dependency resolution terminated. Skipped swirl Files:\n - " +
            '\n - '.join([i.path for i in self.skipped_swfs]))
        #
        #    ----------------      make rpms with all the files
        #
        # list of user that should be added
        self.users = set()
        rpm_tmp_dir = tempfile.mkdtemp()
        home_rpm_tmp_dir = tempfile.mkdtemp()
        rpm_list = set()
        remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path
        # laydown the file
        for swf in self.files:
            source_path = os.path.join(tar_tmp_dir, str(swf.md5sum),
                                       os.path.basename(swf.path))
            if not os.path.exists(source_path):
                # if the file is not in the archive do not go on
                logger.debug("File " + source_path +
                             " is not present in the archive")
                continue
            # if use_remapping = true swf must be executable
            # if use_remapping = false just follow the first swf.path.startswith("/home/")
            if swf.path.startswith("/home/"):
                # files in /home need special treatment 1. we need to create a user
                # 2 they need to go in /export/home only on the Frontend
                rpm_list.add((home_rpm_tmp_dir, self.roll_name + "-home"))
                tmp_user = swf.path.split("/home/", 1)[1]
                self.users.add(tmp_user.split("/", 1)[0])
                rpm_prefix_dir = home_rpm_tmp_dir + "/export"
            else:
                rpm_list.add((rpm_tmp_dir, self.roll_name))
                rpm_prefix_dir = rpm_tmp_dir
            dest_path = rpm_prefix_dir + swf.path
            if not os.path.exists(os.path.dirname(dest_path)):
                os.makedirs(os.path.dirname(dest_path))
            if getattr(swf, 'executable', False):
                # we need a wrapper script to set the environment
                shutil.copy2(source_path, dest_path + ".orig")
                f = open(dest_path, 'w')
                f.write("#!/bin/bash\n\n")
                ldconf_written = False
                env = None
                if 'ELF' not in swf.type:
                    # this is not a ELF but is a script so we need to get the
                    # env from its parent swirl file (the interpreter)
                    for execswf in self.swirl.execedFiles:
                        if swf in execswf.openedFiles[execswf.path]:
                            env = execswf.env
                            break
                else:
                    env = swf.env
                if env == None:
                    logger.error('Unable to find interpreter for ', swf)
                    logger.error('Failing on ', swf)
                    return False
                for env_variable in env:
                    if '=' not in env_variable:
                        continue
                    if env_variable.startswith('HYDI'):
                        # MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate
                        # subprocesses the control socket
                        continue
                    variable_name = env_variable.split('=')[0]
                    variable_value = env_variable.split('=')[1]
                    if any([
                            env_variable.startswith(i)
                            for i in self._append_variables
                    ]):
                        # for these variables we want to add their content to
                        # the corresponding system variable values
                        if self.swirl.ldconf_paths and env_variable.startswith(
                                'LD_LIBRARY_PATH'):
                            variable_value = variable_value + ':' + ':'.join(
                                self.swirl.ldconf_paths)
                            ldconf_written = True
                        f.write("export " + variable_name + "=\"" +
                                variable_value + ":$" + variable_name + "\"\n")
                    else:
                        # for all the other variables we simply want to define them
                        # if they are not already defined them
                        f.write("if [ -z \"$" + variable_name +
                                "\" ]; then export " + variable_name + "=\"" +
                                variable_value + "\"; fi\n")
                if not ldconf_written and self.swirl.ldconf_paths:
                    f.write("export LD_LIBRARY_PATH=\"" +
                            ':'.join(self.swirl.ldconf_paths) +
                            ':$LD_LIBRARY_PATH\"\n')
                f.write("\n")
                if use_remapping and 'ELF' in swf.type:
                    f.write(self._remapper_executable + " ")
                    loader = self.swirl.getLoader(swf)
                    if loader:
                        f.write(self._remapper_base_path +\
     loader.path + " ")
                f.write(swf.path + ".orig $@\n")
                f.close()
                os.chmod(dest_path, 0755)
            else:
                if use_remapping:
                    tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path)
                    if not os.path.exists(tmp_path):
                        os.makedirs(tmp_path)
                    shutil.copy2(source_path,
                                 tmp_path + '/' + os.path.basename(swf.path))
                else:
                    shutil.copy2(source_path, dest_path)
            # if use remapping we don't need the symlinks
            if use_remapping and not getattr(swf, 'executable', False):
                continue
            # and the symlinks
            for i in swf.links:
                dest_link = rpm_prefix_dir + i
                # source link must be without the rpm_tmp_dir part
                if not os.path.isdir(os.path.dirname(dest_link)):
                    os.makedirs(os.path.dirname(dest_link))
                os.symlink(swf.path, dest_link)
        #
        #    ----------------      create file mapping and include remapper in the RPM
        #
        if use_remapping:
            if not os.path.exists(rpm_tmp_dir + "/etc"):
                os.mkdir(rpm_tmp_dir + "/etc")
            make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping",
                              self._remapper_base_path)
            build_remapper_path = fingerprint_base_path + '/remapper'
            (output, retcode) = utils.getOutputAsList(
                ["make", "-C", build_remapper_path])
            if retcode:
                logger.error("Unable to built remapper")
                logger.error("You need to install make and gcc")
                logger.error(" > " + "\n > ".join(output))
                return False
            logger.debug(' > ' + '\n > '.join(output))
            remapper_basedir = rpm_tmp_dir + os.path.dirname(
                self._remapper_executable)
            if not os.path.exists(remapper_basedir):
                os.makedirs(remapper_basedir)
            shutil.copy2(build_remapper_path + "/remapper", remapper_basedir)
            #let's notify we have to build the base RPM
            rpm_list.add((rpm_tmp_dir, self.roll_name))
        #
        #    ----------------      files are in place so let's make the RPMs
        #
        for (base_dir, rpm_name) in rpm_list:
            if self._make_rpm(base_dir, rpm_name):
                if '-home-' not in rpm_name:
                    self.packages.add(rpm_name)
            else:
                return False
        shutil.rmtree(temp_workdir)
        #
        #    ----------------      create roll copy files there and build
        #
        logger.info("Creating roll " + self.roll_name)
        (output, retcode) = utils.getOutputAsList(
            ["rocks", "create", "new", "roll", self.roll_name])
        if retcode:
            logger.error("Unable to create the roll")
            if os.path.exists(self.roll_name):
                logger.error("Remove the direcotry: rm -rf %s" %
                             (self.roll_name))
            logger.error(" > " + "\n > ".join(output))
            return False
        shutil.rmtree(self.roll_name + "/src/" + self.roll_name)
        shutil.rmtree(self.roll_name + "/src/usersguide")
        os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml")
        dest = self.roll_name + "/RPMS/" + platform.machine()
        os.makedirs(dest)
        # copying global RPM
        source = glob.glob(self.roll_name + "-1.0-*.rpm")
        if len(source) == 1:
            logger.info("Coping RPM in: " + dest + "/" + source[0])
            shutil.copy2(source[0], dest)
            # create the base-nodea.xml
            node_base_xml = self._node_base_xml_top
            #   1. install packages
            for package in self.packages:
                node_base_xml += '<package>' + package + '</package>\n'
            #   2. remove pakcages
            for package in self.disable_pcks:
                node_base_xml += '<package>-' + package + '</package>\n'
            #   3. set the paths
            new_paths = set()
            for swf in self.swirl.execedFiles:
                new_paths |= set([os.path.dirname(i) for i in swf.getPaths()])
            node_base_xml += self._node_base_xml_bottom % (self.roll_name,
                                                           ' '.join(new_paths))
            self._write_file(
                self.roll_name + "/nodes/" + self.roll_name + "-base.xml",
                node_base_xml)
        # copying -home- RPM
        source = glob.glob(self.roll_name + "-home-1.0-*.rpm")
        if len(source) == 1:
            logger.info("Coping RPM in: " + dest + "/" + source[0])
            shutil.copy2(source[0], dest)
            # create the server-node
            self._write_file(
                self.roll_name + "/nodes/" + self.roll_name + "-server.xml",
                self._node_server_xml % (self.roll_name, ' '.join(self.users)))
        # create the graph xml
        self._write_file(
            self.roll_name + "/graphs/default/" + self.roll_name + ".xml",
            self._graph_node_xml %
            (self.roll_name, self.roll_name, self.roll_name))
        # make the roll
        os.chdir(self.roll_name)
        (output, retcode) = utils.getOutputAsList(["make", "roll"])
        os.chdir("..")
        roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso")
        if retcode or len(roll_path) < 1:
            # error :-(
            logger.error("Unable to make the roll")
            logger.error(' > ' + '\n > '.join(output))
            return False
        logger.error(
            "Roll %s succesfully created.\nTo add it to your distribution:" %
            roll_path[0])
        logger.error("rocks add roll " + roll_path[0])
        logger.error("rocks enable roll " + self.roll_name)
        logger.error("cd /export/rocks/install")
        logger.error("rocks create distro")
        logger.error("rocks run roll " + self.roll_name + " | bash")
        return True