def _detectedPackageManager(self): """ set the proper _getPackage*(self, path) function to handle rpm or dpkg based on /etc/issue content""" #rpm based OSes rpmOSs = ["red hat", "fedora", "suse", "centos", "scientific linux"] #dpkg based OSes dpkgOSs = ["debian", "ubuntu"] f=open('/etc/issue') issues=f.read() f.close() if any(os in issues.lower() for os in rpmOSs): self._getPackage = self._getPackageRpm if any(os in issues.lower() for os in dpkgOSs): self._getPackage = self._getPackageDpkg if not '_getPackage' in dir(self): #we could not detect the pakcage manager self._getPackage = lambda p : None
def is_special_file(path): """ :type path: string :param path: a path to a file :rtype: bool :return: it returns true if the path points to a file which contains personal data """ return any([ path.endswith(i) for i in specialFile ])
def is_special_file(path): """ :type path: string :param path: a path to a file :rtype: bool :return: it returns true if the path points to a file which contains personal data """ return any([path.endswith(i) for i in specialFile])
def _detectedPackageManager(self): """ set the proper _getPackage*(self, path) function to handle rpm or dpkg based on /etc/issue content""" #rpm based OSes rpmOSs = ["red hat", "fedora", "suse", "centos", "scientific linux"] #dpkg based OSes dpkgOSs = ["debian", "ubuntu"] if os.path.exists('/etc/issue.net'): f = open('/etc/issue.net') issues = f.read() f.close() if any(os in issues.lower() for os in rpmOSs): self._getPackage = self._getPackageRpm return elif any(os in issues.lower() for os in dpkgOSs): self._getPackage = self._getPackageDpkg return if not '_getPackage' in dir(self): #we could not detect the pakcage manager self._getPackage = lambda p: None
def is_special_folder(path): """ return true if path is to be considered special, which means it should be skipped from archivingi, checksumming, etc. :type path: string :param path: an absolute path to the file :rtype: bool :return: True if the given path is special """ return any([ path.startswith(i) for i in specialFolders ])
def is_special_folder(path): """ return true if path is to be considered special, which means it should be skipped from archivingi, checksumming, etc. :type path: string :param path: an absolute path to the file :rtype: bool :return: True if the given path is special """ return any([path.startswith(i) for i in specialFolders])
def _make_rpm(self, base_path, rpm_name): """ makes an rpm called rpm_name starting from base_path return False if something went wrong """ # rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/ logger.info("RPM " + rpm_name + " root dir " + base_path) (output, retcode) = utils.getOutputAsList( ["rocks", "create", "package", base_path + "/*", rpm_name, "prefix=/"]) if any([i for i in output if 'RPM build errors' in i ]): logger.error(' > ' + '\n > '.join(output)) logger.error("Error building " + rpm_name + " RPM package\n") return False logger.debug(' > '+ '\n > '.join(output)) shutil.rmtree(base_path) return True
def _make_rpm(self, base_path, rpm_name): """ makes an rpm called rpm_name starting from base_path return False if something went wrong """ # rocks create package "/tmp/tmpAFDASDF/*" pakcagename prefix=/ logger.info("RPM " + rpm_name + " root dir " + base_path) (output, retcode) = utils.getOutputAsList([ "rocks", "create", "package", base_path + "/*", rpm_name, "prefix=/" ]) if any([i for i in output if 'RPM build errors' in i]): logger.error(' > ' + '\n > '.join(output)) logger.error("Error building " + rpm_name + " RPM package\n") return False logger.debug(' > ' + '\n > '.join(output)) shutil.rmtree(base_path) return True
def make_roll(self, fingerprint_base_path, use_remapping = False): """ It creates a roll from a swirl archive. :type fingerprint_base_path: string :param fingerprint_base_path: a string pointing to the base path of the fingerprint source code. Used to find the remapper source code :type use_remapping: bool :param use_remapping: if True it will use the remapper technology when creating the roll :rtype: bool :return: it returns false in case of failure """ if not os.path.exists(self.archive_filename) : logger.error("The file " + self.archive_filename + " does not exist" + " (specify a different one with -f option)") return False # this is the list of package we will have to hadd self.packages = set() self.skipped_swfs = set() # this is a list of swirlFile which will need to be installed # the additional self.files[0].source_path attribute has been added self.files = [] # keeps track of already processed package in self._resolve_file() # so we do not process a package twice self.processed_package = [] # internal swirl package we want to include in the final rpm self.wanted_pcks = set() # list of rpm pakcage we have to exclude self.disable_pcks = set() # # ---------------- read the content of the archive # temp_workdir = tempfile.mkdtemp() logger.info("Extracting archive in %s..." % temp_workdir) tar_tmp_dir = os.path.join(temp_workdir, def_base_dir) archive_file = tarfile.open(self.archive_filename, 'r:gz') archive_file.extractall(temp_workdir) archive_file.close() # open swirl logger.info("Reading swirl %s" % os.path.join(temp_workdir, def_swirl_path)) self.swirl = sergeant.readFromPickle(os.path.join(temp_workdir, def_swirl_path)).swirl # # ---------------- recursively resolve all dependencies of the execedFile # for swf in self.swirl.execedFiles: self._resolve_file(swf, use_remapping) logger.debug("Dependency resolution terminated. Skipped swirl Files:\n - " + '\n - '.join([i.path for i in self.skipped_swfs])) # # ---------------- make rpms with all the files # # list of user that should be added self.users = set() rpm_tmp_dir = tempfile.mkdtemp() home_rpm_tmp_dir = tempfile.mkdtemp() rpm_list = set() remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path # laydown the file for swf in self.files: source_path = os.path.join(tar_tmp_dir, str(swf.md5sum), os.path.basename(swf.path)) if not os.path.exists(source_path) : # if the file is not in the archive do not go on logger.debug("File " + source_path + " is not present in the archive") continue # if use_remapping = true swf must be executable # if use_remapping = false just follow the first swf.path.startswith("/home/") if swf.path.startswith("/home/"): # files in /home need special treatment 1. we need to create a user # 2 they need to go in /export/home only on the Frontend rpm_list.add((home_rpm_tmp_dir,self.roll_name + "-home")) tmp_user = swf.path.split("/home/",1)[1] self.users.add(tmp_user.split("/",1)[0]) rpm_prefix_dir = home_rpm_tmp_dir + "/export" else: rpm_list.add((rpm_tmp_dir,self.roll_name)) rpm_prefix_dir = rpm_tmp_dir dest_path = rpm_prefix_dir + swf.path if not os.path.exists( os.path.dirname(dest_path) ): os.makedirs( os.path.dirname(dest_path) ) if getattr(swf, 'executable', False): # we need a wrapper script to set the environment shutil.copy2(source_path, dest_path + ".orig") f=open(dest_path, 'w') f.write("#!/bin/bash\n\n") ldconf_written = False env = None if 'ELF' not in swf.type: # this is not a ELF but is a script so we need to get the # env from its parent swirl file (the interpreter) for execswf in self.swirl.execedFiles: if swf in execswf.openedFiles[execswf.path]: env = execswf.env break else: env = swf.env if env == None: logger.error('Unable to find interpreter for ', swf) logger.error('Failing on ', swf) return False for env_variable in env: if '=' not in env_variable: continue if env_variable.startswith('HYDI'): # MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate # subprocesses the control socket continue variable_name = env_variable.split('=')[0] variable_value = env_variable.split('=')[1] if any([ env_variable.startswith(i) for i in self._append_variables]): # for these variables we want to add their content to # the corresponding system variable values if self.swirl.ldconf_paths and env_variable.startswith('LD_LIBRARY_PATH'): variable_value = variable_value + ':' + ':'.join(self.swirl.ldconf_paths) ldconf_written = True f.write("export " + variable_name + "=\"" + variable_value + ":$" + variable_name + "\"\n") else: # for all the other variables we simply want to define them # if they are not already defined them f.write("if [ -z \"$" + variable_name + "\" ]; then export " + variable_name + "=\"" + variable_value + "\"; fi\n") if not ldconf_written and self.swirl.ldconf_paths: f.write("export LD_LIBRARY_PATH=\"" + ':'.join( self.swirl.ldconf_paths ) + ':$LD_LIBRARY_PATH\"\n') f.write("\n") if use_remapping and 'ELF' in swf.type: f.write(self._remapper_executable + " ") loader = self.swirl.getLoader(swf) if loader: f.write(self._remapper_base_path +\ loader.path + " ") f.write(swf.path + ".orig $@\n") f.close() os.chmod(dest_path, 0755) else: if use_remapping: tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path) if not os.path.exists(tmp_path): os.makedirs(tmp_path) shutil.copy2(source_path, tmp_path + '/' + os.path.basename(swf.path)) else: shutil.copy2(source_path, dest_path) # if use remapping we don't need the symlinks if use_remapping and not getattr(swf, 'executable', False): continue # and the symlinks for i in swf.links: dest_link = rpm_prefix_dir + i # source link must be without the rpm_tmp_dir part if not os.path.isdir(os.path.dirname(dest_link)): os.makedirs(os.path.dirname(dest_link)) os.symlink( swf.path, dest_link) # # ---------------- create file mapping and include remapper in the RPM # if use_remapping : if not os.path.exists(rpm_tmp_dir + "/etc"): os.mkdir(rpm_tmp_dir + "/etc") make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping", self._remapper_base_path) build_remapper_path = fingerprint_base_path + '/remapper' (output, retcode) = utils.getOutputAsList( ["make", "-C", build_remapper_path] ) if retcode : logger.error("Unable to built remapper") logger.error("You need to install make and gcc") logger.error(" > " + "\n > ".join(output)) return False logger.debug(' > '+ '\n > '.join(output)) remapper_basedir = rpm_tmp_dir + os.path.dirname(self._remapper_executable) if not os.path.exists(remapper_basedir): os.makedirs(remapper_basedir) shutil.copy2(build_remapper_path + "/remapper", remapper_basedir) #let's notify we have to build the base RPM rpm_list.add((rpm_tmp_dir,self.roll_name)) # # ---------------- files are in place so let's make the RPMs # for (base_dir, rpm_name) in rpm_list: if self._make_rpm(base_dir, rpm_name): if '-home-' not in rpm_name: self.packages.add(rpm_name) else: return False shutil.rmtree(temp_workdir) # # ---------------- create roll copy files there and build # logger.info("Creating roll " + self.roll_name) (output, retcode) = utils.getOutputAsList( ["rocks", "create", "new", "roll", self.roll_name] ) if retcode : logger.error("Unable to create the roll") if os.path.exists(self.roll_name): logger.error("Remove the direcotry: rm -rf %s" % (self.roll_name)) logger.error(" > " + "\n > ".join(output)) return False shutil.rmtree(self.roll_name + "/src/" + self.roll_name) shutil.rmtree(self.roll_name + "/src/usersguide") os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml") dest = self.roll_name + "/RPMS/" + platform.machine() os.makedirs(dest) # copying global RPM source = glob.glob(self.roll_name + "-1.0-*.rpm") if len(source) == 1: logger.info("Coping RPM in: " + dest + "/" + source[0]) shutil.copy2(source[0], dest) # create the base-nodea.xml node_base_xml = self._node_base_xml_top # 1. install packages for package in self.packages: node_base_xml += '<package>' + package + '</package>\n' # 2. remove pakcages for package in self.disable_pcks: node_base_xml += '<package>-' + package + '</package>\n' # 3. set the paths new_paths = set() for swf in self.swirl.execedFiles: new_paths |= set([os.path.dirname(i) for i in swf.getPaths()]) node_base_xml += self._node_base_xml_bottom % (self.roll_name, ' '.join(new_paths)) self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-base.xml", node_base_xml) # copying -home- RPM source = glob.glob(self.roll_name + "-home-1.0-*.rpm") if len(source) == 1: logger.info("Coping RPM in: " + dest + "/" + source[0]) shutil.copy2(source[0], dest) # create the server-node self._write_file(self.roll_name + "/nodes/" + self.roll_name + "-server.xml", self._node_server_xml % (self.roll_name, ' '.join(self.users))) # create the graph xml self._write_file(self.roll_name + "/graphs/default/" + self.roll_name + ".xml", self._graph_node_xml % (self.roll_name, self.roll_name, self.roll_name)) # make the roll os.chdir(self.roll_name) (output, retcode) = utils.getOutputAsList(["make", "roll"]) os.chdir("..") roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso") if retcode or len(roll_path) < 1: # error :-( logger.error("Unable to make the roll") logger.error(' > ' + '\n > '.join(output)) return False logger.error("Roll %s succesfully created.\nTo add it to your distribution:" % roll_path[0]) logger.error("rocks add roll " + roll_path[0]) logger.error("rocks enable roll " + self.roll_name) logger.error("cd /export/rocks/install") logger.error("rocks create distro") logger.error("rocks run roll " + self.roll_name + " | bash") return True
def make_roll(self, fingerprint_base_path, use_remapping=False): """ It creates a roll from a swirl archive. :type fingerprint_base_path: string :param fingerprint_base_path: a string pointing to the base path of the fingerprint source code. Used to find the remapper source code :type use_remapping: bool :param use_remapping: if True it will use the remapper technology when creating the roll :rtype: bool :return: it returns false in case of failure """ if not os.path.exists(self.archive_filename): logger.error("The file " + self.archive_filename + " does not exist" + " (specify a different one with -f option)") return False # this is the list of package we will have to hadd self.packages = set() self.skipped_swfs = set() # this is a list of swirlFile which will need to be installed # the additional self.files[0].source_path attribute has been added self.files = [] # keeps track of already processed package in self._resolve_file() # so we do not process a package twice self.processed_package = [] # internal swirl package we want to include in the final rpm self.wanted_pcks = set() # list of rpm pakcage we have to exclude self.disable_pcks = set() # # ---------------- read the content of the archive # temp_workdir = tempfile.mkdtemp() logger.info("Extracting archive in %s..." % temp_workdir) tar_tmp_dir = os.path.join(temp_workdir, def_base_dir) archive_file = tarfile.open(self.archive_filename, 'r:gz') archive_file.extractall(temp_workdir) archive_file.close() # open swirl logger.info("Reading swirl %s" % os.path.join(temp_workdir, def_swirl_path)) self.swirl = sergeant.readFromPickle( os.path.join(temp_workdir, def_swirl_path)).swirl # # ---------------- recursively resolve all dependencies of the execedFile # for swf in self.swirl.execedFiles: self._resolve_file(swf, use_remapping) logger.debug( "Dependency resolution terminated. Skipped swirl Files:\n - " + '\n - '.join([i.path for i in self.skipped_swfs])) # # ---------------- make rpms with all the files # # list of user that should be added self.users = set() rpm_tmp_dir = tempfile.mkdtemp() home_rpm_tmp_dir = tempfile.mkdtemp() rpm_list = set() remapper_rpm_tmp_dir = rpm_tmp_dir + self._remapper_base_path # laydown the file for swf in self.files: source_path = os.path.join(tar_tmp_dir, str(swf.md5sum), os.path.basename(swf.path)) if not os.path.exists(source_path): # if the file is not in the archive do not go on logger.debug("File " + source_path + " is not present in the archive") continue # if use_remapping = true swf must be executable # if use_remapping = false just follow the first swf.path.startswith("/home/") if swf.path.startswith("/home/"): # files in /home need special treatment 1. we need to create a user # 2 they need to go in /export/home only on the Frontend rpm_list.add((home_rpm_tmp_dir, self.roll_name + "-home")) tmp_user = swf.path.split("/home/", 1)[1] self.users.add(tmp_user.split("/", 1)[0]) rpm_prefix_dir = home_rpm_tmp_dir + "/export" else: rpm_list.add((rpm_tmp_dir, self.roll_name)) rpm_prefix_dir = rpm_tmp_dir dest_path = rpm_prefix_dir + swf.path if not os.path.exists(os.path.dirname(dest_path)): os.makedirs(os.path.dirname(dest_path)) if getattr(swf, 'executable', False): # we need a wrapper script to set the environment shutil.copy2(source_path, dest_path + ".orig") f = open(dest_path, 'w') f.write("#!/bin/bash\n\n") ldconf_written = False env = None if 'ELF' not in swf.type: # this is not a ELF but is a script so we need to get the # env from its parent swirl file (the interpreter) for execswf in self.swirl.execedFiles: if swf in execswf.openedFiles[execswf.path]: env = execswf.env break else: env = swf.env if env == None: logger.error('Unable to find interpreter for ', swf) logger.error('Failing on ', swf) return False for env_variable in env: if '=' not in env_variable: continue if env_variable.startswith('HYDI'): # MVAPICH 2.x HYDI_CONTROL_FD is used be hydra_proxy_mpi to comunicate # subprocesses the control socket continue variable_name = env_variable.split('=')[0] variable_value = env_variable.split('=')[1] if any([ env_variable.startswith(i) for i in self._append_variables ]): # for these variables we want to add their content to # the corresponding system variable values if self.swirl.ldconf_paths and env_variable.startswith( 'LD_LIBRARY_PATH'): variable_value = variable_value + ':' + ':'.join( self.swirl.ldconf_paths) ldconf_written = True f.write("export " + variable_name + "=\"" + variable_value + ":$" + variable_name + "\"\n") else: # for all the other variables we simply want to define them # if they are not already defined them f.write("if [ -z \"$" + variable_name + "\" ]; then export " + variable_name + "=\"" + variable_value + "\"; fi\n") if not ldconf_written and self.swirl.ldconf_paths: f.write("export LD_LIBRARY_PATH=\"" + ':'.join(self.swirl.ldconf_paths) + ':$LD_LIBRARY_PATH\"\n') f.write("\n") if use_remapping and 'ELF' in swf.type: f.write(self._remapper_executable + " ") loader = self.swirl.getLoader(swf) if loader: f.write(self._remapper_base_path +\ loader.path + " ") f.write(swf.path + ".orig $@\n") f.close() os.chmod(dest_path, 0755) else: if use_remapping: tmp_path = remapper_rpm_tmp_dir + os.path.dirname(swf.path) if not os.path.exists(tmp_path): os.makedirs(tmp_path) shutil.copy2(source_path, tmp_path + '/' + os.path.basename(swf.path)) else: shutil.copy2(source_path, dest_path) # if use remapping we don't need the symlinks if use_remapping and not getattr(swf, 'executable', False): continue # and the symlinks for i in swf.links: dest_link = rpm_prefix_dir + i # source link must be without the rpm_tmp_dir part if not os.path.isdir(os.path.dirname(dest_link)): os.makedirs(os.path.dirname(dest_link)) os.symlink(swf.path, dest_link) # # ---------------- create file mapping and include remapper in the RPM # if use_remapping: if not os.path.exists(rpm_tmp_dir + "/etc"): os.mkdir(rpm_tmp_dir + "/etc") make_mapping_file(self.files, rpm_tmp_dir + "/etc/fp_mapping", self._remapper_base_path) build_remapper_path = fingerprint_base_path + '/remapper' (output, retcode) = utils.getOutputAsList( ["make", "-C", build_remapper_path]) if retcode: logger.error("Unable to built remapper") logger.error("You need to install make and gcc") logger.error(" > " + "\n > ".join(output)) return False logger.debug(' > ' + '\n > '.join(output)) remapper_basedir = rpm_tmp_dir + os.path.dirname( self._remapper_executable) if not os.path.exists(remapper_basedir): os.makedirs(remapper_basedir) shutil.copy2(build_remapper_path + "/remapper", remapper_basedir) #let's notify we have to build the base RPM rpm_list.add((rpm_tmp_dir, self.roll_name)) # # ---------------- files are in place so let's make the RPMs # for (base_dir, rpm_name) in rpm_list: if self._make_rpm(base_dir, rpm_name): if '-home-' not in rpm_name: self.packages.add(rpm_name) else: return False shutil.rmtree(temp_workdir) # # ---------------- create roll copy files there and build # logger.info("Creating roll " + self.roll_name) (output, retcode) = utils.getOutputAsList( ["rocks", "create", "new", "roll", self.roll_name]) if retcode: logger.error("Unable to create the roll") if os.path.exists(self.roll_name): logger.error("Remove the direcotry: rm -rf %s" % (self.roll_name)) logger.error(" > " + "\n > ".join(output)) return False shutil.rmtree(self.roll_name + "/src/" + self.roll_name) shutil.rmtree(self.roll_name + "/src/usersguide") os.remove(self.roll_name + "/nodes/" + self.roll_name + ".xml") dest = self.roll_name + "/RPMS/" + platform.machine() os.makedirs(dest) # copying global RPM source = glob.glob(self.roll_name + "-1.0-*.rpm") if len(source) == 1: logger.info("Coping RPM in: " + dest + "/" + source[0]) shutil.copy2(source[0], dest) # create the base-nodea.xml node_base_xml = self._node_base_xml_top # 1. install packages for package in self.packages: node_base_xml += '<package>' + package + '</package>\n' # 2. remove pakcages for package in self.disable_pcks: node_base_xml += '<package>-' + package + '</package>\n' # 3. set the paths new_paths = set() for swf in self.swirl.execedFiles: new_paths |= set([os.path.dirname(i) for i in swf.getPaths()]) node_base_xml += self._node_base_xml_bottom % (self.roll_name, ' '.join(new_paths)) self._write_file( self.roll_name + "/nodes/" + self.roll_name + "-base.xml", node_base_xml) # copying -home- RPM source = glob.glob(self.roll_name + "-home-1.0-*.rpm") if len(source) == 1: logger.info("Coping RPM in: " + dest + "/" + source[0]) shutil.copy2(source[0], dest) # create the server-node self._write_file( self.roll_name + "/nodes/" + self.roll_name + "-server.xml", self._node_server_xml % (self.roll_name, ' '.join(self.users))) # create the graph xml self._write_file( self.roll_name + "/graphs/default/" + self.roll_name + ".xml", self._graph_node_xml % (self.roll_name, self.roll_name, self.roll_name)) # make the roll os.chdir(self.roll_name) (output, retcode) = utils.getOutputAsList(["make", "roll"]) os.chdir("..") roll_path = glob.glob(self.roll_name + "/" + self.roll_name + "*.iso") if retcode or len(roll_path) < 1: # error :-( logger.error("Unable to make the roll") logger.error(' > ' + '\n > '.join(output)) return False logger.error( "Roll %s succesfully created.\nTo add it to your distribution:" % roll_path[0]) logger.error("rocks add roll " + roll_path[0]) logger.error("rocks enable roll " + self.roll_name) logger.error("cd /export/rocks/install") logger.error("rocks create distro") logger.error("rocks run roll " + self.roll_name + " | bash") return True