def package_it(self): print("Packaging ...") width = self.parser.options.use_64_bit and "64" or "32" libgoto_info_str = "%s-%s-%s-%s%s" % ( self.parser.cpu_id, self.parser.options.fcompiler, self.parser.short_version, width, self.parser.options.smp and "p" or "") libgoto_vers_str = "%s-r%s" % (libgoto_info_str, self.parser.options.goto_version) static_library_name, dynamic_library_name = ("libgoto-%s.a" % (libgoto_vers_str), "libgoto-%s.so" % (libgoto_vers_str)) info_name = "README.libgoto-%s" % (libgoto_vers_str) sep_str = "-" * 50 readme_lines = [ sep_str ] + self.parser.get_compile_options().split("\n") + [ "Compile times: %s" % (", ".join([ "%s: %s" % (key, logging_tools.get_diff_time_str(self.time_dict[key])) for key in list(self.time_dict.keys()) ])), sep_str, "" ] if self.parser.options.include_log: readme_lines.extend( ["Compile logs:"] + sum([["%s:" % (key)] + self.log_dict[key].split("\n") + [sep_str] for key in list(self.log_dict.keys())], [])) libgoto_static_file_name, libgoto_dynamic_file_name = ( "/static_not_found", "/dynamic_not_found") codecs.open("{}/info".format(self.tempdir), "w", encoding='utf-8').write("\n".join(readme_lines)) for ent in os.listdir("{}/{}".format(self.tempdir, self.dirname)): fpath = "{}/{}/{}".format(self.tempdir, self.dirname, ent) if ent.endswith(".a") and (os.path.isfile(fpath) and not os.path.islink(fpath)): libgoto_static_file_name = fpath if ent.endswith(".so") and (os.path.isfile(fpath) and not os.path.islink(fpath)): libgoto_dynamic_file_name = fpath fpath = "" package_name, package_version, package_release = ( "libgoto-%s" % (libgoto_info_str), self.parser.options.goto_version, self.parser.options.release) # copy libgoto.a codecs.open("{}.static".format(libgoto_static_file_name), "wb").write(open(libgoto_static_file_name, "rb").read()) dummy_args = argparse.Namespace( name=package_name, version=package_version, release=package_release, package_group="Libraries/Math", description="BLAS", summary="BLAS Package", ) new_p = rpm_build_tools.build_package(dummy_args) if self.parser.options.arch: new_p["arch"] = self.parser.options.arch new_p["inst_options"] = " -p " content = rpm_build_tools.file_content_list([ "%s.static:%s/%s" % (libgoto_static_file_name, self.parser.options.target_dir, static_library_name), "%s:%s/%s" % (libgoto_dynamic_file_name, self.parser.options.target_dir, dynamic_library_name), "%s/info:%s/%s" % (self.tempdir, self.parser.options.target_dir, info_name), "%s:%s/Makefile.rule.%s" % (self.orig_rulefile_name, self.parser.options.target_dir, libgoto_vers_str) ]) new_p.create_tgz_file(content) new_p.write_specfile(content) new_p.build_package() if new_p.build_ok: print("Build successfull, package locations:") print(new_p.long_package_name) print(new_p.src_package_name) success = True else: print("Something went wrong, please check tempdir %s" % (self.tempdir)) success = False return success
def package_it(self): print("Packaging ...") info_name = "README.%s" % (self.parser.package_name) sep_str = "-" * 50 readme_lines = [sep_str] + \ self.parser.get_compile_options().split("\n") + \ ["Compile times: %s" % (", ".join(["%s: %s" % (key, logging_tools.get_diff_time_str(self.time_dict[key]["diff"])) for key in list(self.time_dict.keys())])), sep_str, ""] if self.parser.options.include_log: readme_lines.extend(["Compile logs:"] + \ sum([self.log_dict[key].split("\n") + [sep_str] for key in list(self.log_dict.keys())], [])) open("%s/%s" % (self.tempdir, info_name), "w").write("\n".join(readme_lines)) package_name, package_version, package_release = ( self.parser.package_name, self.parser.options.fftw_version, self.parser.options.release) # create cluster module file cmod = cluster_module_tools.cluster_module( name=package_name, version=package_version, release=package_release, description="FFTW", #arch=self.parse ) cmod.add_env_variable( cluster_module_tools.cluster_module_env(name="PATH", mode="append", value="%s/bin" % (self.parser.fftw_dir))) cmod.add_env_variable( cluster_module_tools.cluster_module_env(name="LD_LIBRARY", mode="append", value="%s/lib" % (self.parser.fftw_dir))) open("%s/cmod" % (self.tempdir), "w").write(cmod.get_xml_representation()) new_p = rpm_build_tools.build_package() if self.parser.options.arch: new_p["arch"] = self.parser.options.arch new_p["name"] = package_name new_p["version"] = package_version new_p["release"] = package_release new_p["package_group"] = "System/Libraries" new_p["inst_options"] = " -p " # remove old info if present if os.path.isfile("%s/%s" % (self.parser.fftw_dir, info_name)): os.unlink("%s/%s" % (self.parser.fftw_dir, info_name)) content = rpm_build_tools.file_content_list([ self.parser.fftw_dir, "%s/%s:%s/%s" % (self.tempdir, info_name, self.parser.fftw_dir, info_name), "*%s/cmod:/opt/cluster/modules/%s" % (self.tempdir, cmod.get_name()) ]) new_p.create_tgz_file(content) new_p.write_specfile(content) new_p.build_package() if new_p.build_ok: print("Build successfull, package locations:") print(new_p.long_package_name) print(new_p.src_package_name) success = True else: print("Something went wrong, please check tempdir %s" % (self.tempdir)) success = False return success
def package_it(self): print("Packaging ...") self.tempdir_2 = tempfile.mkdtemp("_hpl") info_name = "README.%s" % (self.parser.package_name) sep_str = "-" * 50 readme_lines = [sep_str] + \ self.parser.get_compile_options().split("\n") + \ [ "Compile times: {}".format( ", ".join( [ "{}: {}".format( key, logging_tools.get_diff_time_str(self.time_dict[key]["diff"]) ) for key in list(self.time_dict.keys()) ] ) ), sep_str, "" ] if self.parser.options.include_log: readme_lines.extend( [ "Compile logs:" ] + sum( [ self.log_dict[key].split("\n") + [sep_str] for key in list(self.log_dict.keys()) ], [] ) ) open("%s/%s" % (self.tempdir_2, info_name), "w").write("\n".join(readme_lines)) package_name, package_version, package_release = (self.parser.package_name, self.parser.options.hpl_version, "1") hpl_base_dir = os.path.join( self._hpl_dir, "bin", self.cpu_arch ) xhpl_file_name = "%s/xhpl" % (hpl_base_dir) if not os.path.isfile(xhpl_file_name): print("Cannot find %s" % (xhpl_file_name)) success = False else: open("%s/xhpl" % (self.tempdir_2), "wb").write(open(xhpl_file_name, "rb").read()) open("%s/Make.%s" % (self.tempdir_2, self.parser.package_name), "wb").write( open( os.path.join(self._hpl_dir, "Make.{}".format(self.cpu_arch)), "rb" ).read() ) open("%s/HPL.dat" % (self.tempdir_2), "wb").write(open("%s/HPL.dat" % (hpl_base_dir), "rb").read()) os.chmod("%s/xhpl" % (self.tempdir_2), 0o775) dummy_args = argparse.Namespace( name=package_name, version=package_version, release=package_release, package_group="Tools/Benchmark", arch=self.parser.options.arch, description="HPL", summary="HPL", ) new_p = rpm_build_tools.build_package(dummy_args) new_p["inst_options"] = " -p " content = rpm_build_tools.file_content_list(["%s:%s" % (self.tempdir_2, self.parser.hpl_dir)]) new_p.create_tgz_file(content) new_p.write_specfile(content) new_p.build_package() if new_p.build_ok: print("Build successfull, package locations:") print(new_p.long_package_name) print(new_p.src_package_name) success = True else: print("Something went wrong, please check tempdir %s" % (self.tempdir)) success = False return success
def package_it(self): print("Packaging ...") info_name = "README.%s" % (self.parser.package_name) sep_str = "-" * 50 readme_lines = [ sep_str ] + self.parser.get_compile_options().split("\n") + [ "Compile times: {}".format(", ".join([ "%s: %s" % (key, logging_tools.get_diff_time_str(self.time_dict[key]["diff"])) for key in list(self.time_dict.keys()) ])), sep_str, "" ] if self.parser.options.include_log: readme_lines.extend(["Compile logs:"] + sum([ self.log_dict[key].split("\n") + [sep_str] for key in list(self.log_dict.keys()) ], [])) open("%s/%s" % (self.tempdir, info_name), "w").write("\n".join(readme_lines)) package_name, package_version, package_release = ( self.parser.package_name, self.parser.options.mpi_version, self.parser.options.release) if self.parser.options.module_file: self._create_module_file() if self.parser.options.mpi_selector: self._create_mpi_selector_file() dummy_args = argparse.Namespace( name=package_name, version=package_version, release=package_release, package_group="System/Libraries", arch=self.parser.options.arch, description="MPI via {}".format(self.parser.mode), summary="{}".format(self.parser.mode), ) new_p = rpm_build_tools.build_package(dummy_args) new_p["inst_options"] = " -p " # set flag remove_mca = False if self.parser.mode == "openmpi": # remove config files for file_name in os.listdir("%s/etc" % (self.parser.mpi_dir)): try: os.unlink("%s/etc/%s" % (self.parser.mpi_dir, file_name)) except: pass # generate empty hosts file open("%s/etc/openmpi-default-hostfile" % (self.parser.mpi_dir), "w").write("") # generate link mca_local_file = "/etc/openmpi-mca-params.conf" os.symlink( mca_local_file, "%s/etc/openmpi-mca-params.conf" % (self.parser.mpi_dir)) # generate dummy file if not existent if not os.path.exists(mca_local_file): remove_mca = True open(mca_local_file, "w").write("") # remove old info if present if os.path.isfile("%s/%s" % (self.parser.mpi_dir, info_name)): os.unlink("%s/%s" % (self.parser.mpi_dir, info_name)) fc_list = [ self.parser.mpi_dir, "%s/%s:%s/%s" % (self.tempdir, info_name, self.parser.mpi_dir, info_name) ] if self.parser.options.mpi_selector: fc_list.append("{0}{1}:{1}".format( self.tempdir, os.path.join(self.mpi_selector_dir, self.mpi_selector_sh_name))) fc_list.append("{0}{1}:{1}".format( self.tempdir, os.path.join(self.mpi_selector_dir, self.mpi_selector_csh_name))) if self.parser.options.module_file: fc_list.append("{0}{1}:{1}".format( self.tempdir, os.path.join(self.parser.options.module_dir, self.modulefile_name), )) content = rpm_build_tools.file_content_list(fc_list) new_p.create_tgz_file(content) new_p.write_specfile(content) new_p.build_package() if new_p.build_ok: print("Build successfull, package locations:") print(new_p.long_package_name) print(new_p.src_package_name) success = True else: print("Something went wrong, please check tempdir %s" % (self.tempdir)) success = False if remove_mca: os.unlink(mca_local_file) return success
def main(): package_group = "System/Monitoring" dc = None config_dict = { "dist_dir": "", "copy_to": "", "exc_dir_names": [], } if not (config_dict["dist_dir"] or config_dict["copy_to"]): if os.path.isfile("/etc/debian_version"): config_dict["dist_dir"] = "/packages/debian" config_dict["copy_to"] = "/opt/cluster/system/packages/debian" else: config_dict["dist_dir"] = "/packages/RPMs" config_dict["copy_to"] = "/opt/cluster/system/packages/RPMs" long_package_name = None script_dict = {} # build_package entity my_arg = rpm_build_tools.package_parser() opts = my_arg.parse_args() build_p = rpm_build_tools.build_package(opts) if not long_package_name: build_it = True if build_it: act_cl = rpm_build_tools.file_content_list( opts.args) # , exclude_dir_names=config_dict["exc_dir_names"]) if not act_cl: print("Need a file and/or directory-list !") if dc: dc.release() sys.exit(-1) act_cl.show_content() # if release == "AUTO_INCR": # release = check_auto_incr_release(dc, name, version) if dc: sql_str = "SELECT p.pgroup, COUNT(DISTINCT p.package_idx) AS c FROM inst_package ip, package p WHERE ip.package=p.package_idx GROUP BY p.pgroup" dc.execute(sql_str) all_g = [x["pgroup"] for x in dc.fetchall()] if package_group not in all_g: print( "Package Group '%s' is new (already existing: %s); continue ?" % (package_group, ", ".join(all_g) or "<none>")) ret = None while ret not in ["yes", "no"]: ret = getinput(" - [yes/no]") if ret == "no": build_it = False elif ret == "yes": break if build_it: # build_p["version"] = version # build_p["release"] = release # build_p["name"] = name # build_p["package_group"] = package_group # set scripts for key, value in script_dict.items(): build_p[key] = value build_p.create_tgz_file(act_cl) build_p.write_specfile(act_cl) build_p.build_package() if build_p.build_ok: if dc: print("Success, insert info into database...") else: print("Success, local mode") long_package_name = build_p.long_package_name print("Package locations:") print(build_p.long_package_name) print(build_p.src_package_name) else: print("Build went wrong, exiting") build_it = False if not build_it: if dc: dc.release() sys.exit(0) short_package_name = os.path.basename(long_package_name) if dc: p_idx = None stat, out = subprocess.getstatusoutput( "/usr/local/sbin/insert_package_info.py -p %s" % (long_package_name)) if stat: print("Some error occured (%d) : " % (stat)) for line in [x.strip() for x in out.split("\n")]: print(" %s" % (line)) if dc: dc.release() sys.exit(3) else: for line in [x.strip() for x in out.split("\n")]: if line.startswith("pidx"): p_idx = int(line.split()[1]) elif line.startswith("pname"): name = line.split(None, 1)[1] # p_idx now holds the index of the package if p_idx is None: print("Can't get db-index of package info %s" % (long_package_name)) if dc: dc.release() sys.exit(2) if dc: # read name, version and release from db dc.execute( "SELECT p.name, p.version, p.release, p.buildhost, p.packager FROM package p WHERE p.package_idx=%d" % (p_idx)) db_stuff = dc.fetchone() name, version, release = (db_stuff["name"], db_stuff["version"], db_stuff["release"]) set_dict = { "package": p_idx, "location": os.path.normpath( "%s/%s" % (config_dict["dist_dir"], short_package_name)), "last_build": time.time() } write_entry = check_for_association(dc, p_idx, name, version, release, set_dict) if write_entry: sql_str, sql_tuple = ( "INSERT INTO inst_package SET %s" % (", ".join( ["%s=%%s" % (key) for key in list(set_dict.keys())])), tuple([set_dict[key] for key in list(set_dict.keys())])) ok = dc.execute(sql_str, sql_tuple) if not ok: print( "Error: something went wrong while inserting info into inst_package: %s" % (sql_str)) if config_dict["copy_to"]: if os.path.isdir(config_dict["copy_to"]): if os.path.normpath( "%s/%s" % (os.getcwd(), long_package_name)) == os.path.normpath( "%s/%s" % (config_dict["copy_to"], short_package_name)): # same file, no copy pass else: print("Copying %s to %s/%s ..." % (short_package_name, config_dict["copy_to"], short_package_name)) try: shutil.copy2( long_package_name, "%s/%s" % (config_dict["copy_to"], short_package_name)) except: print("An error occured: %s" % (process_tools.get_except_info())) else: print("Error: target directory '%s' does not exist " % (config_dict["copy_to"])) if dc: dc.release() print("done")