def package_it(self):
print("Packaging ...")
width = self.parser.options.use_64_bit and "64" or "32"
libgoto_info_str = "%s-%s-%s-%s%s" % (
self.parser.cpu_id, self.parser.options.fcompiler,
self.parser.short_version, width, self.parser.options.smp and "p"
or "")
libgoto_vers_str = "%s-r%s" % (libgoto_info_str,
self.parser.options.goto_version)
static_library_name, dynamic_library_name = ("libgoto-%s.a" %
(libgoto_vers_str),
"libgoto-%s.so" %
(libgoto_vers_str))
info_name = "README.libgoto-%s" % (libgoto_vers_str)
sep_str = "-" * 50
readme_lines = [
sep_str
] + self.parser.get_compile_options().split("\n") + [
"Compile times: %s" % (", ".join([
"%s: %s" %
(key, logging_tools.get_diff_time_str(self.time_dict[key]))
for key in list(self.time_dict.keys())
])), sep_str, ""
]
if self.parser.options.include_log:
readme_lines.extend(
["Compile logs:"] +
sum([["%s:" %
(key)] + self.log_dict[key].split("\n") + [sep_str]
for key in list(self.log_dict.keys())], []))
libgoto_static_file_name, libgoto_dynamic_file_name = (
"/static_not_found", "/dynamic_not_found")
codecs.open("{}/info".format(self.tempdir), "w",
encoding='utf-8').write("\n".join(readme_lines))
for ent in os.listdir("{}/{}".format(self.tempdir, self.dirname)):
fpath = "{}/{}/{}".format(self.tempdir, self.dirname, ent)
if ent.endswith(".a") and (os.path.isfile(fpath)
and not os.path.islink(fpath)):
libgoto_static_file_name = fpath
if ent.endswith(".so") and (os.path.isfile(fpath)
and not os.path.islink(fpath)):
libgoto_dynamic_file_name = fpath
fpath = ""
package_name, package_version, package_release = (
"libgoto-%s" % (libgoto_info_str),
self.parser.options.goto_version, self.parser.options.release)
# copy libgoto.a
codecs.open("{}.static".format(libgoto_static_file_name),
"wb").write(open(libgoto_static_file_name, "rb").read())
dummy_args = argparse.Namespace(
name=package_name,
version=package_version,
release=package_release,
package_group="Libraries/Math",
description="BLAS",
summary="BLAS Package",
)
new_p = rpm_build_tools.build_package(dummy_args)
if self.parser.options.arch:
new_p["arch"] = self.parser.options.arch
new_p["inst_options"] = " -p "
content = rpm_build_tools.file_content_list([
"%s.static:%s/%s" %
(libgoto_static_file_name, self.parser.options.target_dir,
static_library_name),
"%s:%s/%s" %
(libgoto_dynamic_file_name, self.parser.options.target_dir,
dynamic_library_name),
"%s/info:%s/%s" %
(self.tempdir, self.parser.options.target_dir, info_name),
"%s:%s/Makefile.rule.%s" %
(self.orig_rulefile_name, self.parser.options.target_dir,
libgoto_vers_str)
])
new_p.create_tgz_file(content)
new_p.write_specfile(content)
new_p.build_package()
if new_p.build_ok:
print("Build successfull, package locations:")
print(new_p.long_package_name)
print(new_p.src_package_name)
success = True
else:
print("Something went wrong, please check tempdir %s" %
(self.tempdir))
success = False
return success
def package_it(self):
print("Packaging ...")
info_name = "README.%s" % (self.parser.package_name)
sep_str = "-" * 50
readme_lines = [sep_str] + \
self.parser.get_compile_options().split("\n") + \
["Compile times: %s" % (", ".join(["%s: %s" % (key, logging_tools.get_diff_time_str(self.time_dict[key]["diff"])) for key in list(self.time_dict.keys())])), sep_str, ""]
if self.parser.options.include_log:
readme_lines.extend(["Compile logs:"] + \
sum([self.log_dict[key].split("\n") + [sep_str] for key in list(self.log_dict.keys())], []))
open("%s/%s" % (self.tempdir, info_name),
"w").write("\n".join(readme_lines))
package_name, package_version, package_release = (
self.parser.package_name, self.parser.options.fftw_version,
self.parser.options.release)
# create cluster module file
cmod = cluster_module_tools.cluster_module(
name=package_name,
version=package_version,
release=package_release,
description="FFTW",
#arch=self.parse
)
cmod.add_env_variable(
cluster_module_tools.cluster_module_env(name="PATH",
mode="append",
value="%s/bin" %
(self.parser.fftw_dir)))
cmod.add_env_variable(
cluster_module_tools.cluster_module_env(name="LD_LIBRARY",
mode="append",
value="%s/lib" %
(self.parser.fftw_dir)))
open("%s/cmod" % (self.tempdir),
"w").write(cmod.get_xml_representation())
new_p = rpm_build_tools.build_package()
if self.parser.options.arch:
new_p["arch"] = self.parser.options.arch
new_p["name"] = package_name
new_p["version"] = package_version
new_p["release"] = package_release
new_p["package_group"] = "System/Libraries"
new_p["inst_options"] = " -p "
# remove old info if present
if os.path.isfile("%s/%s" % (self.parser.fftw_dir, info_name)):
os.unlink("%s/%s" % (self.parser.fftw_dir, info_name))
content = rpm_build_tools.file_content_list([
self.parser.fftw_dir,
"%s/%s:%s/%s" %
(self.tempdir, info_name, self.parser.fftw_dir, info_name),
"*%s/cmod:/opt/cluster/modules/%s" %
(self.tempdir, cmod.get_name())
])
new_p.create_tgz_file(content)
new_p.write_specfile(content)
new_p.build_package()
if new_p.build_ok:
print("Build successfull, package locations:")
print(new_p.long_package_name)
print(new_p.src_package_name)
success = True
else:
print("Something went wrong, please check tempdir %s" %
(self.tempdir))
success = False
return success
def package_it(self):
print("Packaging ...")
self.tempdir_2 = tempfile.mkdtemp("_hpl")
info_name = "README.%s" % (self.parser.package_name)
sep_str = "-" * 50
readme_lines = [sep_str] + \
self.parser.get_compile_options().split("\n") + \
[
"Compile times: {}".format(
", ".join(
[
"{}: {}".format(
key,
logging_tools.get_diff_time_str(self.time_dict[key]["diff"])
) for key in list(self.time_dict.keys())
]
)
),
sep_str, ""
]
if self.parser.options.include_log:
readme_lines.extend(
[
"Compile logs:"
] + sum(
[
self.log_dict[key].split("\n") + [sep_str] for key in list(self.log_dict.keys())
],
[]
)
)
open("%s/%s" % (self.tempdir_2, info_name), "w").write("\n".join(readme_lines))
package_name, package_version, package_release = (self.parser.package_name,
self.parser.options.hpl_version,
"1")
hpl_base_dir = os.path.join(
self._hpl_dir,
"bin",
self.cpu_arch
)
xhpl_file_name = "%s/xhpl" % (hpl_base_dir)
if not os.path.isfile(xhpl_file_name):
print("Cannot find %s" % (xhpl_file_name))
success = False
else:
open("%s/xhpl" % (self.tempdir_2), "wb").write(open(xhpl_file_name, "rb").read())
open("%s/Make.%s" % (self.tempdir_2, self.parser.package_name), "wb").write(
open(
os.path.join(self._hpl_dir, "Make.{}".format(self.cpu_arch)),
"rb"
).read()
)
open("%s/HPL.dat" % (self.tempdir_2), "wb").write(open("%s/HPL.dat" % (hpl_base_dir), "rb").read())
os.chmod("%s/xhpl" % (self.tempdir_2), 0o775)
dummy_args = argparse.Namespace(
name=package_name,
version=package_version,
release=package_release,
package_group="Tools/Benchmark",
arch=self.parser.options.arch,
description="HPL",
summary="HPL",
)
new_p = rpm_build_tools.build_package(dummy_args)
new_p["inst_options"] = " -p "
content = rpm_build_tools.file_content_list(["%s:%s" % (self.tempdir_2, self.parser.hpl_dir)])
new_p.create_tgz_file(content)
new_p.write_specfile(content)
new_p.build_package()
if new_p.build_ok:
print("Build successfull, package locations:")
print(new_p.long_package_name)
print(new_p.src_package_name)
success = True
else:
print("Something went wrong, please check tempdir %s" % (self.tempdir))
success = False
return success
def package_it(self):
print("Packaging ...")
info_name = "README.%s" % (self.parser.package_name)
sep_str = "-" * 50
readme_lines = [
sep_str
] + self.parser.get_compile_options().split("\n") + [
"Compile times: {}".format(", ".join([
"%s: %s" %
(key,
logging_tools.get_diff_time_str(self.time_dict[key]["diff"]))
for key in list(self.time_dict.keys())
])), sep_str, ""
]
if self.parser.options.include_log:
readme_lines.extend(["Compile logs:"] + sum([
self.log_dict[key].split("\n") + [sep_str]
for key in list(self.log_dict.keys())
], []))
open("%s/%s" % (self.tempdir, info_name),
"w").write("\n".join(readme_lines))
package_name, package_version, package_release = (
self.parser.package_name, self.parser.options.mpi_version,
self.parser.options.release)
if self.parser.options.module_file:
self._create_module_file()
if self.parser.options.mpi_selector:
self._create_mpi_selector_file()
dummy_args = argparse.Namespace(
name=package_name,
version=package_version,
release=package_release,
package_group="System/Libraries",
arch=self.parser.options.arch,
description="MPI via {}".format(self.parser.mode),
summary="{}".format(self.parser.mode),
)
new_p = rpm_build_tools.build_package(dummy_args)
new_p["inst_options"] = " -p "
# set flag
remove_mca = False
if self.parser.mode == "openmpi":
# remove config files
for file_name in os.listdir("%s/etc" % (self.parser.mpi_dir)):
try:
os.unlink("%s/etc/%s" % (self.parser.mpi_dir, file_name))
except:
pass
# generate empty hosts file
open("%s/etc/openmpi-default-hostfile" % (self.parser.mpi_dir),
"w").write("")
# generate link
mca_local_file = "/etc/openmpi-mca-params.conf"
os.symlink(
mca_local_file,
"%s/etc/openmpi-mca-params.conf" % (self.parser.mpi_dir))
# generate dummy file if not existent
if not os.path.exists(mca_local_file):
remove_mca = True
open(mca_local_file, "w").write("")
# remove old info if present
if os.path.isfile("%s/%s" % (self.parser.mpi_dir, info_name)):
os.unlink("%s/%s" % (self.parser.mpi_dir, info_name))
fc_list = [
self.parser.mpi_dir,
"%s/%s:%s/%s" %
(self.tempdir, info_name, self.parser.mpi_dir, info_name)
]
if self.parser.options.mpi_selector:
fc_list.append("{0}{1}:{1}".format(
self.tempdir,
os.path.join(self.mpi_selector_dir,
self.mpi_selector_sh_name)))
fc_list.append("{0}{1}:{1}".format(
self.tempdir,
os.path.join(self.mpi_selector_dir,
self.mpi_selector_csh_name)))
if self.parser.options.module_file:
fc_list.append("{0}{1}:{1}".format(
self.tempdir,
os.path.join(self.parser.options.module_dir,
self.modulefile_name),
))
content = rpm_build_tools.file_content_list(fc_list)
new_p.create_tgz_file(content)
new_p.write_specfile(content)
new_p.build_package()
if new_p.build_ok:
print("Build successfull, package locations:")
print(new_p.long_package_name)
print(new_p.src_package_name)
success = True
else:
print("Something went wrong, please check tempdir %s" %
(self.tempdir))
success = False
if remove_mca:
os.unlink(mca_local_file)
return success
def main():
package_group = "System/Monitoring"
dc = None
config_dict = {
"dist_dir": "",
"copy_to": "",
"exc_dir_names": [],
}
if not (config_dict["dist_dir"] or config_dict["copy_to"]):
if os.path.isfile("/etc/debian_version"):
config_dict["dist_dir"] = "/packages/debian"
config_dict["copy_to"] = "/opt/cluster/system/packages/debian"
else:
config_dict["dist_dir"] = "/packages/RPMs"
config_dict["copy_to"] = "/opt/cluster/system/packages/RPMs"
long_package_name = None
script_dict = {}
# build_package entity
my_arg = rpm_build_tools.package_parser()
opts = my_arg.parse_args()
build_p = rpm_build_tools.build_package(opts)
if not long_package_name:
build_it = True
if build_it:
act_cl = rpm_build_tools.file_content_list(
opts.args) # , exclude_dir_names=config_dict["exc_dir_names"])
if not act_cl:
print("Need a file and/or directory-list !")
if dc:
dc.release()
sys.exit(-1)
act_cl.show_content()
# if release == "AUTO_INCR":
# release = check_auto_incr_release(dc, name, version)
if dc:
sql_str = "SELECT p.pgroup, COUNT(DISTINCT p.package_idx) AS c FROM inst_package ip, package p WHERE ip.package=p.package_idx GROUP BY p.pgroup"
dc.execute(sql_str)
all_g = [x["pgroup"] for x in dc.fetchall()]
if package_group not in all_g:
print(
"Package Group '%s' is new (already existing: %s); continue ?"
% (package_group, ", ".join(all_g) or "<none>"))
ret = None
while ret not in ["yes", "no"]:
ret = getinput(" - [yes/no]")
if ret == "no":
build_it = False
elif ret == "yes":
break
if build_it:
# build_p["version"] = version
# build_p["release"] = release
# build_p["name"] = name
# build_p["package_group"] = package_group
# set scripts
for key, value in script_dict.items():
build_p[key] = value
build_p.create_tgz_file(act_cl)
build_p.write_specfile(act_cl)
build_p.build_package()
if build_p.build_ok:
if dc:
print("Success, insert info into database...")
else:
print("Success, local mode")
long_package_name = build_p.long_package_name
print("Package locations:")
print(build_p.long_package_name)
print(build_p.src_package_name)
else:
print("Build went wrong, exiting")
build_it = False
if not build_it:
if dc:
dc.release()
sys.exit(0)
short_package_name = os.path.basename(long_package_name)
if dc:
p_idx = None
stat, out = subprocess.getstatusoutput(
"/usr/local/sbin/insert_package_info.py -p %s" %
(long_package_name))
if stat:
print("Some error occured (%d) : " % (stat))
for line in [x.strip() for x in out.split("\n")]:
print(" %s" % (line))
if dc:
dc.release()
sys.exit(3)
else:
for line in [x.strip() for x in out.split("\n")]:
if line.startswith("pidx"):
p_idx = int(line.split()[1])
elif line.startswith("pname"):
name = line.split(None, 1)[1]
# p_idx now holds the index of the package
if p_idx is None:
print("Can't get db-index of package info %s" %
(long_package_name))
if dc:
dc.release()
sys.exit(2)
if dc:
# read name, version and release from db
dc.execute(
"SELECT p.name, p.version, p.release, p.buildhost, p.packager FROM package p WHERE p.package_idx=%d"
% (p_idx))
db_stuff = dc.fetchone()
name, version, release = (db_stuff["name"], db_stuff["version"],
db_stuff["release"])
set_dict = {
"package":
p_idx,
"location":
os.path.normpath(
"%s/%s" % (config_dict["dist_dir"], short_package_name)),
"last_build":
time.time()
}
write_entry = check_for_association(dc, p_idx, name, version,
release, set_dict)
if write_entry:
sql_str, sql_tuple = (
"INSERT INTO inst_package SET %s" % (", ".join(
["%s=%%s" % (key) for key in list(set_dict.keys())])),
tuple([set_dict[key] for key in list(set_dict.keys())]))
ok = dc.execute(sql_str, sql_tuple)
if not ok:
print(
"Error: something went wrong while inserting info into inst_package: %s"
% (sql_str))
if config_dict["copy_to"]:
if os.path.isdir(config_dict["copy_to"]):
if os.path.normpath(
"%s/%s" %
(os.getcwd(), long_package_name)) == os.path.normpath(
"%s/%s" %
(config_dict["copy_to"], short_package_name)):
# same file, no copy
pass
else:
print("Copying %s to %s/%s ..." %
(short_package_name, config_dict["copy_to"],
short_package_name))
try:
shutil.copy2(
long_package_name, "%s/%s" %
(config_dict["copy_to"], short_package_name))
except:
print("An error occured: %s" %
(process_tools.get_except_info()))
else:
print("Error: target directory '%s' does not exist " %
(config_dict["copy_to"]))
if dc:
dc.release()
print("done")