def getParentProjects(project, version=None, cmtconfig=None, top_dir=None):
log = logging.getLogger()
if Project.isProject(project) :
pcfg = Project.getProject(project)
if not top_dir :
top_dir = pcfg.ReleaseArea()
if not version :
pattern = "%s_*" % pcfg.NAME()
maindir = os.path.join(top_dir, pcfg.NAME())
version = str(Version.getVersionsFromDir(maindir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (pcfg.Name(), version))
elif Package.isPackage(project) :
pcfg = Package.getPackage(project)
if not top_dir :
top_dir = pcfg.ReleaseArea()
if not version :
maindir = os.path.join(top_dir, pcfg.releasePrefix())
version = str(Version.getVersionsFromDir(maindir, pattern=None, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (pcfg.Name(), version))
prj_dict = getTarDependencies(project, version, cmtconfig,
top_dir, project_only=False, full=False)
par_dict = {}
fulllist = [ x.upper() for x in Project.project_names ]
fulllist += [ x.upper() for x in Package.package_names ]
for p in prj_dict.keys() :
if not prj_dict[p] :
continue
if p.upper() not in fulllist :
continue
par_dict[p] = prj_dict[p]
return par_dict
def buildLCGTar(project, version=None, cmtconfig=None,
top_dir=None, output_dir=None,
overwrite=False, update=False,
md5=True, html=True):
log = logging.getLogger()
status = 0
if Project.isProject(project) :
log.debug("%s is a software project" % project)
prj_conf = Project.getProject(project)
if not top_dir :
top_dir = prj_conf.ReleaseArea()
if not output_dir :
output_dir = prj_conf.LCGTarBallDir()
if not version :
pattern = "%s_*" % prj_conf.NAME()
maindir = os.path.join(top_dir, prj_conf.NAME())
version = str(Version.getVersionsFromDir(maindir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (prj_conf.Name(), version))
if not cmtconfig :
cmtconfig = Platform.binary_opt_list
for c in cmtconfig :
status = generateLCGTar(project, version, c, top_dir, output_dir,
overwrite, update, md5, html)
if status == 1 :
return status
else :
log.fatal("%s is not a software project" % project)
status = 1
return status
def checkTar(project, version=None, cmtconfig=None, input_dir=None,
keep_going=False):
log = logging.getLogger()
good = True
if Project.isProject(project) :
log.debug("%s is a software project" % project)
prj_conf = Project.getProject(project)
if not input_dir :
input_dir = prj_conf.TarBallDir()
if not version :
pattern = "%s_*" % prj_conf.NAME()
version = str(Version.getVersionsFromDir(input_dir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (prj_conf.Name(), version))
if not cmtconfig :
cmtconfig = Platform.binary_list
for c in cmtconfig :
log.debug("="*100)
status = checkMD5(project, version, c, input_dir)
if not status :
good = False
if not keep_going and not good:
return good
log.debug("="*100)
status = checkMD5(project, version, None, input_dir)
elif Package.isPackage(project) :
log.debug("%s is a data package" % project)
status = checkDataMD5(project, version, input_dir)
else :
log.fatal("%s is neither a software project nor a data package" % project)
status = 1
if not status :
good = False
return good
def checkMD5(project, version=None, cmtconfig=None, input_dir=None):
log = logging.getLogger()
prj_conf = Project.getProject(project)
good = False
if not input_dir :
input_dir = prj_conf.TarBallDir()
if not version :
pattern = "%s_*" % prj_conf.NAME()
version = str(Version.getVersionsFromDir(input_dir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (prj_conf.Name(), version))
filename = os.path.join(input_dir, prj_conf.tarBallName(version, cmtconfig, full=True))
if os.path.exists(filename) :
md5fname = os.path.join(input_dir, prj_conf.md5FileName(version, cmtconfig))
target_name = os.path.basename(filename)
if os.path.exists(md5fname) :
log.info("Checking %s for %s" % (md5fname, filename))
if checkMD5Info(md5fname, target_name) :
log.info("The file %s is OK" % md5fname)
good = True
else :
log.error("The file %s is wrong" % md5fname)
good = False
else :
log.error("The file %s doesn't exist" % md5fname)
good = False
else :
log.warning("The file %s doesn't exist. Skipping md5 check." % filename)
good = True
return good
def generateDataTar(package, version=None, top_dir=None,
output_dir=None, overwrite=False,
md5=True, html=True):
log = logging.getLogger()
pkg_conf = Package.getPackage(package)
if not top_dir :
top_dir = pkg_conf.ReleaseArea()
if not output_dir :
output_dir = pkg_conf.TarBallDir()
if not version :
maindir = os.path.join(top_dir, pkg_conf.releasePrefix())
version = str(Version.getVersionsFromDir(maindir, pattern=None, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (pkg_conf.Name(), version))
prefix = pkg_conf.releasePrefix(version)
srcdirs = [os.path.join(top_dir, prefix)]
log.debug("="*100)
filename = os.path.join(output_dir, pkg_conf.tarBallName(version, full=True))
pathfilter = projectFilter
if os.path.exists(filename) and not overwrite :
log.info("The file %s already exists. Skipping." % filename)
status = 2
else :
status = createTarBallFromFilter(srcdirs, filename, pathfilter,
prefix=prefix, dereference=pkg_conf.dereferenceTar(),
update=False)
if status == 1 :
log.fatal("The source directories do not exist")
return status
if md5 :
generateDataMD5(package, version, output_dir)
if html :
generateDataHTML(package, version, top_dir, output_dir)
return status
def getTarBallNameItems(tar_name):
name = None
version = None
binary = None
if not tar_name.endswith(".tar.gz") :
raise NotATarBall
core_name = tar_name.replace(".tar.gz", "")
for b in Platform.binary_list + Platform.extra_binary_list :
if core_name.endswith(b) :
binary = b
core_name = core_name.replace("_%s" %b, "")
break
cptes = core_name.split("_")
nm = cptes[0]
if nm.upper() in [ x.upper() for x in Project.project_names ] :
version = Version.CoreVersion(cptes[-1]).name()
prjcfg = Project.getProject(cptes[0])
if prjcfg.tarBallName(version, cmtconfig=binary, full=True) == tar_name :
name = prjcfg.Name()
else :
raise NotATarBall
elif nm.upper() in [ x.upper() for x in Package.project_names ] :
version = Version.CoreVersion(cptes[-1]).name()
pkgnm = None
for c in cptes :
if c in Package.package_names :
pkgnm = c
break
if pkgnm :
pkgcfg = Package.getPackage(pkgnm)
else :
raise NotATarBall
if pkgcfg.tarBallName(version, full=True) == tar_name :
name = pkgcfg.Name()
else :
raise NotATarBall
else :
if nm.upper() == "LCGCMT" :
name = "LCGCMT"
version = Version.LCGVersion(cptes[1]).name()
return name, version, binary
def generateDataHTML(package, version, top_dir=None, output_dir=None) :
log = logging.getLogger()
pkg_conf = Package.getPackage(package)
if not top_dir :
top_dir = pkg_conf.ReleaseArea()
if not output_dir :
output_dir = pkg_conf.HTMLDir()
if not version :
maindir = os.path.join(top_dir, pkg_conf.releasePrefix())
version = str(Version.getVersionsFromDir(maindir, pattern=None, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (pkg_conf.Name(), version))
def buildTar(project, version=None, cmtconfig=None,
top_dir=None, output_dir=None,
overwrite=False, overwrite_shared=False, update=False,
md5=True, html=True, recursive=False):
log = logging.getLogger()
status = 0
if Project.isProject(project) :
log.debug("%s is a software project" % project)
prj_conf = Project.getProject(project)
if not top_dir :
top_dir = prj_conf.ReleaseArea()
if not output_dir :
output_dir = prj_conf.TarBallDir()
if not version :
pattern = "%s_*" % prj_conf.NAME()
maindir = os.path.join(top_dir, prj_conf.NAME())
version = str(Version.getVersionsFromDir(maindir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (prj_conf.Name(), version))
if not cmtconfig :
cmtconfig = Platform.binary_list
if overwrite :
overwrite_shared = True
update_shared = False
for c in cmtconfig :
status = generateTar(project, version, c, top_dir, output_dir,
overwrite, update, md5, html)
if status == 1 :
return status
if status != 2 :
update_shared = True
if update_shared :
if overwrite_shared :
status = generateTar(project, version, cmtconfig=None,
top_dir=top_dir, output_dir=output_dir,
overwrite=True, update=True, md5=md5, html=html)
else :
status = generateTar(project, version, cmtconfig=None,
top_dir=top_dir, output_dir=output_dir,
overwrite=False, update=True, md5=md5, html=html)
elif Package.isPackage(project) :
log.debug("%s is a data package" % project)
status = generateDataTar(project, version, top_dir, output_dir, overwrite, md5, html)
else :
log.fatal("%s is neither a software project nor a data package")
status = 1
if recursive :
par_prj = getParentProjects(project, version, None, top_dir)
for p in par_prj.keys() :
status = buildTar(p, par_prj[p], cmtconfig, top_dir, output_dir, overwrite, overwrite_shared, update, md5, html, recursive)
return status
def generateHTML(project, version, cmtconfig=None, top_dir=None, output_dir=None):
log = logging.getLogger()
prj_conf = Project.getProject(project)
if not top_dir :
top_dir = prj_conf.ReleaseArea()
if not output_dir :
output_dir = prj_conf.HTMLDir()
tb_dir = prj_conf.TarBallDir()
else :
tb_dir = output_dir
if not version :
pattern = "%s_*" % prj_conf.NAME()
maindir = os.path.join(top_dir, prj_conf.NAME())
version = str(Version.getVersionsFromDir(maindir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (prj_conf.Name(), version))
filename = os.path.join(tb_dir, prj_conf.tarBallName(version, cmtconfig, full=True))
if os.path.exists(filename) :
title = "Project %s version %s" % (prj_conf.Name(), version)
if cmtconfig :
depcont = "<H3>%s (%s binary files) </H3>\n" % (title, cmtconfig)
else :
depcont = "<H3>%s (source files) </H3>\n" % title
depcont += "%s\n" % getHTMLLink(project, version, cmtconfig, with_name=True)
dep_list = getTarDependencies(project, version, cmtconfig, top_dir, project_only=False, full=True)
tobeadded = []
if "LCGCMT" in dep_list.keys() :
tobeadded += [ "[%s]\n" % x for x in getLCGHTMLLinks(project, version, dep_list, cmtconfig) ]
for p in dep_list.keys() :
if Project.isProject(p) :
tobeadded.append( "[%s]\n" % getHTMLLink(p, dep_list[p], cmtconfig, with_name=False))
for p in dep_list.keys() :
if Package.isPackage(p) :
tobeadded.append("[%s]\n" % getHTMLDataLink(p, dep_list[p], with_name=False))
if tobeadded :
depcont += "<MENU><LI>\n"
depcont += "".join(tobeadded)
depcont += "</MENU>\n"
htmlname = os.path.join(output_dir, prj_conf.htmlFileName(version, cmtconfig))
if os.path.exists(htmlname) :
os.remove(htmlname)
log.info("Replacing %s for %s" % (htmlname, filename))
else :
log.info("Creating %s for %s" % (htmlname, filename))
hf = AFSLockFile(htmlname, "w", force=False)
hf.write(depcont)
hf.close()
def generateTar(project, version=None, cmtconfig=None,
top_dir=None, output_dir=None, overwrite=False,
update=False, md5=True, html=True):
log = logging.getLogger()
prj_conf = Project.getProject(project)
status = 0
if not top_dir :
top_dir = prj_conf.ReleaseArea()
if not output_dir :
output_dir = prj_conf.TarBallDir()
if not version :
pattern = "%s_*" % prj_conf.NAME()
maindir = os.path.join(top_dir, prj_conf.NAME())
version = str(Version.getVersionsFromDir(maindir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (prj_conf.Name(), version))
prefix = prj_conf.releasePrefix(version)
srcdirs = [os.path.join(top_dir, prefix)]
log.debug("="*100)
filename = os.path.join(output_dir, prj_conf.tarBallName(version, cmtconfig, full=True))
if os.path.exists(filename) and not (overwrite or update) :
log.info("The file %s already exists. Skipping." % filename)
status = 2
else :
if overwrite :
os.remove(filename)
if cmtconfig :
pathfilter = lambda x : projectFilter(x, cmtconfig)
else :
pathfilter = projectFilter
prefix = prj_conf.releasePrefix(version)
status = createTarBallFromFilter(srcdirs, filename, pathfilter,
prefix=prefix, dereference=False, update=update)
if status != 0 :
if status == 1 :
log.fatal("The source directories do not exist")
else :
if md5 :
generateMD5(project, version, cmtconfig, output_dir)
if html :
generateHTML(project, version, cmtconfig, top_dir, output_dir)
return status
def generateLCGTar(project, version=None, cmtconfig=None,
top_dir=None, output_dir=None, overwrite=False,
update=False, md5=True, html=True):
log = logging.getLogger()
prj_conf = Project.getProject(project)
status = 0
if not prj_conf.LCGTarBallName() :
log.debug("The project %s has no LCG tarball dependency" % prj_conf.Name())
else :
if not top_dir :
top_dir = prj_conf.ReleaseArea()
if not output_dir :
output_dir = prj_conf.LCGTarBallDir()
if not version :
pattern = "%s_*" % prj_conf.NAME()
maindir = os.path.join(top_dir, prj_conf.NAME())
version = str(Version.getVersionsFromDir(maindir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (prj_conf.Name(), version))
prefix = prj_conf.releasePrefix(version)
ppath = os.path.join(top_dir, prefix)
prj = CMT.Project(ppath)
for data in CMT.walk(top=prj, cmtprojectpath=os.environ["CMTPROJECTPATH"]) :
if data[0].name() == "LCGCMT" :
lcg_prj = data[0]
break
lcg_version = version
if prj_conf.LCGTarBallName() == "LCGCMT" :
lcg_version = lcg_prj.version().split("_")[-1]
filename = os.path.join(output_dir, prj_conf.LCGTarBallName(lcg_version, cmtconfig, full=True))
log.info("The output file name is %s" % filename)
if os.path.exists(filename) and not (overwrite or update) :
log.info("The file %s already exists. Skipping." % filename)
status = 2
else :
shared_pathfilter = projectFilter
if cmtconfig :
binary_pathfilter = lambda x : projectFilter(x, cmtconfig)
ext_shared_dict = {}
ext_binary_dict = {}
shared_location = lcg_prj.fullLocation()
if os.path.exists(shared_location) :
shared_prefix = os.sep.join(["external", lcg_prj.name(), lcg_prj.version()])
ext_shared_dict[shared_location] = shared_prefix
log.debug("Adding %s to %s" % (shared_location, shared_prefix))
if cmtconfig :
for bin_location, bin_prefix in listTarBallObjects(shared_location, binary_pathfilter, shared_prefix, top_most=True) :
if os.path.exists(bin_location):
ext_binary_dict[bin_location] = bin_prefix
log.debug("Adding %s to %s" % (bin_location, bin_prefix))
os.environ["CMTEXTRATAGS"] = "LHCb,LHCbGrid"
ext_dict = {}
lcg_relloc = None
lcg_extloc = None
for p in prj.binaryExternalPackages(cmtprojectpath=os.environ["CMTPROJECTPATH"],
binary=cmtconfig) :
if p.name().split(os.sep)[0] == "LCG_Interfaces" :
if not lcg_relloc :
lcg_relloc = fixWinMacroValue(p.getMacroValue("LCG_release"))
if not lcg_extloc :
lcg_relloc = fixWinMacroValue(p.getMacroValue("LCG_external"))
ext_name = os.sep.join(p.name().split(os.sep)[1:])
ext_dict[ext_name] = []
ext_dict[ext_name].append(p.fullLocation())
ext_home = fixWinMacroValue(p.getMacroValue("%s_home" % ext_name))
ext_dict[ext_name].append(ext_home)
ext_config_version = p.getMacroValue("%s_config_version" % ext_name)
ext_dict[ext_name].append(ext_config_version)
ext_native_version = p.getMacroValue("%s_native_version" % ext_name)
ext_dict[ext_name].append(ext_native_version)
if lcg_extloc and ext_home.startswith(lcg_extloc) :
lcg_prefix = lcg_extloc
elif lcg_relloc and ext_home.startswith(lcg_relloc) :
lcg_prefix = lcg_relloc
else :
lcg_prefix = ""
ext_dict[ext_name].append(lcg_prefix)
ext_coreloc = ""
ext_pathloc = ""
if ext_home :
core_list = []
ext_coreloc = ext_home.replace(lcg_prefix + os.sep,"")
for c in ext_coreloc.split(os.sep) :
core_list.append(c)
if c == ext_native_version :
break
if core_list :
ext_coreloc = os.sep.join(core_list)
path_list = []
ext_pathloc = ext_home
for c in ext_home.split(os.sep) :
path_list.append(c)
if c == ext_native_version :
break
if path_list :
ext_pathloc = os.sep.join(path_list)
ext_dict[ext_name].append(ext_pathloc)
ext_dict[ext_name].append(ext_coreloc)
for p in ext_dict.keys() :
if ext_dict[p][1] :
shared_prefix = os.path.join("external",ext_dict[p][6])
shared_location = ext_dict[p][5]
if shared_location.endswith(shared_prefix) and os.path.exists(shared_location):
ext_shared_dict[shared_location] = shared_prefix
log.debug("Adding %s to %s" % (shared_location, shared_prefix))
if cmtconfig :
for bin_location, bin_prefix in listTarBallObjects(shared_location, binary_pathfilter, shared_prefix, top_most=True) :
if os.path.exists(bin_location):
ext_binary_dict[bin_location] = bin_prefix
log.debug("Adding %s to %s" % (bin_location, bin_prefix))
log.info("Adding %d shared locations" % len(ext_shared_dict))
log.info("Adding %d binary locations" % len(ext_binary_dict))
log.debug("="*100)
if overwrite :
os.remove(filename)
status = createTarBallFromFilterDict(ext_shared_dict, filename, shared_pathfilter,
dereference=False, update=update,
use_tmp=True)
if status != 0 :
if status == 1 :
log.fatal("The source directories do not exist")
else:
status = updateTarBallFromFilterDict(ext_binary_dict, filename, binary_pathfilter, dereference=False)
if status != 0 :
if status == 1 :
log.fatal("The source directories do not exist")
else :
if md5 :
generateLCGMD5(project, version, cmtconfig, output_dir)
if html :
generateLCGHTML(project, version, cmtconfig, output_dir)
# else :
# if md5 :
# generateMD5(project, version, cmtconfig, output_dir)
# if html :
# generateHTML(project, version, cmtconfig, top_dir, output_dir)
if status == 0 :
log.info("The file %s was successfully generated" % filename)
return status
def getTarDependencies(project, version=None, cmtconfig=None, top_dir=None,
project_only=False, full=True):
prj_dict = {}
log = logging.getLogger()
cmtprojectpath = os.environ["CMTPROJECTPATH"]
if Project.isProject(project) :
pcfg = Project.getProject(project)
if not top_dir :
top_dir = pcfg.ReleaseArea()
if not version :
pattern = "%s_*" % pcfg.NAME()
maindir = os.path.join(top_dir, pcfg.NAME())
version = str(Version.getVersionsFromDir(maindir, pattern, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (pcfg.Name(), version))
prefix = pcfg.releasePrefix(version)
ppath = os.path.join(top_dir, prefix)
prj = CMT.Project(ppath)
if not full :
for p in prj.base(cmtprojectpath=cmtprojectpath) :
prj_dict[p.name()] = p.version()
else :
for data in CMT.walk(top=prj, cmtprojectpath=cmtprojectpath) :
prj_dict[data[0].name()] = data[0].version()
for p in prj_dict.keys() :
if p == "LCGCMT" :
if prj_dict[p] :
prj_dict[p] = prj_dict[p].split("_")[1]
elif p == "CMTUSERCONTEXT" :
del prj_dict[p]
continue
elif p.upper() == project.upper() :
del prj_dict[p]
continue
else :
ver = Version.extractVersion(prj_dict[p])
if ver :
prj_dict[p] = ver.name()
if not project_only :
has_dataproj = False
for p in Package.project_names :
if p in prj_dict.keys() :
has_dataproj = True
break
if has_dataproj :
for p in prj.externalPackages(cmtprojectpath=cmtprojectpath) :
parprj = p.parentProject()
if parprj.name() in Package.project_names :
prj_dict[p.coreName()] = p.version()
elif Package.isPackage(project) :
pcfg = Package.getPackage(project)
if not top_dir :
top_dir = pcfg.ReleaseArea()
if not version :
maindir = os.path.join(top_dir, pcfg.releasePrefix())
version = str(Version.getVersionsFromDir(maindir, pattern=None, reverse=True)[0])
log.debug("Guessed version for %s is %s" % (pcfg.Name(), version))
prefix = pcfg.releasePrefix(version)
# filter out items without version
for k in prj_dict.keys() :
if not prj_dict[k] :
del prj_dict[k]
return prj_dict