def testDefaultProductPath(self):
man = Manifest("numpy", "1.6.1+1")
self.assertEquals("afw/4.5", man.defaultProductPath("afw", "4.5"))
self.assertEquals("external/afw/4.5",
man.defaultProductPath("afw", "4.5", "external"))
self.assertEquals("goofy/afw/4.5",
man.defaultProductPath("afw", "4.5", "goofy"))
self.assertEquals("afw/4.5", man.defaultProductPath("afw", "4.5+3"))
def getBestManifest(prodname, useprods, reftags):
manfile = None
if prodname in useprods:
# instructed to use a specific version
ver = useprods[prodname]
# look for a deployed version
manfile = os.path.join(opts.serverdir, "manifests",
"%s-%s.manifest" % (prodname, ver))
if not os.path.exists(manfile):
manfile = getUndeployedManifestFile(prodname, ver)
if not manfile or not os.path.exists(manfile):
# consult the tag file
ver = reftags.getVersion(prodname)
manfile = os.path.join(opts.serverdir, "manifests",
"%s-%s.manifest" % (prodname, ver))
if not os.path.exists(manfile):
manfile = getUndeployedManifestFile(prodname, ver)
if not os.path.exists(manfile):
raise RuntimeError("Can't find a manifest file for %s %s" %
(prodname, ver))
return Manifest.fromFile(manfile)
def getBestManifest(prodname, useprods, reftags):
manfile = None
if prodname in useprods:
# instructed to use a specific version
ver = useprods[prodname]
# look for a deployed version
manfile = os.path.join(opts.serverdir, "manifests",
"%s-%s.manifest" % (prodname, ver))
if not os.path.exists(manfile):
manfile = getUndeployedManifestFile(prodname, ver)
if not manfile or not os.path.exists(manfile):
# consult the tag file
ver = reftags.getVersion(prodname)
manfile = os.path.join(opts.serverdir, "manifests",
"%s-%s.manifest" % (prodname, ver))
if not os.path.exists(manfile):
manfile = getUndeployedManifestFile(prodname, ver)
if not os.path.exists(manfile):
raise RuntimeError("Can't find a manifest file for %s %s" %
(prodname, ver))
return Manifest.fromFile(manfile)
def testWrite1(self):
self.assertEquals("server-tmp", os.path.basename(self.serverroot))
self.assert_(os.path.exists(self.serverroot))
newpyfitsfile = "tests/server-tmp/external/pyfits/2.4.0/b2.manifest"
newmplfile = "tests/server-tmp/external/matplotlib/1.0.1/b2.manifest"
self.assert_(not os.path.exists(newpyfitsfile))
self.assert_(not os.path.exists(newmplfile))
rel = UpdateDependents([("numpy", "1.6.1+1")], self.serverroot)
updated = rel.createManifests()
self.assertEquals(2, len(updated))
prod = filter(lambda p: p[0] == "pyfits", updated)
self.assertEquals(1, len(prod))
prod = prod[0]
self.assertEquals("2.4.0", prod[1])
self.assertEquals(2, prod[2])
self.assert_(prod[3].endswith(newpyfitsfile))
prod = filter(lambda p: p[0] == "matplotlib", updated)
self.assertEquals(1, len(prod))
prod = prod[0]
self.assertEquals("1.0.1", prod[1])
self.assertEquals(2, prod[2])
self.assert_(prod[3].endswith(newmplfile))
self.assert_(os.path.exists(newpyfitsfile))
self.assert_(os.path.exists(newmplfile))
man = Manifest.fromFile(newpyfitsfile)
rec = man.getProduct("pyfits")
self.assert_(rec is not None)
self.assertEquals(rec[2], "2.4.0+2")
rec = man.getProduct("python")
self.assert_(rec is not None)
self.assertEquals(rec[2], "2.7.2+1")
man = Manifest.fromFile(newmplfile)
rec = man.getProduct("matplotlib")
self.assert_(rec is not None)
self.assertEquals(rec[2], "1.0.1+2")
def testWrite1(self):
self.assertEquals("server-tmp", os.path.basename(self.serverroot))
self.assert_(os.path.exists(self.serverroot))
newpyfitsfile = "tests/server-tmp/external/pyfits/2.4.0/b2.manifest"
newmplfile = "tests/server-tmp/external/matplotlib/1.0.1/b2.manifest"
self.assert_(not os.path.exists(newpyfitsfile))
self.assert_(not os.path.exists(newmplfile))
rel = UpdateDependents([("numpy", "1.6.1+1")], self.serverroot)
updated = rel.createManifests()
self.assertEquals(2, len(updated))
prod = filter(lambda p: p[0] == "pyfits", updated)
self.assertEquals(1, len(prod))
prod = prod[0]
self.assertEquals("2.4.0", prod[1])
self.assertEquals(2, prod[2])
self.assert_(prod[3].endswith(newpyfitsfile))
prod = filter(lambda p: p[0] == "matplotlib", updated)
self.assertEquals(1, len(prod))
prod = prod[0]
self.assertEquals("1.0.1", prod[1])
self.assertEquals(2, prod[2])
self.assert_(prod[3].endswith(newmplfile))
self.assert_(os.path.exists(newpyfitsfile))
self.assert_(os.path.exists(newmplfile))
man = Manifest.fromFile(newpyfitsfile)
rec = man.getProduct("pyfits")
self.assert_(rec is not None)
self.assertEquals(rec[2], "2.4.0+2")
rec = man.getProduct("python")
self.assert_(rec is not None)
self.assertEquals(rec[2], "2.7.2+1")
man = Manifest.fromFile(newmplfile)
rec = man.getProduct("matplotlib")
self.assert_(rec is not None)
self.assertEquals(rec[2], "1.0.1+2")
def testRoundTrip(self):
path = os.path.join(self.server, "manifests", "numpy-1.6.1+1.manifest")
filecontents = StringIO()
line = None
with open(path) as fd:
for line in fd:
filecontents.write(line)
man = Manifest.fromFile(path)
mancontents = StringIO()
man.write(mancontents)
self.assertEquals(filecontents.getvalue(), mancontents.getvalue())
def testRoundTrip(self):
path = os.path.join(self.server, "manifests", "numpy-1.6.1+1.manifest")
filecontents = StringIO()
line = None
with open(path) as fd:
for line in fd:
filecontents.write(line)
man = Manifest.fromFile(path)
mancontents = StringIO()
man.write(mancontents)
self.assertEquals(filecontents.getvalue(), mancontents.getvalue())
def testDefaultProductPath(self):
man = Manifest("numpy", "1.6.1+1")
self.assertEquals("afw/4.5", man.defaultProductPath("afw", "4.5"))
self.assertEquals("external/afw/4.5",
man.defaultProductPath("afw", "4.5", "external"))
self.assertEquals("goofy/afw/4.5",
man.defaultProductPath("afw", "4.5", "goofy"))
self.assertEquals("afw/4.5", man.defaultProductPath("afw", "4.5+3"))
def testAddExtRecord(self):
man = Manifest("numpy", "1.6.1+1")
self.assert_(not man.hasProduct("python"))
man.addExtRecord("python", "2.7.2", build=2, id="bld")
self.assert_(man.hasProduct("python"))
self.assert_(man.hasRecord("python", "generic", "2.7.2+2"))
self.assert_(not man.hasRecord("python", "generic", "2.7.2"))
rec = man.getRecord("python", "2.7.2+2")
self.assert_(rec)
dep = Dependency(rec)
self.assertEquals("python", dep.data[dep.NAME])
self.assertEquals("generic", dep.data[dep.FLAVOR])
self.assertEquals("2.7.2+2", dep.data[dep.VERSION])
self.assertEquals("external/python/2.7.2/python.table",
dep.data[dep.TABLEFILE])
self.assertEquals("external/python/2.7.2+2", dep.data[dep.INSTALLDIR])
self.assertEquals("lsstbuild:external/python/2.7.2/python.bld",
dep.data[dep.INSTALLID])
def testAddExtRecord(self):
man = Manifest("numpy", "1.6.1+1")
self.assert_(not man.hasProduct("python"))
man.addExtRecord("python", "2.7.2", build=2, id="bld")
self.assert_(man.hasProduct("python"))
self.assert_(man.hasRecord("python", "generic", "2.7.2+2"))
self.assert_(not man.hasRecord("python", "generic", "2.7.2"))
rec = man.getRecord("python", "2.7.2+2")
self.assert_(rec)
dep = Dependency(rec)
self.assertEquals("python", dep.data[dep.NAME])
self.assertEquals("generic", dep.data[dep.FLAVOR])
self.assertEquals("2.7.2+2", dep.data[dep.VERSION])
self.assertEquals("external/python/2.7.2/python.table",
dep.data[dep.TABLEFILE])
self.assertEquals("external/python/2.7.2+2", dep.data[dep.INSTALLDIR])
self.assertEquals("lsstbuild:external/python/2.7.2/python.bld",
dep.data[dep.INSTALLID])
def testAddSelfRecord(self):
man = Manifest("python", "2.7.2+5", "external")
self.assert_(not man.hasProduct("python"))
self.assert_(man.getSelf() is None)
# pdb.set_trace()
man.addSelfRecord()
self.assert_(man.hasProduct("python"))
rec = man.getSelf()
self.assert_(rec)
dep = Dependency(rec)
self.assertEquals("python", dep.data[dep.NAME])
self.assertEquals("generic", dep.data[dep.FLAVOR])
self.assertEquals("2.7.2+5", dep.data[dep.VERSION])
self.assertEquals("external/python/2.7.2/python.table",
dep.data[dep.TABLEFILE])
self.assertEquals("external/python/2.7.2+5", dep.data[dep.INSTALLDIR])
self.assertEquals(
"lsstbuild:external/python/2.7.2/python-2.7.2.tar.gz",
dep.data[dep.INSTALLID])
def testAddSelfRecord(self):
man = Manifest("python", "2.7.2+5", "external")
self.assert_(not man.hasProduct("python"))
self.assert_(man.getSelf() is None)
# pdb.set_trace()
man.addSelfRecord();
self.assert_(man.hasProduct("python"))
rec = man.getSelf()
self.assert_(rec)
dep = Dependency(rec)
self.assertEquals("python", dep.data[dep.NAME])
self.assertEquals("generic", dep.data[dep.FLAVOR])
self.assertEquals("2.7.2+5", dep.data[dep.VERSION])
self.assertEquals("external/python/2.7.2/python.table",
dep.data[dep.TABLEFILE])
self.assertEquals("external/python/2.7.2+5", dep.data[dep.INSTALLDIR])
self.assertEquals("lsstbuild:external/python/2.7.2/python-2.7.2.tar.gz",
dep.data[dep.INSTALLID])
def testCtor(self):
man = Manifest("numpy", "1.6.1+1")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("generic", id[2])
man = Manifest("numpy", "1.6.1+1", "external")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("generic", id[2])
man = Manifest("numpy", "1.6.1+1", flavor="Darwin86")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("Darwin86", id[2])
man = Manifest("numpy", "1.6.1+1", "external", flavor="Darwin86")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("Darwin86", id[2])
def testFromFile(self):
path = os.path.join(self.server, "manifests", "numpy-1.6.1+1.manifest")
man = Manifest.fromFile(path)
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("generic", id[2])
self.assert_(man.hasProduct("tcltk"))
self.assert_(man.hasProduct("python"))
self.assert_(man.hasProduct("numpy"))
rec = man.getRecord("python", "2.7.2+1")
self.assert_(rec)
dep = Dependency(rec)
self.assertEquals("python", dep.data[dep.NAME])
self.assertEquals("generic", dep.data[dep.FLAVOR])
self.assertEquals("2.7.2+1", dep.data[dep.VERSION])
self.assertEquals("external/python/2.7.2/python.table",
dep.data[dep.TABLEFILE])
self.assertEquals("external/python/2.7.2+1", dep.data[dep.INSTALLDIR])
self.assertEquals("lsstbuild:external/python/2.7.2/python.bld",
dep.data[dep.INSTALLID])
def testFromFile(self):
path = os.path.join(self.server, "manifests", "numpy-1.6.1+1.manifest")
man = Manifest.fromFile(path)
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("generic", id[2])
self.assert_(man.hasProduct("tcltk"))
self.assert_(man.hasProduct("python"))
self.assert_(man.hasProduct("numpy"))
rec = man.getRecord("python", "2.7.2+1")
self.assert_(rec)
dep = Dependency(rec)
self.assertEquals("python", dep.data[dep.NAME])
self.assertEquals("generic", dep.data[dep.FLAVOR])
self.assertEquals("2.7.2+1", dep.data[dep.VERSION])
self.assertEquals("external/python/2.7.2/python.table",
dep.data[dep.TABLEFILE])
self.assertEquals("external/python/2.7.2+1", dep.data[dep.INSTALLDIR])
self.assertEquals("lsstbuild:external/python/2.7.2/python.bld",
dep.data[dep.INSTALLID])
def testCtor(self):
man = Manifest("numpy", "1.6.1+1")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("generic", id[2])
man = Manifest("numpy", "1.6.1+1", "external")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("generic", id[2])
man = Manifest("numpy", "1.6.1+1", flavor="Darwin86")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("Darwin86", id[2])
man = Manifest("numpy", "1.6.1+1", "external", flavor="Darwin86")
id = man.getNameVerFlav()
self.assertEquals("numpy", id[0])
self.assertEquals("1.6.1+1", id[1])
self.assertEquals("Darwin86", id[2])
def main():
(opts, args) = options()
if len(args) < 1:
raise RuntimeError("Missing manifest and tag file names")
if not opts.sdir:
raise RuntimeError("Missing server directory")
sdir = opts.sdir
mdir = os.path.join(sdir, "manifests")
inman = Manifest.fromFile(args[0])
prodname = inman.name
version = inman.vers
if len(args) > 1:
tagfile = args[1]
elif opts.tag:
tagfile = os.path.join(sdir, "%s.list" % opts.tag)
else:
raise RuntimeError("No tag or tagfile specified")
tagged = TagDef(tagfile)
deps = inman.getDeps()
adjusted = []
depnames = set()
# iterate backwards
for i in range(-1, -1 * len(deps) - 1, -1):
dep = deps[i]
if dep.data[dep.NAME] not in depnames:
if dep.data[dep.NAME] == prodname:
# dep.data[dep.FLAVOR] = "generic"
if opts.bnum:
if extRe.search(dep.data[dep.VERSION]):
dep.data[dep.VERSION] = extRe.sub(
r"\g<1>%s" % opts.bnum, dep.data[dep.VERSION])
dep.data[dep.INSTALLDIR] = extRe.sub(
r"\g<1>%s" % opts.bnum, dep.data[dep.INSTALLDIR])
else:
dep.data[dep.VERSION] += "+%s" % opts.bnum
dep.data[dep.INSTALLDIR] += "+%s" % opts.bnum
adjusted.insert(0, dep)
else:
depver = tagged.getVersion(dep.data[dep.NAME])
if depver:
mfile = os.path.join(
mdir, "%s-%s.manifest" % (dep.data[dep.NAME], depver))
if not depver or not os.path.exists(mfile):
adjusted.insert(0, dep)
else:
adeps = Manifest.fromFile(mfile).getDeps()
for j in range(-1, -1 * len(adeps) - 1, -1):
adep = adeps[j]
if adep.data[adep.NAME] not in depnames:
adjusted.insert(0, adep)
depnames.add(adep.data[adep.NAME])
depnames.add(dep.data[dep.NAME])
else:
p = map(
lambda p1: p1[0],
filter(lambda i: i[1].data[0] == dep.data[dep.NAME],
enumerate(adjusted)))
if len(p) > 1:
raise RuntimeError(
"corrupted dep list: too many entries for product " +
dep.data[dep.NAME])
if len(p) == 0:
raise RuntimeError(
"corrupted dep list: missing entry for product " +
dep.data[dep.NAME])
adjusted.insert(0, adjusted.pop(p[0]))
if opts.bnum:
version = textRe.sub(r"\g<1>%s" % opts.bnum, version)
outman = Manifest(prodname, version)
for dep in adjusted:
outman.addRecord(*dep.data)
outman.write(sys.stdout)
def main():
(opts, args) = options()
if len(args) < 1:
raise RuntimeError("Missing manifest and tag file names")
if not opts.sdir:
raise RuntimeError("Missing server directory")
sdir = opts.sdir
mdir = os.path.join(sdir, "manifests")
inman = Manifest.fromFile(args[0])
prodname = inman.name
version = inman.vers
if len(args) > 1:
tagfile = args[1]
elif opts.tag:
tagfile = os.path.join(sdir, "%s.list" % opts.tag)
else:
raise RuntimeError("No tag or tagfile specified")
tagged = TagDef(tagfile)
deps = inman.getDeps()
adjusted = []
depnames = set()
# iterate backwards
for i in range(-1, -1*len(deps)-1, -1):
dep = deps[i]
if dep.data[dep.NAME] not in depnames:
if dep.data[dep.NAME] == prodname:
# dep.data[dep.FLAVOR] = "generic"
if opts.bnum:
if extRe.search(dep.data[dep.VERSION]):
dep.data[dep.VERSION] = extRe.sub(r"\g<1>%s" % opts.bnum,
dep.data[dep.VERSION])
dep.data[dep.INSTALLDIR] = extRe.sub(r"\g<1>%s" % opts.bnum,
dep.data[dep.INSTALLDIR])
else:
dep.data[dep.VERSION] += "+%s" % opts.bnum
dep.data[dep.INSTALLDIR] += "+%s" % opts.bnum
adjusted.insert(0, dep)
else:
depver = tagged.getVersion(dep.data[dep.NAME])
if depver:
mfile = os.path.join(mdir, "%s-%s.manifest" %
(dep.data[dep.NAME], depver))
if not depver or not os.path.exists(mfile):
adjusted.insert(0, dep)
else:
adeps = Manifest.fromFile(mfile).getDeps()
for j in range(-1, -1*len(adeps)-1, -1):
adep = adeps[j]
if adep.data[adep.NAME] not in depnames:
adjusted.insert(0, adep)
depnames.add(adep.data[adep.NAME])
depnames.add(dep.data[dep.NAME])
else:
p = map(lambda p1: p1[0],
filter(lambda i: i[1].data[0] == dep.data[dep.NAME],
enumerate(adjusted)))
if len(p) > 1:
raise RuntimeError("corrupted dep list: too many entries for product " + dep.data[dep.NAME])
if len(p) == 0:
raise RuntimeError("corrupted dep list: missing entry for product " + dep.data[dep.NAME])
adjusted.insert(0, adjusted.pop(p[0]))
if opts.bnum:
version = textRe.sub(r"\g<1>%s" % opts.bnum, version)
outman = Manifest(prodname, version)
for dep in adjusted:
outman.addRecord(*dep.data)
outman.write(sys.stdout)
def testDefaultId(self):
man = Manifest("numpy", "1.6.1+1")
self.assertEquals("lsstbuild:afw/4.5/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5"))
self.assertEquals("lsstbuild:external/afw/4.5/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5", "external"))
self.assertEquals(
"lsstbuild:external/afw/4.5/Linux/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5", "external", "Linux"))
self.assertEquals(
"lsstbuild:afw-4.5/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5", path="afw-4.5"))
self.assertEquals("lsstbuild:afw/4.5/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5"))
self.assertEquals("lsstbuild:external/afw/4.5/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5", "external"))
self.assertEquals(
"lsstbuild:external/afw/4.5/Linux/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5", "external", "Linux"))
self.assertEquals(
"lsstbuild:afw-4.5/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5", path="afw-4.5"))
self.assertEquals("lsstbuild:afw/4.5/afw.bld",
man.defaultID("bld", "afw", "4.5"))
self.assertEquals("lsstbuild:external/afw/4.5/afw.bld",
man.defaultID("bld", "afw", "4.5", "external"))
self.assertEquals(
"lsstbuild:external/afw/4.5/Linux/afw.bld",
man.defaultID("bld", "afw", "4.5", "external", "Linux"))
self.assertEquals("lsstbuild:afw-4.5/afw.bld",
man.defaultID("bld", "afw", "4.5", path="afw-4.5"))
def testDefaultId(self):
man = Manifest("numpy", "1.6.1+1")
self.assertEquals("lsstbuild:afw/4.5/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5"))
self.assertEquals("lsstbuild:external/afw/4.5/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5", "external"))
self.assertEquals("lsstbuild:external/afw/4.5/Linux/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5", "external", "Linux"))
self.assertEquals("lsstbuild:afw-4.5/afw-4.5.tar.gz",
man.defaultID("tarball", "afw", "4.5", path="afw-4.5"))
self.assertEquals("lsstbuild:afw/4.5/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5"))
self.assertEquals("lsstbuild:external/afw/4.5/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5", "external"))
self.assertEquals("lsstbuild:external/afw/4.5/Linux/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5", "external", "Linux"))
self.assertEquals("lsstbuild:afw-4.5/afw-4.5.tar.gz",
man.defaultID("lsstbuild", "afw", "4.5", path="afw-4.5"))
self.assertEquals("lsstbuild:afw/4.5/afw.bld",
man.defaultID("bld", "afw", "4.5"))
self.assertEquals("lsstbuild:external/afw/4.5/afw.bld",
man.defaultID("bld", "afw", "4.5", "external"))
self.assertEquals("lsstbuild:external/afw/4.5/Linux/afw.bld",
man.defaultID("bld", "afw", "4.5", "external", "Linux"))
self.assertEquals("lsstbuild:afw-4.5/afw.bld",
man.defaultID("bld", "afw", "4.5", path="afw-4.5"))