def __init__(self, name, srcloc=None):
self._name = ""
self.setName(name)
self._srcLoc = []
self.addSourceLocation(srcloc)
self._exclude = list()
self._converter = dict()
self._destLoc = None
if not self._destLoc:
self._destLoc = os.path.abspath(os.getcwd())
self._tmpdir = TempDir(suffix="tempdir", prefix=self._name)
def setUp(self):
unittest.TestCase.setUp(self)
self.tmp = TempDir("testInstallProject")
self.tmpdir = self.tmp.getName()
self.distdir = os.path.join(self.tmpdir, "dist")
os.mkdir(self.distdir)
self.vodir = os.path.join(self.tmpdir, "vo")
os.mkdir(self.vodir)
self.mysiteroot = os.path.join(self.vodir, "lib")
os.mkdir(self.mysiteroot)
self.ip = os.path.join(self.mysiteroot, "install_project.py")
shutil.copy(_lblg_ip, self.ip)
def setUp(self):
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.ERROR)
unittest.TestCase.setUp(self)
self.tmpdir = TempDir(suffix="tempdir", prefix="testfixcvsroot")
self.dir1 = os.path.join(self.tmpdir.getName(), "dir1")
os.makedirs(self.dir1)
self.dir2 = os.path.join(self.tmpdir.getName(), "dir2")
os.makedirs(os.path.join(self.dir2, "CVS"))
self.dir3 = os.path.join(self.tmpdir.getName(), "dir3")
os.makedirs(os.path.join(self.dir3, "CVS"))
f = open(os.path.join(self.dir3, "CVS", "Root"), "w")
f.write("something")
f.close()
self.dir4 = os.path.join(self.tmpdir.getName(), "dir4")
os.makedirs(os.path.join(self.dir4, "CVS"))
f = open(os.path.join(self.dir4, "CVS", "Root"), "w")
f.write(":ext:something")
f.close()
self.dir5 = os.path.join(self.tmpdir.getName(), "dir5")
os.makedirs(os.path.join(self.dir5, "CVS"))
f = open(os.path.join(self.dir5, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
self.dir6 = os.path.join(self.tmpdir.getName(), "dir6")
os.makedirs(os.path.join(self.dir6, "CVS"))
f = open(os.path.join(self.dir6, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
self.dir7 = os.path.join(self.tmpdir.getName(), "dir6", "dir7")
os.makedirs(os.path.join(self.dir7, "CVS"))
f = open(os.path.join(self.dir7, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
self.dir8 = os.path.join(self.tmpdir.getName(), "dir6", "dir7", "dir8")
os.makedirs(os.path.join(self.dir8, "CVS"))
f = open(os.path.join(self.dir8, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
class InstallProjectDeploymentTestCase(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.tmp = TempDir("testInstallProject")
self.tmpdir = self.tmp.getName()
self.distdir = os.path.join(self.tmpdir, "dist")
os.mkdir(self.distdir)
self.vodir = os.path.join(self.tmpdir, "vo")
os.mkdir(self.vodir)
self.mysiteroot = os.path.join(self.vodir, "lib")
os.mkdir(self.mysiteroot)
self.ip = os.path.join(self.mysiteroot, "install_project.py")
shutil.copy(_lblg_ip, self.ip)
def tearDown(self):
unittest.TestCase.tearDown(self)
def testSetup(self):
self.assertTrue(os.path.exists(self.distdir))
self.assertTrue(os.path.exists(self.vodir))
self.assertTrue(os.path.exists(self.mysiteroot))
self.assertTrue(os.path.isfile(self.ip))
def testSelfVersion(self):
latest_version = os.popen("python %s --version" % self.ip).read()[:-1]
self.assertEqual(script_version, latest_version)
class TarBall(object):
def __init__(self, name, srcloc=None):
self._name = ""
self.setName(name)
self._srcLoc = []
self.addSourceLocation(srcloc)
self._exclude = list()
self._converter = dict()
self._destLoc = None
if not self._destLoc:
self._destLoc = os.path.abspath(os.getcwd())
self._tmpdir = TempDir(suffix="tempdir", prefix=self._name)
def setName(self, name):
self._name = name
def getName(self):
return self._name
def addSourceLocation(self, srcloc):
self._srcLoc += os.path.abspath(srcloc)
def getSourceLocation(self):
return self._srcLoc
def addExclusion(self, pattern):
self._exclude += pattern
def addConverter(self, target, source):
""" will convert the source into the target before the
creation of the tar ball. A target can have many sources """
self._converter.setdefault(target, []).append(source)
def explodeSources(self):
for s in self._srcLoc:
if os.path.isfile(s):
if tarfile.is_tarfile(s):
srctar = tarfile.open(s,"r:*")
srctar.extractall(self._tmpdir.getName())
srctar.close()
else:
print "%s is not a tar file. Skipping ..." % s
else:
shutil.copytree(s, self._tmpdir.getName())
def create(self):
curdir = os.getcwd()
os.chdir(self._tmpdir.getName())
os.chdir(curdir)
def createTarBalls(sourcedirs,
output_dir=os.getcwd(),
tar_name="package",
get_share=True,
binary_list=[],
tar_type="gz",
symlinks=True,
prefix=None,
exclude=[],
top_dir=False,
top_exclude=None):
tmpdir = TempDir(suffix="tempdir", prefix="name")
tmpdirname = tmpdir.getName()
copySources(sourcedirs, tmpdirname, symlinks, exclude)
# facility to select this binary from the first level of subdirectories
if top_dir:
for t in os.listdir(tmpdirname):
add_to = True
# remove the entries caught by the top filter
for f in top_exclude:
if fnmatch.fnmatch(t, f):
add_to = False
break
if add_to:
binary_list.append(t)
# remove outside links and transform absolute ones into relative links
fixLinks(tmpdirname)
# the binary tar ball
for b in binary_list:
filename = getTarFileName(tar_name, b, tar_type, output_dir)
createTarBall(tmpdirname, filename, b, binary_list, symlinks, prefix)
# the shared tarball contains whatever remains
if get_share:
filename = getTarFileName(tar_name, None, tar_type, output_dir)
createTarBall(tmpdirname, filename, None, binary_list, symlinks,
prefix)
class TestCase(unittest.TestCase):
""" test case for the temporary directory class """
def setUp(self):
unittest.TestCase.setUp(self)
self.samplebasename = "toto"
self.tmpdir = TempDir(suffix="tempdir", prefix=self.samplebasename)
def tearDown(self):
unittest.TestCase.tearDown(self)
def testName(self):
name1 = str(self.tmpdir)
name2 = self.tmpdir.getName()
# the next line should print twice the same thing
print self.tmpdir, name2, " ",
self.assert_(name1 == name2)
def testDestruction(self):
mydir = TempDir(suffix="tempdir", prefix="mydir")
mydirname = mydir.getName()
del mydir
# the temporary directory should have been removed
self.assert_(not os.path.exists(mydirname))
def setUp(self):
unittest.TestCase.setUp(self)
self.samplebasename = "toto"
self.tmpdir = TempDir(suffix="tempdir", prefix=self.samplebasename)
def testDestruction(self):
mydir = TempDir(suffix="tempdir", prefix="mydir")
mydirname = mydir.getName()
del mydir
# the temporary directory should have been removed
self.assert_(not os.path.exists(mydirname))
def createTarBallFromFilterDict(srcdict,
filename,
pathfilter=None,
dereference=False,
update=False,
use_tmp=False):
""" function to create a tarball. The srcdict parameter is a dictionary
containing the path of the source as the key and the prefix in the tarball as value.
@param srcdict: dictionary with the src directory and their prefix
@param filename: output file name
@param pathfilter: filter function that takes as input a path and returns True or False if
the path as to be included in the tarball or not.
@param dereference: dereference links in the tarball or not
@param update: if the tarball has to be either updated or replaced
@param use_tmp: if True, creates the file in a temporary directory and move it.
"""
log = logging.getLogger()
status = 0
if not update:
log.debug("Creating %s" % filename)
if use_tmp:
tmpdir = TempDir(suffix="tempdir",
prefix="createTarBallFromFilter")
real_name = filename
filename = os.path.join(tmpdir.getName(),
os.path.basename(filename))
log.debug("Using temporary file %s" % filename)
tarf, lock = openTar(filename, tar_mode="w")
keep_tar = False
srcexist = False
for dirname in srcdict.keys():
if os.path.exists(dirname):
srcexist = True
break
if not srcexist:
os.remove(filename)
status = 1 # status=1 means that the src directories do not exist
else:
for dirname in srcdict.keys():
if os.path.exists(dirname):
prefix = srcdict[dirname]
if prefix:
log.debug("Adding %s to %s" % (dirname, prefix))
else:
log.debug("Adding %s" % dirname)
for fullo, relo in listTarBallObjects(
dirname, pathfilter, prefix):
keep_tar = True
tarf.add(fullo, relo, recursive=False)
else:
log.debug("The directory %s doesn't exist. Skipping." %
dirname)
closeTar(tarf, lock, filename)
if not keep_tar:
log.warning("%s file is empty. Removing it." % filename)
os.remove(filename)
status = 2 # status=2 means that the tarball was empty
if use_tmp:
copy2(filename, real_name)
log.debug("Copying %s to %s" % (filename, real_name))
else:
status = updateTarBallFromFilterDict(srcdict, filename, pathfilter,
dereference)
return status
def updateTarBallFromFilterDict(srcdict,
filename,
pathfilter=None,
dereference=False):
""" function to update an already existing tarball. The srcdict parameter is a dictionary
containing the path of the source as the key and the prefix in the tarball as value.
@param srcdict: dictionary with the src directory and their prefix
@param filename: output file name
@param pathfilter: filter function that takes as input a path and returns True or False if
the path as to be included in the tarball or not.
@param dereference: dereference links in the tarball or not
"""
log = logging.getLogger()
status = 0
if not os.path.exists(filename):
status = createTarBallFromFilterDict(srcdict,
filename,
pathfilter,
dereference,
update=False)
else:
log.debug("Updating %s" % filename)
tarf, lock = openTar(filename, tar_mode="r")
tmpdir = TempDir(suffix="tempdir", prefix="updateTarballFromFilter")
tarf.extractall(tmpdir.getName())
extracted_objs = [x[1] for x in listTarBallObjects(tmpdir.getName())]
closeTar(tarf, lock, filename)
tobeupdated = False
srcexist = False
for dirname in srcdict.keys():
if os.path.exists(dirname):
srcexist = True
break
if not srcexist:
return 1
for dirname in srcdict.keys():
if os.path.exists(dirname):
prefix = srcdict[dirname]
if prefix:
log.debug("Adding %s to %s" % (dirname, prefix))
else:
log.debug("Adding %s" % dirname)
if prefix:
dstprefix = os.path.join(tmpdir.getName(), prefix)
if not os.path.exists(dstprefix):
os.makedirs(dstprefix)
for y in listTarBallObjects(dirname, pathfilter, prefix):
if y[1] not in extracted_objs:
tobeupdated = True
src = y[0]
dst = os.path.join(tmpdir.getName(), y[1])
if not dereference and os.path.islink(src):
linkto = os.readlink(src)
os.symlink(linkto, dst)
elif os.path.isdir(src):
os.mkdir(dst)
else:
copy2(src, dst)
else:
log.debug("The directory %s doesn't exist. Skipping." %
dirname)
if tobeupdated:
log.debug("Updating tarball %s" % filename)
status = createTarBallFromFilter([tmpdir.getName()],
filename,
pathfilter=None,
prefix=None,
dereference=dereference,
update=False)
return status
def setUp(self):
unittest.TestCase.setUp(self)
self._tmpdir = TempDir(suffix="tempdir", prefix="testlinks")
@author: Ben Couturier
'''
import os
import re
import sys
import logging
from string import Template
from subprocess import Popen, PIPE
from LbLegacy.Utils import getStatusOutput
from LbUtils.Temporary import TempDir
from LbUtils.CMT.Common import CMTCommand as CMT
from LbUtils.CMT.Common import isCMTMessage
tmpdir = TempDir(prefix="LHCbExternalsRpmSpec")
__log__ = logging.getLogger(__name__)
#
# Utilities imported from mkLCGCMTtar to extract list of native versions
# from dependencies !
#
# BEWARE: This is a copy for code from mkLCGCMtar and it needs a lot of
# cleanup before proper production.
#
###############################################################################
# Method added to facilitate the lookup of macro values
def get_macro_value(cmtdir, macro, extratags):
""" Returns the value of a macro """
here = os.getcwd()
if cmtdir != None:
class TestCase(unittest.TestCase):
def setUp(self):
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.ERROR)
unittest.TestCase.setUp(self)
self.tmpdir = TempDir(suffix="tempdir", prefix="testfixcvsroot")
self.dir1 = os.path.join(self.tmpdir.getName(), "dir1")
os.makedirs(self.dir1)
self.dir2 = os.path.join(self.tmpdir.getName(), "dir2")
os.makedirs(os.path.join(self.dir2, "CVS"))
self.dir3 = os.path.join(self.tmpdir.getName(), "dir3")
os.makedirs(os.path.join(self.dir3, "CVS"))
f = open(os.path.join(self.dir3, "CVS", "Root"), "w")
f.write("something")
f.close()
self.dir4 = os.path.join(self.tmpdir.getName(), "dir4")
os.makedirs(os.path.join(self.dir4, "CVS"))
f = open(os.path.join(self.dir4, "CVS", "Root"), "w")
f.write(":ext:something")
f.close()
self.dir5 = os.path.join(self.tmpdir.getName(), "dir5")
os.makedirs(os.path.join(self.dir5, "CVS"))
f = open(os.path.join(self.dir5, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
self.dir6 = os.path.join(self.tmpdir.getName(), "dir6")
os.makedirs(os.path.join(self.dir6, "CVS"))
f = open(os.path.join(self.dir6, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
self.dir7 = os.path.join(self.tmpdir.getName(), "dir6", "dir7")
os.makedirs(os.path.join(self.dir7, "CVS"))
f = open(os.path.join(self.dir7, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
self.dir8 = os.path.join(self.tmpdir.getName(), "dir6", "dir7", "dir8")
os.makedirs(os.path.join(self.dir8, "CVS"))
f = open(os.path.join(self.dir8, "CVS", "Root"), "w")
f.write(":kserver:something")
f.close()
def tearDown(self):
unittest.TestCase.tearDown(self)
def testSimpleEmptyDirectory(self):
fixRootFile(self.dir1, ":kserver:", ":ext:")
self.assertFalse(os.path.exists(os.path.join(self.dir1, "CVS")))
self.assertFalse(os.path.exists(os.path.join(self.dir1, "CVS",
"Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir1, "CVS", "Root.new")))
self.assert_(
not os.path.exists(os.path.join(self.dir1, "CVS", "Root.bak")))
def testSimpleDirectoryWithoutRoot(self):
fixRootFile(self.dir2, ":kserver:", ":ext:")
self.assertTrue(os.path.exists(os.path.join(self.dir2, "CVS")))
self.assertFalse(os.path.exists(os.path.join(self.dir2, "CVS",
"Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir2, "CVS", "Root.new")))
self.assertFalse(
os.path.exists(os.path.join(self.dir2, "CVS", "Root.bak")))
def testSimpleDirectoryWithFakeRoot(self):
fixRootFile(self.dir3, ":kserver:", ":ext:")
self.assertTrue(os.path.exists(os.path.join(self.dir3, "CVS")))
self.assertTrue(os.path.exists(os.path.join(self.dir3, "CVS", "Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir3, "CVS", "Root.new")))
self.assertFalse(
os.path.exists(os.path.join(self.dir3, "CVS", "Root.bak")))
f = open(os.path.join(self.dir3, "CVS", "Root"), "r")
self.assertTrue(f.read() == "something")
f.close()
def testSimpleDirectoryWithWrongRoot(self):
fixRootFile(self.dir4, ":kserver:", ":ext:")
self.assertTrue(os.path.exists(os.path.join(self.dir4, "CVS")))
self.assertTrue(os.path.exists(os.path.join(self.dir4, "CVS", "Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir4, "CVS", "Root.new")))
self.assertFalse(
os.path.exists(os.path.join(self.dir4, "CVS", "Root.bak")))
f = open(os.path.join(self.dir4, "CVS", "Root"), "r")
self.assertTrue(f.read() == ":ext:something")
f.close()
def testSimpleDirectoryWithRoot(self):
fixRootFile(self.dir5, ":kserver:", ":ext:")
self.assertTrue(os.path.exists(os.path.join(self.dir5, "CVS")))
self.assertTrue(os.path.exists(os.path.join(self.dir5, "CVS", "Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir5, "CVS", "Root.new")))
self.assertTrue(
os.path.exists(os.path.join(self.dir5, "CVS", "Root.bak")))
f = open(os.path.join(self.dir5, "CVS", "Root"), "r")
self.assertTrue(f.read() == ":ext:something")
f.close()
def testDirectory(self):
fixRoot(self.dir6, ":kserver:", ":ext:", True)
self.assertTrue(os.path.exists(os.path.join(self.dir6, "CVS")))
self.assertTrue(os.path.exists(os.path.join(self.dir6, "CVS", "Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir6, "CVS", "Root.new")))
self.assertTrue(
os.path.exists(os.path.join(self.dir6, "CVS", "Root.bak")))
f = open(os.path.join(self.dir6, "CVS", "Root"), "r")
self.assertTrue(f.read() == ":ext:something")
f.close()
self.assertTrue(os.path.exists(os.path.join(self.dir7, "CVS")))
self.assertTrue(os.path.exists(os.path.join(self.dir7, "CVS", "Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir7, "CVS", "Root.new")))
self.assertTrue(
os.path.exists(os.path.join(self.dir7, "CVS", "Root.bak")))
f = open(os.path.join(self.dir7, "CVS", "Root"), "r")
self.assertTrue(f.read() == ":ext:something")
f.close()
self.assertTrue(os.path.exists(os.path.join(self.dir8, "CVS")))
self.assertTrue(os.path.exists(os.path.join(self.dir8, "CVS", "Root")))
self.assertFalse(
os.path.exists(os.path.join(self.dir8, "CVS", "Root.new")))
self.assertTrue(
os.path.exists(os.path.join(self.dir8, "CVS", "Root.bak")))
f = open(os.path.join(self.dir8, "CVS", "Root"), "r")
self.assertTrue(f.read() == ":ext:something")
f.close()