def _packageFolder(tfile, relbasepath, docrootbasepath):
getLogger().debug("Walking folder %s..." % docrootbasepath)
for entry in FileSystemManager.instance().getdir(docrootbasepath):
name, apptype = entry['name'], entry['type']
if apptype == FileSystemManager.APPTYPE_DIR:
relpath = "%s%s/" % (relbasepath, name) # the name of the current item within the package
docrootpath = "%s%s/" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding directory %s..." % relpath)
tarinfo = tarfile.TarInfo(relpath)
tarinfo.type = tarfile.DIRTYPE
tarinfo.mode = 0755
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
tfile.addfile(tarinfo)
_packageFolder(tfile, relbasepath = relpath, docrootbasepath = docrootpath)
else:
relname = "%s%s" % (relbasepath, name) # the name of the current item within the package
docrootname = "%s%s" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding file %s..." % relname)
tarinfo = tarfile.TarInfo(relname)
tarinfo.type = tarfile.AREGTYPE
tarinfo.mode = 0644
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
content = FileSystemManager.instance().read(docrootname)
tarinfo.size = len(content)
contentObj = StringIO.StringIO(content)
tfile.addfile(tarinfo, contentObj)
contentObj.close()
getLogger().debug("File %s added to package file (%s bytes)" % (relname, tarinfo.size))
def schedulePackage(path,
username,
session,
at,
script=None,
profileName=None):
"""
Schedules a package to start at <at>.
If script is provided, the package's default-script attribute is ignored
and script is used instead.
@since: 1.3
"""
getLogger().info(">> schedulePackage(%s, script = %s)" % (path, script))
if not path.startswith('/'): path = '/' + path
try:
metadata = Package.getPackageMetadata(path)
if not script:
script = metadata['default-script']
scriptFilename = "%s/src/%s" % (path, script)
scriptPath = scriptFilename
label = "%s:%s" % ('/'.join(path.split('/')[2:]), script)
getLogger().info("Using package script filename %s, path %s" %
(scriptFilename, scriptPath))
if script.endswith(".campaign"):
res = scheduleCampaign(FileSystemManager.instance().read(
scriptFilename).decode('utf-8'),
label,
username,
session,
at=at,
path=scriptPath)
elif script.endswith(".ats"):
res = scheduleAts(FileSystemManager.instance().read(
scriptFilename).decode('utf-8'),
label,
username,
session,
at=at,
path=scriptPath)
else:
raise Exception(
"Invalid script/default script for package execution (unrecognized job type based on extension - %s)"
% script)
except Exception as e:
e = Exception("Scheduling error: %s" % (str(e)))
getLogger().info("<< schedulePackage(...): Fault:\n%s" %
Tools.getBacktrace())
raise (e)
getLogger().info("<< schedulePackage(...): %s" % str(res))
return res
def removeDirectory(path, recursive=False):
"""
Removes an empty directory, unless recursive is set to True.
@since: 1.1
@type path: string
@param path: the docroot-path to the directory to delete
@type recursive: bool
@param recursive: True if we should delete files and directories in it. DANGEROUS.
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeDirectory(%s)" % path)
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rmdir(path, recursive)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< removeDirectory(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< removeDirectory(): %s" % str(res))
return res
def getDirectoryListing(path):
"""
Returns the contents of a directory.
Also filters some 'internal' files (in particular __init__.py files)
@since: 1.0
@type path: string
@param path: the path of the directory within the docroot
@rtype: list of dict{'name': string, 'type': string in [ ats, campaign, module, log, directory, package, ... ] }
@returns: the dir contents, with a name (with extension) relative to the dir,
and an associated application type.
Returns None if the directory was not accessible or in case of an error.
"""
getLogger().info(">> getDirectoryListing(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = []
try:
res = FileSystemManager.instance().getdir(path)
if res is None:
raise Exception("Unable to get directory contents through backend")
# Sort the entries, so that it is useless to implement it in all clients ?
res.sort(key=operator.itemgetter('name'))
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getDirectoryListing(...): Fault:\n%s" % str(e))
# Well, actually, we do not return a fault in this case...
res = None
def rename(path, newName):
"""
Renames a file or a directory to newName, in the same folder.
@since: 1.3
@type path: string
@param path: the docroot-path to the object to rename
@type newName: string
@param newName: the new name (basename) of the object, including extension,
if applicable.
@rtype: bool
@returns: False if newName already exists. True otherwise.
"""
getLogger().info(">> rename(%s, %s)" % (path, newName))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rename(path, newName)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< rename(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< rename(): %s" % str(res))
return res
def getDirectoryListing(path):
"""
Returns the contents of a directory.
Also filters some 'internal' files (in particular __init__.py files)
@since: 1.0
@type path: string
@param path: the path of the directory within the docroot
@rtype: list of dict{'name': string, 'type': string in [ ats, campaign, module, log, directory, package, ... ] }
@returns: the dir contents, with a name (with extension) relative to the dir,
and an associated application type.
Returns None if the directory was not accessible or in case of an error.
"""
getLogger().info(">> getDirectoryListing(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = []
try:
res = FileSystemManager.instance().getdir(path)
if res is None:
raise Exception("Unable to get directory contents through backend")
# Sort the entries, so that it is useless to implement it in all clients ?
res.sort(key = operator.itemgetter('name'))
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getDirectoryListing(...): Fault:\n%s" % str(e))
# Well, actually, we do not return a fault in this case...
res = None
def getFileInfo(path):
"""
Gets some info about a file.
Returns a dict{ 'size': integer, 'timestamp': float }
where the size is optional (in bytes, if provided), and timestamp is
the file modification time.
@since: 1.0
@type path: string
@param path: the path to the file whose info we want to get
@rtype: a dict, or None
@returns: None on error, or the dict of attributes.
"""
getLogger().info(">> getFileInfo(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = None
try:
attributes = FileSystemManager.instance().attributes(path)
if attributes:
res = {}
if attributes.size is not None:
res['size'] = attributes.size
if attributes.mtime is not None:
res['timestamp'] = attributes.mtime
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getFileInfo(...): Fault:\n" + str(e))
raise(e)
def rename(path, newName):
"""
Renames a file or a directory to newName, in the same folder.
@since: 1.3
@type path: string
@param path: the docroot-path to the object to rename
@type newName: string
@param newName: the new name (basename) of the object, including extension,
if applicable.
@rtype: bool
@returns: False if newName already exists. True otherwise.
"""
getLogger().info(">> rename(%s, %s)" % (path, newName))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rename(path, newName)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< rename(...): Fault:\n" + str(e))
raise(e)
def getFileInfo(path):
"""
Gets some info about a file.
Returns a dict{ 'size': integer, 'timestamp': float }
where the size is optional (in bytes, if provided), and timestamp is
the file modification time.
@since: 1.0
@type path: string
@param path: the path to the file whose info we want to get
@rtype: a dict, or None
@returns: None on error, or the dict of attributes.
"""
getLogger().info(">> getFileInfo(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = None
try:
attributes = FileSystemManager.instance().attributes(path)
if attributes:
res = {}
if attributes.size is not None:
res['size'] = attributes.size
if attributes.mtime is not None:
res['timestamp'] = attributes.mtime
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getFileInfo(...): Fault:\n" + str(e))
raise (e)
def removeFile(path):
"""
Removes a file.
@since: 1.0
@type path: string
@param path: the docroot-path to the file to delete
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeFile(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = False
try:
res = FileSystemManager.instance().unlink(path)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< removeFile(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< removeFile(): %s" % str(res))
return res
def makeDirectory(path):
"""
Creates a directory and all the needed directories to it, if any.
@since: 1.3
@type path: string
@param path: the docroot-path to the directory to create
@rtype: bool
@returns: True if the directory was created, False otherwise.
"""
getLogger().info(">> makeDirectory(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().mkdir(path)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< makeDirectory(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< makeDirectory(): %s" % str(res))
return res
def createPackage(path):
"""
Creates a new package structure with path as package root folder.
"""
if FileSystemManager.instance().isfile(path):
getLogger().info("Cannot create package as %s: is a file" % path)
raise Exception("Invalid package path: is a file")
if FileSystemManager.instance().isdir(path):
getLogger().info("Cannot create package as %s: path already exist" % path)
raise Exception("Invalid package path: this path already exist")
FileSystemManager.instance().mkdir("%s/profiles" % path, False)
FileSystemManager.instance().mkdir("%s/src" % path, False)
FileSystemManager.instance().write("%s/package.xml" % path, DEFAULT_PACKAGE_DESCRIPTION, notify = False)
FileSystemManager.instance()._notifyDirCreated(path)
return True
def importPackageFile(content, path):
"""
Expand a package file to a docroot folder.
"""
try:
checkPackageFile(content)
except Exception as e:
getLogger().info("Invalid package file: %s" % e)
raise Exception("Invalid package file: %s" % e)
if FileSystemManager.instance().isfile(path):
getLogger().info("Cannot import package to %s: not a path to package" %
path)
raise Exception("Invalid destination package path: is a file")
if FileSystemManager.instance().isdir(path):
getLogger().info(
"Cannot import package from %s: destination path already exist" %
path)
raise Exception(
"Invalid destination package path: this path already exist")
# Minimal package tree
FileSystemManager.instance().mkdir("%s/profiles" % path, False)
FileSystemManager.instance().mkdir("%s/src" % path, False)
# First unpack the package, then notify the package dir creation so that it is seen as a package folder.
tpk = StringIO.StringIO(content)
tfile = tarfile.open("tpk", "r:gz", tpk)
contents = tfile.getmembers()
for c in contents:
# TODO
if c.name.startswith('src/') or c.name.startswith(
'profiles/') or c.name in ['package.xml']:
dst = "%s/%s" % (path, c.name)
if c.isfile():
getLogger().info("Importing %s to %s..." % (c.name, dst))
content = tfile.extractfile(c).read()
FileSystemManager.instance().write(dst, content, notify=False)
else:
getLogger().info("Discarding importation of %s" % c.name)
FileSystemManager.instance()._notifyDirCreated(path)
return True
def createPackageFile(path):
"""
Creates a testerman package file from a docroot package root folder.
"""
def _packageFolder(tfile, relbasepath, docrootbasepath):
getLogger().debug("Walking folder %s..." % docrootbasepath)
for entry in FileSystemManager.instance().getdir(docrootbasepath):
name, apptype = entry['name'], entry['type']
if apptype == FileSystemManager.APPTYPE_DIR:
relpath = "%s%s/" % (relbasepath, name) # the name of the current item within the package
docrootpath = "%s%s/" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding directory %s..." % relpath)
tarinfo = tarfile.TarInfo(relpath)
tarinfo.type = tarfile.DIRTYPE
tarinfo.mode = 0755
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
tfile.addfile(tarinfo)
_packageFolder(tfile, relbasepath = relpath, docrootbasepath = docrootpath)
else:
relname = "%s%s" % (relbasepath, name) # the name of the current item within the package
docrootname = "%s%s" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding file %s..." % relname)
tarinfo = tarfile.TarInfo(relname)
tarinfo.type = tarfile.AREGTYPE
tarinfo.mode = 0644
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
content = FileSystemManager.instance().read(docrootname)
tarinfo.size = len(content)
contentObj = StringIO.StringIO(content)
tfile.addfile(tarinfo, contentObj)
contentObj.close()
getLogger().debug("File %s added to package file (%s bytes)" % (relname, tarinfo.size))
getLogger().info("Creating package file from %s..." % path)
if not FileSystemManager.instance().isdir(path):
getLogger().info("Cannot create package from %s: not a path to package" % path)
return None
tpk = StringIO.StringIO()
tfile = tarfile.open("tpk", "w:gz", tpk)
# Now, traverse the files into path
if not path.endswith('/'):
path = "%s/" % path
_packageFolder(tfile, '', path)
tfile.close()
contents = tpk.getvalue()
tpk.close()
getLogger().info("Package file for %s created, %s bytes" % (path, len(contents)))
return contents
def getDependencies(path, recursive=False):
"""
Computes the file dependencies of the file referenced by path.
If recursive is set to True, also searches for additional dependencies
recursively; otherwise only direct dependencies are computed.
A dependency for an ATS is a module it imports.
A depencendy for a module is a module it imports.
A dependency for a campaign is a a script (ats or campaign) it calls.
This method may be used by a client to create a package.
@since: 1.3
@type path: string
@param path: a docroot path to a module, ats or campaign
@type recursive: boolean
@param recursive: False for direct depencencies only. True for all
dependencies.
@rtype: list of strings
@returns: a list of dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getDependencies(%s, %s)" % (path, recursive))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
res = DependencyResolver.python_getDependencyFilenames(
source, path, recursive)
elif path.endswith('.ats'):
res = DependencyResolver.python_getDependencyFilenames(
source, path, recursive)
elif path.endswith('.campaign'):
res = DependencyResolver.campaign_getDependencyFilenames(
source,
os.path.split(path)[0], recursive, path)
else:
raise Exception(
'Unsupported file format, cannot resolve dependencies')
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getDependencies(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< getDependencies(): %s" % str(res))
return res
def getReverseDependencies(path):
"""
Computes the reverse file dependencies of the file referenced by path,
i.e. the list of files in the repository that reference this path (which
is typically a module - campaigns referencing an ats or a campaign won't
be retrieved for now).
A reverse dependency for a module is another module or ATS that imports it.
This method may be used by a client to check if a module is currently in use
or not.
Only reverse dependencies at call time are searched - if older revisions
of files reference this module, it won't be checked.
@type path: string
@param path: a docroot path to a module
@rtype: list of strings
@returns: a list of reverse dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getReverseDependencies(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
# Well, we need to scan all our python and ATS files into our repository (the candidate)
# do a:
# candidateSource = getFile(candidatePath)
# try:
# deps = DependencyResolver.python_getDependencyFilenames(candidateSource, candidatePath, recursive = False)
# except:
# deps = []
# if path in deps and candidatePath not in res:
# res.append(candidatePath)
res = []
else:
# Reverse dependencies is not supported on something that is not a module
res = []
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getReverseDependencies(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< getReverseDependencies(): %s" % str(res))
return res
def getReverseDependencies(path):
"""
Computes the reverse file dependencies of the file referenced by path,
i.e. the list of files in the repository that reference this path (which
is typically a module - campaigns referencing an ats or a campaign won't
be retrieved for now).
A reverse dependency for a module is another module or ATS that imports it.
This method may be used by a client to check if a module is currently in use
or not.
Only reverse dependencies at call time are searched - if older revisions
of files reference this module, it won't be checked.
@type path: string
@param path: a docroot path to a module
@rtype: list of strings
@returns: a list of reverse dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getReverseDependencies(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
# Well, we need to scan all our python and ATS files into our repository (the candidate)
# do a:
# candidateSource = getFile(candidatePath)
# try:
# deps = DependencyResolver.python_getDependencyFilenames(candidateSource, candidatePath, recursive = False)
# except:
# deps = []
# if path in deps and candidatePath not in res:
# res.append(candidatePath)
res = []
else:
# Reverse dependencies is not supported on something that is not a module
res = []
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getReverseDependencies(...): Fault:\n" + str(e))
raise(e)
def schedulePackage(path, username, session, at, script = None, profileName = None):
"""
Schedules a package to start at <at>.
If script is provided, the package's default-script attribute is ignored
and script is used instead.
@since: 1.3
"""
getLogger().info(">> schedulePackage(%s, script = %s)" % (path, script))
if not path.startswith('/'): path = '/' + path
try:
metadata = Package.getPackageMetadata(path)
if not script:
script = metadata['default-script']
scriptFilename = "%s/src/%s" % (path, script)
scriptPath = os.path.split(scriptFilename)[0]
label = "%s:%s" % ('/'.join(path.split('/')[2:]), script)
getLogger().info("Using package script filename %s, path %s" % (scriptFilename, scriptPath))
if script.endswith(".campaign"):
res = scheduleCampaign(FileSystemManager.instance().read(scriptFilename).decode('utf-8'), label, username, session, at = at, path = scriptPath)
elif script.endswith(".ats"):
res = scheduleAts(FileSystemManager.instance().read(scriptFilename).decode('utf-8'), label, username, session, at = at, path = scriptPath)
else:
raise Exception("Invalid script/default script for package execution (unrecognized job type based on extension - %s)" % script)
except Exception, e:
e = Exception("Scheduling error: %s" % (str(e)))
getLogger().info("<< schedulePackage(...): Fault:\n%s" % Tools.getBacktrace())
raise(e)
def move(source, destination):
"""
Moves a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be moved.
Logs associated to a scripts, if any, are
NOT moved. They are kept available in the archives,
but not associated to the script any more.
FIXME: Revisions should be moved, however.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> move(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().move(source, destination)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< move(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< move(): %s" % str(res))
return res
def getDependencies(path, recursive = False):
"""
Computes the file dependencies of the file referenced by path.
If recursive is set to True, also searches for additional dependencies
recursively; otherwise only direct dependencies are computed.
A dependency for an ATS is a module it imports.
A depencendy for a module is a module it imports.
A dependency for a campaign is a a script (ats or campaign) it calls.
This method may be used by a client to create a package.
@since: 1.3
@type path: string
@param path: a docroot path to a module, ats or campaign
@type recursive: boolean
@param recursive: False for direct depencencies only. True for all
dependencies.
@rtype: list of strings
@returns: a list of dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getDependencies(%s, %s)" % (path, recursive))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
res = DependencyResolver.python_getDependencyFilenames(source, path, recursive)
elif path.endswith('.ats'):
res = DependencyResolver.python_getDependencyFilenames(source, path, recursive)
elif path.endswith('.campaign'):
res = DependencyResolver.campaign_getDependencyFilenames(source, os.path.split(path)[0], recursive, path)
else:
raise Exception('Unsupported file format, cannot resolve dependencies')
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getDependencies(...): Fault:\n" + str(e))
raise(e)
def getFile(path, useCompression=False):
"""
Retrieves a file according to the path.
The path is relative to the document root.
If useCompression is set, compress the output before encoding it to mime64.
@since: 1.0
@type path: string
@param path: a path to a file
@type useCompression: bool
@param useCompression: if True, the output is gziped before being mime64-encoded
@rtype: string (utf-8 or buffer, encoded in base64), or None
@returns: None if the file was not found,
or the file contents in base64 encoding, optionally compressed
"""
getLogger().info(">> getFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
ret = None
try:
contents = FileSystemManager.instance().read(path)
if contents is None:
ret = None
else:
if useCompression:
ret = base64.encodestring(zlib.compress(contents))
else:
ret = base64.encodestring(contents)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getFile(...): Fault:\n%s" % str(e))
ret = None
if ret is not None:
getLogger().info("<< getFile(%s): %d bytes returned" %
(path, len(ret)))
else:
getLogger().info("<< getFile(%s): file not found" % (path))
return ret
def move(source, destination):
"""
Moves a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be moved.
Logs associated to a scripts, if any, are
NOT moved. They are kept available in the archives,
but not associated to the script any more.
FIXME: Revisions should be moved, however.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> move(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().move(source, destination)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< move(...): Fault:\n" + str(e))
raise(e)
def copy(source, destination):
"""
Copies a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be copied.
Meta children (Revisions and logs, if any) are NOT copied.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> copy(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().copy(source, destination)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< copy(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< copy(): %s" % str(res))
return res
def putFile(content, path, useCompression=False, username=None):
"""
Writes a file to docroot/path
@since: 1.0
@type content: utf-8 encoded (or buffer) string, encoded in mime64
@param content: the content of the file
@type path: string
@param path: a complete path, with filename and extension, relative to the document root.
@type useCompression: bool
@param useCompression: (since 1.3) if set to True, the content is gziped before being mime64-encoded.
@type username: string
@param username: (since 1.7) the committer/writer
@rtype: bool
@returns: True if OK, False otherwise
"""
getLogger().info(">> putFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
res = False
try:
content = base64.decodestring(content)
if useCompression:
content = zlib.decompress(content)
revision = FileSystemManager.instance().write(path,
content,
username=username)
# No revision handling for now
# We should return the new filepath in case of a success
# /repository/samples/[email protected]
# etc
res = True
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< putFile(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< putFile(): %s" % str(res))
return res
def getPackageMetadata(path):
"""
Extracts the different package metadata contained into the package.xml root folder.
@type path: string
@param path: the docroot path to the package's root folder
@rtype: dict[string] of unicode string
@returns: the metadata, as a dict containing the following keys:
author, description, default-script, status
"""
descriptionFile = "%s/package.xml" % path
try:
content = FileSystemManager.instance().read(descriptionFile)
if content is None:
raise Exception("Unable to read the package description file for package %s" % path)
# Now, parse the document
return parsePackageDescription(content)
except Exception, e:
raise e
def copy(source, destination):
"""
Copies a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be copied.
Meta children (Revisions and logs, if any) are NOT copied.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> copy(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().copy(source, destination)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< copy(...): Fault:\n" + str(e))
raise(e)
def makeDirectory(path):
"""
Creates a directory and all the needed directories to it, if any.
@since: 1.3
@type path: string
@param path: the docroot-path to the directory to create
@rtype: bool
@returns: True if the directory was created, False otherwise.
"""
getLogger().info(">> makeDirectory(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().mkdir(path)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< makeDirectory(...): Fault:\n" + str(e))
raise(e)
def removeFile(path):
"""
Removes a file.
@since: 1.0
@type path: string
@param path: the docroot-path to the file to delete
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeFile(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = False
try:
res = FileSystemManager.instance().unlink(path)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< removeFile(...): Fault:\n" + str(e))
raise(e)
def putFile(content, path, useCompression = False, username = None):
"""
Writes a file to docroot/path
@since: 1.0
@type content: utf-8 encoded (or buffer) string, encoded in mime64
@param content: the content of the file
@type path: string
@param path: a complete path, with filename and extension, relative to the document root.
@type useCompression: bool
@param useCompression: (since 1.3) if set to True, the content is gziped before being mime64-encoded.
@type username: string
@param username: (since 1.7) the committer/writer
@rtype: bool
@returns: True if OK, False otherwise
"""
getLogger().info(">> putFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
res = False
try:
content = base64.decodestring(content)
if useCompression:
content = zlib.decompress(content)
revision = FileSystemManager.instance().write(path, content, username = username)
# No revision handling for now
# We should return the new filepath in case of a success
# /repository/samples/[email protected]
# etc
res = True
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< putFile(...): Fault:\n" + str(e))
raise(e)
def getFile(path, useCompression = False):
"""
Retrieves a file according to the path.
The path is relative to the document root.
If useCompression is set, compress the output before encoding it to mime64.
@since: 1.0
@type path: string
@param path: a path to a file
@type useCompression: bool
@param useCompression: if True, the output is gziped before being mime64-encoded
@rtype: string (utf-8 or buffer, encoded in base64), or None
@returns: None if the file was not found,
or the file contents in base64 encoding, optionally compressed
"""
getLogger().info(">> getFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
ret = None
try:
contents = FileSystemManager.instance().read(path)
if contents is None:
ret = None
else:
if useCompression:
ret = base64.encodestring(zlib.compress(contents))
else:
ret = base64.encodestring(contents)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getFile(...): Fault:\n%s" % str(e))
ret = None
def removeDirectory(path, recursive = False):
"""
Removes an empty directory, unless recursive is set to True.
@since: 1.1
@type path: string
@param path: the docroot-path to the directory to delete
@type recursive: bool
@param recursive: True if we should delete files and directories in it. DANGEROUS.
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeDirectory(%s)" % path)
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rmdir(path, recursive)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< removeDirectory(...): Fault:\n" + str(e))
raise(e)
def campaign_getDependencyFilenames(source, sourcePath = None, recursive = False, sourceFilename = "<local>", moduleRootDir = '/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the campaign depends on.
Calls pythong_getDependencyFilenames on ats files if recursive is True,
and thus only userland dependencies are searched.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourcePath: string
@param sourcePath: docroot path used as a 'working dir' to search for
relative dependencies. Typically the path where the source has been extracted
from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type sourceFilename: string
@param sourceFilename: a string to help identify the file that yielded the source.
For non-repository files, use <local>, by convention.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
currentDependencies = []
getLogger().info("%s: parsing campaign file to look for its dependencies" % sourceFilename)
# The path of the campaign within the docroot.
path = sourcePath
# Based on a file parsing.
indent = 0
parentPresent = True
lc = 0
for line in source.splitlines():
lc += 1
# Remove comments
line = line.split('#', 1)[0].rstrip()
if not line:
continue # empty line
m = re.match(r'(?P<indent>\s*)((?P<branch>\w+|\*)\s+)?(?P<type>\w+)\s+(?P<filename>[^\s]+)(\s+with\s+(?P<mapping>.*)\s*)?', line)
if not m:
raise Exception('%s: parse error at line %s: invalid line format' % (sourceFilename, lc))
type_ = m.group('type')
filename = m.group('filename')
indentDiff = len(m.group('indent')) - indent
indent = indent + indentDiff
# Filename creation within the docroot
if filename.startswith('/'):
# absolute path within the *repository*
filename = '/%s%s' % (cm.get_transient('constants.repository'), filename)
else:
# just add the local campaign path
filename = '%s/%s' % (path, filename)
# Type validation
if not type_ in [ 'ats', 'campaign' ]:
raise Exception('%s: error at line %s: invalid job type (%s)' % (sourceFilename, lc, type_))
# Indentation validation
if indentDiff > 1:
raise Exception('%s: parse error at line %s: invalid indentation (too deep)' % (sourceFilename, lc))
elif indentDiff == 1:
if not parentPresent:
raise Exception('%s: parse error at line %s: invalid indentation (invalid initial indentation)' % (sourceFilename, lc))
elif indentDiff == 0:
# the current parent remains the parent
pass
else:
# negative indentation.
pass
# OK, now we have at least one parent.
parentPresent = True
# Branch validation: ignored for dependency resolver
# Now handle the dependency.
if not filename in currentDependencies:
getLogger().info('%s: campaign direct dependency added: %s' % (sourceFilename, filename))
currentDependencies.append(filename)
if recursive and type_ in ['ats', 'campaign']:
nextDependencies = []
nextPath, nextFilename = os.path.split(filename)
nextSource = FileSystemManager.instance().read(filename)
if nextSource is None:
raise Exception('%s: missing dependency: file %s is not in the repository' % (sourceFilename, filename))
if type_ == 'campaign':
nextDependencies = campaign_getDependencyFilenames(nextSource, nextPath, True, filename, moduleRootDir = moduleRootDir)
elif type_ == 'ats':
nextDependencies = python_getDependencyFilenames(nextSource, sourceFilename = filename, recursive = True, moduleRootDir = moduleRootDir)
for dep in nextDependencies:
if not dep in currentDependencies:
getLogger().info('%s: campaign indirect dependency added: %s' % (sourceFilename, dep))
currentDependencies.append(dep)
return currentDependencies
def campaign_getDependencyFilenames(source,
sourcePath=None,
recursive=False,
sourceFilename="<local>",
moduleRootDir='/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the campaign depends on.
Calls pythong_getDependencyFilenames on ats files if recursive is True,
and thus only userland dependencies are searched.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourcePath: string
@param sourcePath: docroot path used as a 'working dir' to search for
relative dependencies. Typically the path where the source has been extracted
from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type sourceFilename: string
@param sourceFilename: a string to help identify the file that yielded the source.
For non-repository files, use <local>, by convention.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
currentDependencies = []
getLogger().info("%s: parsing campaign file to look for its dependencies" %
sourceFilename)
# The path of the campaign within the docroot.
path = sourcePath
# Based on a file parsing.
indent = 0
parentPresent = True
lc = 0
for line in source.splitlines():
lc += 1
# Remove comments
line = line.split('#', 1)[0].rstrip()
if not line:
continue # empty line
m = re.match(
r'(?P<indent>\s*)((?P<branch>\w+|\*)\s+)?(?P<type>\w+)\s+(?P<filename>[^\s]+)(\s+with\s+(?P<mapping>.*)\s*)?',
line)
if not m:
raise Exception('%s: parse error at line %s: invalid line format' %
(sourceFilename, lc))
type_ = m.group('type')
filename = m.group('filename')
indentDiff = len(m.group('indent')) - indent
indent = indent + indentDiff
# Filename creation within the docroot
if filename.startswith('/'):
# absolute path within the *repository*
filename = '/%s%s' % (cm.get_transient('constants.repository'),
filename)
else:
# just add the local campaign path
filename = '%s/%s' % (path, filename)
# Type validation
if not type_ in ['ats', 'campaign']:
raise Exception('%s: error at line %s: invalid job type (%s)' %
(sourceFilename, lc, type_))
# Indentation validation
if indentDiff > 1:
raise Exception(
'%s: parse error at line %s: invalid indentation (too deep)' %
(sourceFilename, lc))
elif indentDiff == 1:
if not parentPresent:
raise Exception(
'%s: parse error at line %s: invalid indentation (invalid initial indentation)'
% (sourceFilename, lc))
elif indentDiff == 0:
# the current parent remains the parent
pass
else:
# negative indentation.
pass
# OK, now we have at least one parent.
parentPresent = True
# Branch validation: ignored for dependency resolver
# Now handle the dependency.
if not filename in currentDependencies:
getLogger().info('%s: campaign direct dependency added: %s' %
(sourceFilename, filename))
currentDependencies.append(filename)
if recursive and type_ in ['ats', 'campaign']:
nextDependencies = []
nextPath, nextFilename = os.path.split(filename)
nextSource = FileSystemManager.instance().read(filename)
if nextSource is None:
raise Exception(
'%s: missing dependency: file %s is not in the repository'
% (sourceFilename, filename))
if type_ == 'campaign':
nextDependencies = campaign_getDependencyFilenames(
nextSource,
nextPath,
True,
filename,
moduleRootDir=moduleRootDir)
elif type_ == 'ats':
nextDependencies = python_getDependencyFilenames(
nextSource,
sourceFilename=filename,
recursive=True,
moduleRootDir=moduleRootDir)
for dep in nextDependencies:
if not dep in currentDependencies:
getLogger().info(
'%s: campaign indirect dependency added: %s' %
(sourceFilename, dep))
currentDependencies.append(dep)
return currentDependencies
def python_getDependencyFilenames(source, sourceFilename, recursive = True, moduleRootDir = '/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the ATS/module depends on.
Only userland dependencies are searched, i.e. modules below $docroot/repository.
Non-userland dependencies, such as Testerman and Python std modules,
are not reported.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourceFilename: string
@param sourceFilename: the full docroot path to the source file.
For non-repository files, should be set to something like /repository/<anonymous.ats>
so that we can deduce the docroot "working dir" dependencies should be searched from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
getLogger().info("Resolving dependencies for file %s" % sourceFilename)
# This is the PYTHONPATH that will be used to run the TE.
# Will be used as a fallback if an import cannot be found in userland (ie in the repo)
pythonPath = '%(root)s/modules' % dict(
root = cm.get_transient("ts.server_root"))
additionalPythonPath = cm.get("testerman.te.python.additional_pythonpath")
if additionalPythonPath:
pythonPath += ':' + additionalPythonPath
pythonPath = pythonPath.split(':')
# Will only include userland dependencies. System dependencies found in PYTHONPATH won't be included.
ret = []
# Bootstrap the deps (stored as (list of imported modules, path of the importing file) )
toResolve = [ (d, sourceFilename) for d in python_getImportedUserlandModules(source, sourceFilename = sourceFilename) ]
# the list of resolved dependencies,
# as a map (import, fromFilename): resolvedFilename
resolvedSoFar = {}
# For each deps to resolve (a list of (import, fromFilename)),
# we need to resolve the filename that will provide this import for this file.
while len(toResolve):
getLogger().debug("List of imports to resolve for script %s:\n%s" % (sourceFilename, "\n".join(["%s (used in %s)" % x for x in toResolve])))
dep, fromFilename = toResolve.pop()
# Some non-userland files - not resolved to build the TE userland package
# How can we detect standard Python includes ?
# fromFilePath starts with the "python home" ? something else ?
getLogger().debug("Resolving import %s from %s..." % (dep, fromFilename))
# Skip some dependencies provided by the Testerman infrastructure
#if dep in [ ]:
# getLogger().info("Resolving import %s from %: skipped, not userland" % (dep, fromFilename))
# continue
# Skip already resolved dependencies
if (dep, fromFilename) in resolvedSoFar:
getLogger().debug("Resolving import %s from %s: already resolved as %s" % (dep, fromFilename, resolvedSoFar[(dep, fromFilename)]))
continue
# Ordered list of filenames within the docroot that could provide the dependency:
# (module path)
# - first search from the local file path, if provided,
# - then search from the userland module paths (limited to '/repository/' for now)
modulePaths = []
# First, try a local module (relative path) (same dir as the currently analyzed file)
sourcePath = os.path.split(fromFilename)[0]
modulePaths.append(sourcePath)
# Then fall back to standard "testerman userland paths"
# If the filename was in a package, look from the package's root dir
# Otherwise this the repository root.
for modulePath in [ moduleRootDir ]:
if modulePath and not modulePath in modulePaths:
modulePaths.append(modulePath)
getLogger().debug("Resolving import %s from %s: searching in paths:\n%s" % (dep, fromFilename, "\n".join(modulePaths)))
found = None
depSource = None
for path in modulePaths:
depFilename = '%s/%s.py' % (path, dep.replace('.', '/'))
try:
depSource = FileSystemManager.instance().read(depFilename)
except Exception:
pass
if depSource is not None:
found = depFilename
break
if not found:
getLogger().debug("Resolving import %s from %s: not available in repository, searched in paths:\n%s" % (dep, fromFilename, "\n".join(modulePaths)))
try:
imp.find_module(dep, pythonPath)
except:
getLogger().debug("Resolving import %s from %s: not available in PYTHONPATH either, searched paths:\n%s" % (dep, fromFilename, "\n".join(pythonPath)))
raise Exception('Missing module: %s (imported from %s) is not available in userland (repository, searched paths: %s) or in TE PYTHONPATH (searched paths: %s)' % (dep, fromFilename, modulePaths, pythonPath))
getLogger().debug("Resolving import %s from %s: OK, found in PYTHONPATH, won't be included in userland dependencies")
if found:
# OK, we resolved a file.
resolvedSoFar[(dep, fromFilename)] = depFilename
getLogger().debug("Resolving import %s from %s: resolved as %s" % (dep, fromFilename, depFilename))
if not depFilename in ret:
ret.append(depFilename)
getLogger().debug("Script %s is now using the following files:\n%s" % (sourceFilename, "\n".join(ret)))
# Now, analyze the resolved file and add its own dependencies to the list to resolve,
# if not already resolved
if recursive:
importedModules = python_getImportedUserlandModules(depSource, depFilename)
for im in importedModules:
if not (im, depFilename) in resolvedSoFar:
toResolve.append((im, depFilename))
else:
getLogger().debug("Resolving import %s from %s: already resolved" % (im, depFilename))
return ret
raise Exception("Missing profiles folder (profiles)")
# Now we should check the package.xml file, too
return True
def importPackageFile(content, path):
"""
Expand a package file to a docroot folder.
"""
try:
checkPackageFile(content)
except Exception, e:
getLogger().info("Invalid package file: %s" % e)
raise Exception("Invalid package file: %s" % e)
if FileSystemManager.instance().isfile(path):
getLogger().info("Cannot import package to %s: not a path to package" % path)
raise Exception("Invalid destination package path: is a file")
if FileSystemManager.instance().isdir(path):
getLogger().info("Cannot import package from %s: destination path already exist" % path)
raise Exception("Invalid destination package path: this path already exist")
# Minimal package tree
FileSystemManager.instance().mkdir("%s/profiles" % path, False)
FileSystemManager.instance().mkdir("%s/src" % path, False)
# First unpack the package, then notify the package dir creation so that it is seen as a package folder.
tpk = StringIO.StringIO(content)
tfile = tarfile.open("tpk", "r:gz", tpk)
def python_getDependencyFilenames(source,
sourceFilename,
recursive=True,
moduleRootDir='/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the ATS/module depends on.
Only userland dependencies are searched, i.e. modules below $docroot/repository.
Non-userland dependencies, such as Testerman and Python std modules,
are not reported.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourceFilename: string
@param sourceFilename: the full docroot path to the source file.
For non-repository files, should be set to something like /repository/<anonymous.ats>
so that we can deduce the docroot "working dir" dependencies should be searched from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
getLogger().info("Resolving dependencies for file %s" % sourceFilename)
# This is the PYTHONPATH that will be used to run the TE.
# Will be used as a fallback if an import cannot be found in userland (ie in the repo)
pythonPath = '%(root)s/modules' % dict(
root=cm.get_transient("ts.server_root"))
additionalPythonPath = cm.get("testerman.te.python.additional_pythonpath")
if additionalPythonPath:
pythonPath += ':' + additionalPythonPath
pythonPath = pythonPath.split(':')
# Will only include userland dependencies. System dependencies found in PYTHONPATH won't be included.
ret = []
# Bootstrap the deps (stored as (list of imported modules, path of the importing file) )
toResolve = [(d, sourceFilename)
for d in python_getImportedUserlandModules(
source, sourceFilename=sourceFilename)]
# the list of resolved dependencies,
# as a map (import, fromFilename): resolvedFilename
resolvedSoFar = {}
# For each deps to resolve (a list of (import, fromFilename)),
# we need to resolve the filename that will provide this import for this file.
while len(toResolve):
getLogger().debug("List of imports to resolve for script %s:\n%s" %
(sourceFilename, "\n".join(
["%s (used in %s)" % x for x in toResolve])))
dep, fromFilename = toResolve.pop()
# Some non-userland files - not resolved to build the TE userland package
# How can we detect standard Python includes ?
# fromFilePath starts with the "python home" ? something else ?
getLogger().debug("Resolving import %s from %s..." %
(dep, fromFilename))
# Skip some dependencies provided by the Testerman infrastructure
#if dep in [ ]:
# getLogger().info("Resolving import %s from %: skipped, not userland" % (dep, fromFilename))
# continue
# Skip already resolved dependencies
if (dep, fromFilename) in resolvedSoFar:
getLogger().debug(
"Resolving import %s from %s: already resolved as %s" %
(dep, fromFilename, resolvedSoFar[(dep, fromFilename)]))
continue
# Ordered list of filenames within the docroot that could provide the dependency:
# (module path)
# - first search from the local file path, if provided,
# - then search from the userland module paths (limited to '/repository/' for now)
modulePaths = []
# First, try a local module (relative path) (same dir as the currently analyzed file)
sourcePath = os.path.split(fromFilename)[0]
modulePaths.append(sourcePath)
# Then fall back to standard "testerman userland paths"
# If the filename was in a package, look from the package's root dir
# Otherwise this the repository root.
for modulePath in [moduleRootDir]:
if modulePath and not modulePath in modulePaths:
modulePaths.append(modulePath)
getLogger().debug(
"Resolving import %s from %s: searching in paths:\n%s" %
(dep, fromFilename, "\n".join(modulePaths)))
found = None
depSource = None
for path in modulePaths:
depFilename = '%s/%s.py' % (path, dep.replace('.', '/'))
try:
depSource = FileSystemManager.instance().read(depFilename)
except Exception:
pass
if depSource is not None:
found = depFilename
break
if not found:
getLogger().debug(
"Resolving import %s from %s: not available in repository, searched in paths:\n%s"
% (dep, fromFilename, "\n".join(modulePaths)))
try:
imp.find_module(dep, pythonPath)
except:
getLogger().debug(
"Resolving import %s from %s: not available in PYTHONPATH either, searched paths:\n%s"
% (dep, fromFilename, "\n".join(pythonPath)))
raise Exception(
'Missing module: %s (imported from %s) is not available in userland (repository, searched paths: %s) or in TE PYTHONPATH (searched paths: %s)'
% (dep, fromFilename, modulePaths, pythonPath))
getLogger().debug(
"Resolving import %s from %s: OK, found in PYTHONPATH, won't be included in userland dependencies"
)
if found:
# OK, we resolved a file.
resolvedSoFar[(dep, fromFilename)] = depFilename
getLogger().debug("Resolving import %s from %s: resolved as %s" %
(dep, fromFilename, depFilename))
if not depFilename in ret:
ret.append(depFilename)
getLogger().debug(
"Script %s is now using the following files:\n%s" %
(sourceFilename, "\n".join(ret)))
# Now, analyze the resolved file and add its own dependencies to the list to resolve,
# if not already resolved
if recursive:
importedModules = python_getImportedUserlandModules(
depSource, depFilename)
for im in importedModules:
if not (im, depFilename) in resolvedSoFar:
toResolve.append((im, depFilename))
else:
getLogger().debug(
"Resolving import %s from %s: already resolved" %
(im, depFilename))
return ret