def schedulePackage(path,
username,
session,
at,
script=None,
profileName=None):
"""
Schedules a package to start at <at>.
If script is provided, the package's default-script attribute is ignored
and script is used instead.
@since: 1.3
"""
getLogger().info(">> schedulePackage(%s, script = %s)" % (path, script))
if not path.startswith('/'): path = '/' + path
try:
metadata = Package.getPackageMetadata(path)
if not script:
script = metadata['default-script']
scriptFilename = "%s/src/%s" % (path, script)
scriptPath = scriptFilename
label = "%s:%s" % ('/'.join(path.split('/')[2:]), script)
getLogger().info("Using package script filename %s, path %s" %
(scriptFilename, scriptPath))
if script.endswith(".campaign"):
res = scheduleCampaign(FileSystemManager.instance().read(
scriptFilename).decode('utf-8'),
label,
username,
session,
at=at,
path=scriptPath)
elif script.endswith(".ats"):
res = scheduleAts(FileSystemManager.instance().read(
scriptFilename).decode('utf-8'),
label,
username,
session,
at=at,
path=scriptPath)
else:
raise Exception(
"Invalid script/default script for package execution (unrecognized job type based on extension - %s)"
% script)
except Exception as e:
e = Exception("Scheduling error: %s" % (str(e)))
getLogger().info("<< schedulePackage(...): Fault:\n%s" %
Tools.getBacktrace())
raise (e)
getLogger().info("<< schedulePackage(...): %s" % str(res))
return res
def _packageFolder(tfile, relbasepath, docrootbasepath):
getLogger().debug("Walking folder %s..." % docrootbasepath)
for entry in FileSystemManager.instance().getdir(docrootbasepath):
name, apptype = entry['name'], entry['type']
if apptype == FileSystemManager.APPTYPE_DIR:
relpath = "%s%s/" % (relbasepath, name) # the name of the current item within the package
docrootpath = "%s%s/" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding directory %s..." % relpath)
tarinfo = tarfile.TarInfo(relpath)
tarinfo.type = tarfile.DIRTYPE
tarinfo.mode = 0755
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
tfile.addfile(tarinfo)
_packageFolder(tfile, relbasepath = relpath, docrootbasepath = docrootpath)
else:
relname = "%s%s" % (relbasepath, name) # the name of the current item within the package
docrootname = "%s%s" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding file %s..." % relname)
tarinfo = tarfile.TarInfo(relname)
tarinfo.type = tarfile.AREGTYPE
tarinfo.mode = 0644
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
content = FileSystemManager.instance().read(docrootname)
tarinfo.size = len(content)
contentObj = StringIO.StringIO(content)
tfile.addfile(tarinfo, contentObj)
contentObj.close()
getLogger().debug("File %s added to package file (%s bytes)" % (relname, tarinfo.size))
def removeFile(path):
"""
Removes a file.
@since: 1.0
@type path: string
@param path: the docroot-path to the file to delete
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeFile(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = False
try:
res = FileSystemManager.instance().unlink(path)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< removeFile(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< removeFile(): %s" % str(res))
return res
def removeDirectory(path, recursive=False):
"""
Removes an empty directory, unless recursive is set to True.
@since: 1.1
@type path: string
@param path: the docroot-path to the directory to delete
@type recursive: bool
@param recursive: True if we should delete files and directories in it. DANGEROUS.
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeDirectory(%s)" % path)
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rmdir(path, recursive)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< removeDirectory(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< removeDirectory(): %s" % str(res))
return res
def getDirectoryListing(path):
"""
Returns the contents of a directory.
Also filters some 'internal' files (in particular __init__.py files)
@since: 1.0
@type path: string
@param path: the path of the directory within the docroot
@rtype: list of dict{'name': string, 'type': string in [ ats, campaign, module, log, directory, package, ... ] }
@returns: the dir contents, with a name (with extension) relative to the dir,
and an associated application type.
Returns None if the directory was not accessible or in case of an error.
"""
getLogger().info(">> getDirectoryListing(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = []
try:
res = FileSystemManager.instance().getdir(path)
if res is None:
raise Exception("Unable to get directory contents through backend")
# Sort the entries, so that it is useless to implement it in all clients ?
res.sort(key=operator.itemgetter('name'))
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getDirectoryListing(...): Fault:\n%s" % str(e))
# Well, actually, we do not return a fault in this case...
res = None
def makeDirectory(path):
"""
Creates a directory and all the needed directories to it, if any.
@since: 1.3
@type path: string
@param path: the docroot-path to the directory to create
@rtype: bool
@returns: True if the directory was created, False otherwise.
"""
getLogger().info(">> makeDirectory(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().mkdir(path)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< makeDirectory(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< makeDirectory(): %s" % str(res))
return res
def getDirectoryListing(path):
"""
Returns the contents of a directory.
Also filters some 'internal' files (in particular __init__.py files)
@since: 1.0
@type path: string
@param path: the path of the directory within the docroot
@rtype: list of dict{'name': string, 'type': string in [ ats, campaign, module, log, directory, package, ... ] }
@returns: the dir contents, with a name (with extension) relative to the dir,
and an associated application type.
Returns None if the directory was not accessible or in case of an error.
"""
getLogger().info(">> getDirectoryListing(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = []
try:
res = FileSystemManager.instance().getdir(path)
if res is None:
raise Exception("Unable to get directory contents through backend")
# Sort the entries, so that it is useless to implement it in all clients ?
res.sort(key = operator.itemgetter('name'))
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getDirectoryListing(...): Fault:\n%s" % str(e))
# Well, actually, we do not return a fault in this case...
res = None
def getFileInfo(path):
"""
Gets some info about a file.
Returns a dict{ 'size': integer, 'timestamp': float }
where the size is optional (in bytes, if provided), and timestamp is
the file modification time.
@since: 1.0
@type path: string
@param path: the path to the file whose info we want to get
@rtype: a dict, or None
@returns: None on error, or the dict of attributes.
"""
getLogger().info(">> getFileInfo(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = None
try:
attributes = FileSystemManager.instance().attributes(path)
if attributes:
res = {}
if attributes.size is not None:
res['size'] = attributes.size
if attributes.mtime is not None:
res['timestamp'] = attributes.mtime
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getFileInfo(...): Fault:\n" + str(e))
raise(e)
def rename(path, newName):
"""
Renames a file or a directory to newName, in the same folder.
@since: 1.3
@type path: string
@param path: the docroot-path to the object to rename
@type newName: string
@param newName: the new name (basename) of the object, including extension,
if applicable.
@rtype: bool
@returns: False if newName already exists. True otherwise.
"""
getLogger().info(">> rename(%s, %s)" % (path, newName))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rename(path, newName)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< rename(...): Fault:\n" + str(e))
raise(e)
def getFileInfo(path):
"""
Gets some info about a file.
Returns a dict{ 'size': integer, 'timestamp': float }
where the size is optional (in bytes, if provided), and timestamp is
the file modification time.
@since: 1.0
@type path: string
@param path: the path to the file whose info we want to get
@rtype: a dict, or None
@returns: None on error, or the dict of attributes.
"""
getLogger().info(">> getFileInfo(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = None
try:
attributes = FileSystemManager.instance().attributes(path)
if attributes:
res = {}
if attributes.size is not None:
res['size'] = attributes.size
if attributes.mtime is not None:
res['timestamp'] = attributes.mtime
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getFileInfo(...): Fault:\n" + str(e))
raise (e)
def rename(path, newName):
"""
Renames a file or a directory to newName, in the same folder.
@since: 1.3
@type path: string
@param path: the docroot-path to the object to rename
@type newName: string
@param newName: the new name (basename) of the object, including extension,
if applicable.
@rtype: bool
@returns: False if newName already exists. True otherwise.
"""
getLogger().info(">> rename(%s, %s)" % (path, newName))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rename(path, newName)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< rename(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< rename(): %s" % str(res))
return res
def createPackage(path):
"""
Creates a new package structure with path as package root folder.
"""
if FileSystemManager.instance().isfile(path):
getLogger().info("Cannot create package as %s: is a file" % path)
raise Exception("Invalid package path: is a file")
if FileSystemManager.instance().isdir(path):
getLogger().info("Cannot create package as %s: path already exist" % path)
raise Exception("Invalid package path: this path already exist")
FileSystemManager.instance().mkdir("%s/profiles" % path, False)
FileSystemManager.instance().mkdir("%s/src" % path, False)
FileSystemManager.instance().write("%s/package.xml" % path, DEFAULT_PACKAGE_DESCRIPTION, notify = False)
FileSystemManager.instance()._notifyDirCreated(path)
return True
def importPackageFile(content, path):
"""
Expand a package file to a docroot folder.
"""
try:
checkPackageFile(content)
except Exception as e:
getLogger().info("Invalid package file: %s" % e)
raise Exception("Invalid package file: %s" % e)
if FileSystemManager.instance().isfile(path):
getLogger().info("Cannot import package to %s: not a path to package" %
path)
raise Exception("Invalid destination package path: is a file")
if FileSystemManager.instance().isdir(path):
getLogger().info(
"Cannot import package from %s: destination path already exist" %
path)
raise Exception(
"Invalid destination package path: this path already exist")
# Minimal package tree
FileSystemManager.instance().mkdir("%s/profiles" % path, False)
FileSystemManager.instance().mkdir("%s/src" % path, False)
# First unpack the package, then notify the package dir creation so that it is seen as a package folder.
tpk = StringIO.StringIO(content)
tfile = tarfile.open("tpk", "r:gz", tpk)
contents = tfile.getmembers()
for c in contents:
# TODO
if c.name.startswith('src/') or c.name.startswith(
'profiles/') or c.name in ['package.xml']:
dst = "%s/%s" % (path, c.name)
if c.isfile():
getLogger().info("Importing %s to %s..." % (c.name, dst))
content = tfile.extractfile(c).read()
FileSystemManager.instance().write(dst, content, notify=False)
else:
getLogger().info("Discarding importation of %s" % c.name)
FileSystemManager.instance()._notifyDirCreated(path)
return True
def getDependencies(path, recursive=False):
"""
Computes the file dependencies of the file referenced by path.
If recursive is set to True, also searches for additional dependencies
recursively; otherwise only direct dependencies are computed.
A dependency for an ATS is a module it imports.
A depencendy for a module is a module it imports.
A dependency for a campaign is a a script (ats or campaign) it calls.
This method may be used by a client to create a package.
@since: 1.3
@type path: string
@param path: a docroot path to a module, ats or campaign
@type recursive: boolean
@param recursive: False for direct depencencies only. True for all
dependencies.
@rtype: list of strings
@returns: a list of dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getDependencies(%s, %s)" % (path, recursive))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
res = DependencyResolver.python_getDependencyFilenames(
source, path, recursive)
elif path.endswith('.ats'):
res = DependencyResolver.python_getDependencyFilenames(
source, path, recursive)
elif path.endswith('.campaign'):
res = DependencyResolver.campaign_getDependencyFilenames(
source,
os.path.split(path)[0], recursive, path)
else:
raise Exception(
'Unsupported file format, cannot resolve dependencies')
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getDependencies(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< getDependencies(): %s" % str(res))
return res
def createPackageFile(path):
"""
Creates a testerman package file from a docroot package root folder.
"""
def _packageFolder(tfile, relbasepath, docrootbasepath):
getLogger().debug("Walking folder %s..." % docrootbasepath)
for entry in FileSystemManager.instance().getdir(docrootbasepath):
name, apptype = entry['name'], entry['type']
if apptype == FileSystemManager.APPTYPE_DIR:
relpath = "%s%s/" % (relbasepath, name) # the name of the current item within the package
docrootpath = "%s%s/" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding directory %s..." % relpath)
tarinfo = tarfile.TarInfo(relpath)
tarinfo.type = tarfile.DIRTYPE
tarinfo.mode = 0755
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
tfile.addfile(tarinfo)
_packageFolder(tfile, relbasepath = relpath, docrootbasepath = docrootpath)
else:
relname = "%s%s" % (relbasepath, name) # the name of the current item within the package
docrootname = "%s%s" % (docrootbasepath, name) # the name of the current item within the docroot
getLogger().debug("Adding file %s..." % relname)
tarinfo = tarfile.TarInfo(relname)
tarinfo.type = tarfile.AREGTYPE
tarinfo.mode = 0644
tarinfo.uid = os.getuid()
tarinfo.gid = os.getgid()
tarinfo.mtime = time.time()
content = FileSystemManager.instance().read(docrootname)
tarinfo.size = len(content)
contentObj = StringIO.StringIO(content)
tfile.addfile(tarinfo, contentObj)
contentObj.close()
getLogger().debug("File %s added to package file (%s bytes)" % (relname, tarinfo.size))
getLogger().info("Creating package file from %s..." % path)
if not FileSystemManager.instance().isdir(path):
getLogger().info("Cannot create package from %s: not a path to package" % path)
return None
tpk = StringIO.StringIO()
tfile = tarfile.open("tpk", "w:gz", tpk)
# Now, traverse the files into path
if not path.endswith('/'):
path = "%s/" % path
_packageFolder(tfile, '', path)
tfile.close()
contents = tpk.getvalue()
tpk.close()
getLogger().info("Package file for %s created, %s bytes" % (path, len(contents)))
return contents
def getReverseDependencies(path):
"""
Computes the reverse file dependencies of the file referenced by path,
i.e. the list of files in the repository that reference this path (which
is typically a module - campaigns referencing an ats or a campaign won't
be retrieved for now).
A reverse dependency for a module is another module or ATS that imports it.
This method may be used by a client to check if a module is currently in use
or not.
Only reverse dependencies at call time are searched - if older revisions
of files reference this module, it won't be checked.
@type path: string
@param path: a docroot path to a module
@rtype: list of strings
@returns: a list of reverse dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getReverseDependencies(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
# Well, we need to scan all our python and ATS files into our repository (the candidate)
# do a:
# candidateSource = getFile(candidatePath)
# try:
# deps = DependencyResolver.python_getDependencyFilenames(candidateSource, candidatePath, recursive = False)
# except:
# deps = []
# if path in deps and candidatePath not in res:
# res.append(candidatePath)
res = []
else:
# Reverse dependencies is not supported on something that is not a module
res = []
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getReverseDependencies(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< getReverseDependencies(): %s" % str(res))
return res
def getReverseDependencies(path):
"""
Computes the reverse file dependencies of the file referenced by path,
i.e. the list of files in the repository that reference this path (which
is typically a module - campaigns referencing an ats or a campaign won't
be retrieved for now).
A reverse dependency for a module is another module or ATS that imports it.
This method may be used by a client to check if a module is currently in use
or not.
Only reverse dependencies at call time are searched - if older revisions
of files reference this module, it won't be checked.
@type path: string
@param path: a docroot path to a module
@rtype: list of strings
@returns: a list of reverse dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getReverseDependencies(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
# Well, we need to scan all our python and ATS files into our repository (the candidate)
# do a:
# candidateSource = getFile(candidatePath)
# try:
# deps = DependencyResolver.python_getDependencyFilenames(candidateSource, candidatePath, recursive = False)
# except:
# deps = []
# if path in deps and candidatePath not in res:
# res.append(candidatePath)
res = []
else:
# Reverse dependencies is not supported on something that is not a module
res = []
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getReverseDependencies(...): Fault:\n" + str(e))
raise(e)
def schedulePackage(path, username, session, at, script = None, profileName = None):
"""
Schedules a package to start at <at>.
If script is provided, the package's default-script attribute is ignored
and script is used instead.
@since: 1.3
"""
getLogger().info(">> schedulePackage(%s, script = %s)" % (path, script))
if not path.startswith('/'): path = '/' + path
try:
metadata = Package.getPackageMetadata(path)
if not script:
script = metadata['default-script']
scriptFilename = "%s/src/%s" % (path, script)
scriptPath = os.path.split(scriptFilename)[0]
label = "%s:%s" % ('/'.join(path.split('/')[2:]), script)
getLogger().info("Using package script filename %s, path %s" % (scriptFilename, scriptPath))
if script.endswith(".campaign"):
res = scheduleCampaign(FileSystemManager.instance().read(scriptFilename).decode('utf-8'), label, username, session, at = at, path = scriptPath)
elif script.endswith(".ats"):
res = scheduleAts(FileSystemManager.instance().read(scriptFilename).decode('utf-8'), label, username, session, at = at, path = scriptPath)
else:
raise Exception("Invalid script/default script for package execution (unrecognized job type based on extension - %s)" % script)
except Exception, e:
e = Exception("Scheduling error: %s" % (str(e)))
getLogger().info("<< schedulePackage(...): Fault:\n%s" % Tools.getBacktrace())
raise(e)
def move(source, destination):
"""
Moves a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be moved.
Logs associated to a scripts, if any, are
NOT moved. They are kept available in the archives,
but not associated to the script any more.
FIXME: Revisions should be moved, however.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> move(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().move(source, destination)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< move(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< move(): %s" % str(res))
return res
def getDependencies(path, recursive = False):
"""
Computes the file dependencies of the file referenced by path.
If recursive is set to True, also searches for additional dependencies
recursively; otherwise only direct dependencies are computed.
A dependency for an ATS is a module it imports.
A depencendy for a module is a module it imports.
A dependency for a campaign is a a script (ats or campaign) it calls.
This method may be used by a client to create a package.
@since: 1.3
@type path: string
@param path: a docroot path to a module, ats or campaign
@type recursive: boolean
@param recursive: False for direct depencencies only. True for all
dependencies.
@rtype: list of strings
@returns: a list of dependencies as docroot-path to filenames.
A dependency is only listed once (no duplicate).
"""
getLogger().info(">> getDependencies(%s, %s)" % (path, recursive))
if not path.startswith('/'): path = '/' + path
res = []
try:
source = FileSystemManager.instance().read(path)
if source is None:
raise Exception('Cannot find %s' % path)
if path.endswith('.py'):
res = DependencyResolver.python_getDependencyFilenames(source, path, recursive)
elif path.endswith('.ats'):
res = DependencyResolver.python_getDependencyFilenames(source, path, recursive)
elif path.endswith('.campaign'):
res = DependencyResolver.campaign_getDependencyFilenames(source, os.path.split(path)[0], recursive, path)
else:
raise Exception('Unsupported file format, cannot resolve dependencies')
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getDependencies(...): Fault:\n" + str(e))
raise(e)
def move(source, destination):
"""
Moves a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be moved.
Logs associated to a scripts, if any, are
NOT moved. They are kept available in the archives,
but not associated to the script any more.
FIXME: Revisions should be moved, however.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> move(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().move(source, destination)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< move(...): Fault:\n" + str(e))
raise(e)
def getFile(path, useCompression=False):
"""
Retrieves a file according to the path.
The path is relative to the document root.
If useCompression is set, compress the output before encoding it to mime64.
@since: 1.0
@type path: string
@param path: a path to a file
@type useCompression: bool
@param useCompression: if True, the output is gziped before being mime64-encoded
@rtype: string (utf-8 or buffer, encoded in base64), or None
@returns: None if the file was not found,
or the file contents in base64 encoding, optionally compressed
"""
getLogger().info(">> getFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
ret = None
try:
contents = FileSystemManager.instance().read(path)
if contents is None:
ret = None
else:
if useCompression:
ret = base64.encodestring(zlib.compress(contents))
else:
ret = base64.encodestring(contents)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< getFile(...): Fault:\n%s" % str(e))
ret = None
if ret is not None:
getLogger().info("<< getFile(%s): %d bytes returned" %
(path, len(ret)))
else:
getLogger().info("<< getFile(%s): file not found" % (path))
return ret
def copy(source, destination):
"""
Copies a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be copied.
Meta children (Revisions and logs, if any) are NOT copied.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> copy(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().copy(source, destination)
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< copy(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< copy(): %s" % str(res))
return res
def putFile(content, path, useCompression=False, username=None):
"""
Writes a file to docroot/path
@since: 1.0
@type content: utf-8 encoded (or buffer) string, encoded in mime64
@param content: the content of the file
@type path: string
@param path: a complete path, with filename and extension, relative to the document root.
@type useCompression: bool
@param useCompression: (since 1.3) if set to True, the content is gziped before being mime64-encoded.
@type username: string
@param username: (since 1.7) the committer/writer
@rtype: bool
@returns: True if OK, False otherwise
"""
getLogger().info(">> putFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
res = False
try:
content = base64.decodestring(content)
if useCompression:
content = zlib.decompress(content)
revision = FileSystemManager.instance().write(path,
content,
username=username)
# No revision handling for now
# We should return the new filepath in case of a success
# /repository/samples/[email protected]
# etc
res = True
except Exception as e:
e = Exception("Unable to perform operation: %s\n%s" %
(str(e), Tools.getBacktrace()))
getLogger().info("<< putFile(...): Fault:\n" + str(e))
raise (e)
getLogger().info("<< putFile(): %s" % str(res))
return res
def getPackageMetadata(path):
"""
Extracts the different package metadata contained into the package.xml root folder.
@type path: string
@param path: the docroot path to the package's root folder
@rtype: dict[string] of unicode string
@returns: the metadata, as a dict containing the following keys:
author, description, default-script, status
"""
descriptionFile = "%s/package.xml" % path
try:
content = FileSystemManager.instance().read(descriptionFile)
if content is None:
raise Exception("Unable to read the package description file for package %s" % path)
# Now, parse the document
return parsePackageDescription(content)
except Exception, e:
raise e
def copy(source, destination):
"""
Copies a file or a directory to destination.
Recursive operation: if the source is a directory, the whole
tree will be copied.
Meta children (Revisions and logs, if any) are NOT copied.
source is a docroot to an existing path or directory.
destination is a docroot path to a destination:
- if source is a dir, destination can be an existing dir
(will create a new dir in it) or a new directory name
(will rename the directory).
- if source is a file, destination can be an existing dir
(will create the file in it, without renaming it), or
a new file name (will rename the file).
@since: 1.3
@type source: string
@param source: docroot-path to the object to move
@type destination: string
@param destination: docroot-path to the destination
(if existing: must be a directory; if not existing, will rename
the object on the fly)
@rtype: bool
@returns: True if the move was OK, False otherwise.
"""
getLogger().info(">> copy(%s, %s)" % (source, destination))
if not source.startswith('/'): source = '/' + source
if not destination.startswith('/'): destination = '/' + destination
res = False
try:
res = FileSystemManager.instance().copy(source, destination)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< copy(...): Fault:\n" + str(e))
raise(e)
def makeDirectory(path):
"""
Creates a directory and all the needed directories to it, if any.
@since: 1.3
@type path: string
@param path: the docroot-path to the directory to create
@rtype: bool
@returns: True if the directory was created, False otherwise.
"""
getLogger().info(">> makeDirectory(%s)" % (path))
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().mkdir(path)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< makeDirectory(...): Fault:\n" + str(e))
raise(e)
def removeFile(path):
"""
Removes a file.
@since: 1.0
@type path: string
@param path: the docroot-path to the file to delete
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeFile(%s)" % path)
if not path.startswith('/'):
path = '/' + path
res = False
try:
res = FileSystemManager.instance().unlink(path)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< removeFile(...): Fault:\n" + str(e))
raise(e)
def putFile(content, path, useCompression = False, username = None):
"""
Writes a file to docroot/path
@since: 1.0
@type content: utf-8 encoded (or buffer) string, encoded in mime64
@param content: the content of the file
@type path: string
@param path: a complete path, with filename and extension, relative to the document root.
@type useCompression: bool
@param useCompression: (since 1.3) if set to True, the content is gziped before being mime64-encoded.
@type username: string
@param username: (since 1.7) the committer/writer
@rtype: bool
@returns: True if OK, False otherwise
"""
getLogger().info(">> putFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
res = False
try:
content = base64.decodestring(content)
if useCompression:
content = zlib.decompress(content)
revision = FileSystemManager.instance().write(path, content, username = username)
# No revision handling for now
# We should return the new filepath in case of a success
# /repository/samples/[email protected]
# etc
res = True
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< putFile(...): Fault:\n" + str(e))
raise(e)
def removeDirectory(path, recursive = False):
"""
Removes an empty directory, unless recursive is set to True.
@since: 1.1
@type path: string
@param path: the docroot-path to the directory to delete
@type recursive: bool
@param recursive: True if we should delete files and directories in it. DANGEROUS.
@rtype: bool
@returns: True if OK, False if nothing deleted. (? to check)
"""
getLogger().info(">> removeDirectory(%s)" % path)
if not path.startswith('/'): path = '/' + path
res = False
try:
res = FileSystemManager.instance().rmdir(path, recursive)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< removeDirectory(...): Fault:\n" + str(e))
raise(e)
def getFile(path, useCompression = False):
"""
Retrieves a file according to the path.
The path is relative to the document root.
If useCompression is set, compress the output before encoding it to mime64.
@since: 1.0
@type path: string
@param path: a path to a file
@type useCompression: bool
@param useCompression: if True, the output is gziped before being mime64-encoded
@rtype: string (utf-8 or buffer, encoded in base64), or None
@returns: None if the file was not found,
or the file contents in base64 encoding, optionally compressed
"""
getLogger().info(">> getFile(%s, %s)" % (path, useCompression))
if not path.startswith('/'):
path = '/' + path
ret = None
try:
contents = FileSystemManager.instance().read(path)
if contents is None:
ret = None
else:
if useCompression:
ret = base64.encodestring(zlib.compress(contents))
else:
ret = base64.encodestring(contents)
except Exception, e:
e = Exception("Unable to perform operation: %s\n%s" % (str(e), Tools.getBacktrace()))
getLogger().info("<< getFile(...): Fault:\n%s" % str(e))
ret = None
mode_input = input("Please enter 1 (Train model) or 2 (Prediction): ")
if int(mode_input) == 1:
clean_run = input(
"Destroy existing resources and train from scratch " +
"(WARNING: this is memory intensive and may take considerable time)? Enter Y/N: "
)
while clean_run.upper() not in ['Y', 'N']:
clean_run = input(
"Please enter Y (clean run) or N (retrain existing resources): ")
if clean_run.upper() == 'Y':
file_manager = FileSystemManager(image_directory, model_directory)
file_manager.clean_run()
extract_dir = file_manager.extract_archive(source_archive)
file_manager.remove_files_except('.png')
file_manager.data_science_fs(category0='benign', category1='malignant')
file_manager.organise_files(extract_dir,
category_rules={
'benign': 'SOB_B_.*.png',
'malignant': 'SOB_M_.*.png'
})
elif clean_run.upper() == 'N':
pass
else:
raise Exception("Missing profiles folder (profiles)")
# Now we should check the package.xml file, too
return True
def importPackageFile(content, path):
"""
Expand a package file to a docroot folder.
"""
try:
checkPackageFile(content)
except Exception, e:
getLogger().info("Invalid package file: %s" % e)
raise Exception("Invalid package file: %s" % e)
if FileSystemManager.instance().isfile(path):
getLogger().info("Cannot import package to %s: not a path to package" % path)
raise Exception("Invalid destination package path: is a file")
if FileSystemManager.instance().isdir(path):
getLogger().info("Cannot import package from %s: destination path already exist" % path)
raise Exception("Invalid destination package path: this path already exist")
# Minimal package tree
FileSystemManager.instance().mkdir("%s/profiles" % path, False)
FileSystemManager.instance().mkdir("%s/src" % path, False)
# First unpack the package, then notify the package dir creation so that it is seen as a package folder.
tpk = StringIO.StringIO(content)
tfile = tarfile.open("tpk", "r:gz", tpk)
def campaign_getDependencyFilenames(source, sourcePath = None, recursive = False, sourceFilename = "<local>", moduleRootDir = '/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the campaign depends on.
Calls pythong_getDependencyFilenames on ats files if recursive is True,
and thus only userland dependencies are searched.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourcePath: string
@param sourcePath: docroot path used as a 'working dir' to search for
relative dependencies. Typically the path where the source has been extracted
from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type sourceFilename: string
@param sourceFilename: a string to help identify the file that yielded the source.
For non-repository files, use <local>, by convention.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
currentDependencies = []
getLogger().info("%s: parsing campaign file to look for its dependencies" % sourceFilename)
# The path of the campaign within the docroot.
path = sourcePath
# Based on a file parsing.
indent = 0
parentPresent = True
lc = 0
for line in source.splitlines():
lc += 1
# Remove comments
line = line.split('#', 1)[0].rstrip()
if not line:
continue # empty line
m = re.match(r'(?P<indent>\s*)((?P<branch>\w+|\*)\s+)?(?P<type>\w+)\s+(?P<filename>[^\s]+)(\s+with\s+(?P<mapping>.*)\s*)?', line)
if not m:
raise Exception('%s: parse error at line %s: invalid line format' % (sourceFilename, lc))
type_ = m.group('type')
filename = m.group('filename')
indentDiff = len(m.group('indent')) - indent
indent = indent + indentDiff
# Filename creation within the docroot
if filename.startswith('/'):
# absolute path within the *repository*
filename = '/%s%s' % (cm.get_transient('constants.repository'), filename)
else:
# just add the local campaign path
filename = '%s/%s' % (path, filename)
# Type validation
if not type_ in [ 'ats', 'campaign' ]:
raise Exception('%s: error at line %s: invalid job type (%s)' % (sourceFilename, lc, type_))
# Indentation validation
if indentDiff > 1:
raise Exception('%s: parse error at line %s: invalid indentation (too deep)' % (sourceFilename, lc))
elif indentDiff == 1:
if not parentPresent:
raise Exception('%s: parse error at line %s: invalid indentation (invalid initial indentation)' % (sourceFilename, lc))
elif indentDiff == 0:
# the current parent remains the parent
pass
else:
# negative indentation.
pass
# OK, now we have at least one parent.
parentPresent = True
# Branch validation: ignored for dependency resolver
# Now handle the dependency.
if not filename in currentDependencies:
getLogger().info('%s: campaign direct dependency added: %s' % (sourceFilename, filename))
currentDependencies.append(filename)
if recursive and type_ in ['ats', 'campaign']:
nextDependencies = []
nextPath, nextFilename = os.path.split(filename)
nextSource = FileSystemManager.instance().read(filename)
if nextSource is None:
raise Exception('%s: missing dependency: file %s is not in the repository' % (sourceFilename, filename))
if type_ == 'campaign':
nextDependencies = campaign_getDependencyFilenames(nextSource, nextPath, True, filename, moduleRootDir = moduleRootDir)
elif type_ == 'ats':
nextDependencies = python_getDependencyFilenames(nextSource, sourceFilename = filename, recursive = True, moduleRootDir = moduleRootDir)
for dep in nextDependencies:
if not dep in currentDependencies:
getLogger().info('%s: campaign indirect dependency added: %s' % (sourceFilename, dep))
currentDependencies.append(dep)
return currentDependencies
def campaign_getDependencyFilenames(source,
sourcePath=None,
recursive=False,
sourceFilename="<local>",
moduleRootDir='/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the campaign depends on.
Calls pythong_getDependencyFilenames on ats files if recursive is True,
and thus only userland dependencies are searched.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourcePath: string
@param sourcePath: docroot path used as a 'working dir' to search for
relative dependencies. Typically the path where the source has been extracted
from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type sourceFilename: string
@param sourceFilename: a string to help identify the file that yielded the source.
For non-repository files, use <local>, by convention.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
currentDependencies = []
getLogger().info("%s: parsing campaign file to look for its dependencies" %
sourceFilename)
# The path of the campaign within the docroot.
path = sourcePath
# Based on a file parsing.
indent = 0
parentPresent = True
lc = 0
for line in source.splitlines():
lc += 1
# Remove comments
line = line.split('#', 1)[0].rstrip()
if not line:
continue # empty line
m = re.match(
r'(?P<indent>\s*)((?P<branch>\w+|\*)\s+)?(?P<type>\w+)\s+(?P<filename>[^\s]+)(\s+with\s+(?P<mapping>.*)\s*)?',
line)
if not m:
raise Exception('%s: parse error at line %s: invalid line format' %
(sourceFilename, lc))
type_ = m.group('type')
filename = m.group('filename')
indentDiff = len(m.group('indent')) - indent
indent = indent + indentDiff
# Filename creation within the docroot
if filename.startswith('/'):
# absolute path within the *repository*
filename = '/%s%s' % (cm.get_transient('constants.repository'),
filename)
else:
# just add the local campaign path
filename = '%s/%s' % (path, filename)
# Type validation
if not type_ in ['ats', 'campaign']:
raise Exception('%s: error at line %s: invalid job type (%s)' %
(sourceFilename, lc, type_))
# Indentation validation
if indentDiff > 1:
raise Exception(
'%s: parse error at line %s: invalid indentation (too deep)' %
(sourceFilename, lc))
elif indentDiff == 1:
if not parentPresent:
raise Exception(
'%s: parse error at line %s: invalid indentation (invalid initial indentation)'
% (sourceFilename, lc))
elif indentDiff == 0:
# the current parent remains the parent
pass
else:
# negative indentation.
pass
# OK, now we have at least one parent.
parentPresent = True
# Branch validation: ignored for dependency resolver
# Now handle the dependency.
if not filename in currentDependencies:
getLogger().info('%s: campaign direct dependency added: %s' %
(sourceFilename, filename))
currentDependencies.append(filename)
if recursive and type_ in ['ats', 'campaign']:
nextDependencies = []
nextPath, nextFilename = os.path.split(filename)
nextSource = FileSystemManager.instance().read(filename)
if nextSource is None:
raise Exception(
'%s: missing dependency: file %s is not in the repository'
% (sourceFilename, filename))
if type_ == 'campaign':
nextDependencies = campaign_getDependencyFilenames(
nextSource,
nextPath,
True,
filename,
moduleRootDir=moduleRootDir)
elif type_ == 'ats':
nextDependencies = python_getDependencyFilenames(
nextSource,
sourceFilename=filename,
recursive=True,
moduleRootDir=moduleRootDir)
for dep in nextDependencies:
if not dep in currentDependencies:
getLogger().info(
'%s: campaign indirect dependency added: %s' %
(sourceFilename, dep))
currentDependencies.append(dep)
return currentDependencies
def python_getDependencyFilenames(source,
sourceFilename,
recursive=True,
moduleRootDir='/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the ATS/module depends on.
Only userland dependencies are searched, i.e. modules below $docroot/repository.
Non-userland dependencies, such as Testerman and Python std modules,
are not reported.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourceFilename: string
@param sourceFilename: the full docroot path to the source file.
For non-repository files, should be set to something like /repository/<anonymous.ats>
so that we can deduce the docroot "working dir" dependencies should be searched from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
getLogger().info("Resolving dependencies for file %s" % sourceFilename)
# This is the PYTHONPATH that will be used to run the TE.
# Will be used as a fallback if an import cannot be found in userland (ie in the repo)
pythonPath = '%(root)s/modules' % dict(
root=cm.get_transient("ts.server_root"))
additionalPythonPath = cm.get("testerman.te.python.additional_pythonpath")
if additionalPythonPath:
pythonPath += ':' + additionalPythonPath
pythonPath = pythonPath.split(':')
# Will only include userland dependencies. System dependencies found in PYTHONPATH won't be included.
ret = []
# Bootstrap the deps (stored as (list of imported modules, path of the importing file) )
toResolve = [(d, sourceFilename)
for d in python_getImportedUserlandModules(
source, sourceFilename=sourceFilename)]
# the list of resolved dependencies,
# as a map (import, fromFilename): resolvedFilename
resolvedSoFar = {}
# For each deps to resolve (a list of (import, fromFilename)),
# we need to resolve the filename that will provide this import for this file.
while len(toResolve):
getLogger().debug("List of imports to resolve for script %s:\n%s" %
(sourceFilename, "\n".join(
["%s (used in %s)" % x for x in toResolve])))
dep, fromFilename = toResolve.pop()
# Some non-userland files - not resolved to build the TE userland package
# How can we detect standard Python includes ?
# fromFilePath starts with the "python home" ? something else ?
getLogger().debug("Resolving import %s from %s..." %
(dep, fromFilename))
# Skip some dependencies provided by the Testerman infrastructure
#if dep in [ ]:
# getLogger().info("Resolving import %s from %: skipped, not userland" % (dep, fromFilename))
# continue
# Skip already resolved dependencies
if (dep, fromFilename) in resolvedSoFar:
getLogger().debug(
"Resolving import %s from %s: already resolved as %s" %
(dep, fromFilename, resolvedSoFar[(dep, fromFilename)]))
continue
# Ordered list of filenames within the docroot that could provide the dependency:
# (module path)
# - first search from the local file path, if provided,
# - then search from the userland module paths (limited to '/repository/' for now)
modulePaths = []
# First, try a local module (relative path) (same dir as the currently analyzed file)
sourcePath = os.path.split(fromFilename)[0]
modulePaths.append(sourcePath)
# Then fall back to standard "testerman userland paths"
# If the filename was in a package, look from the package's root dir
# Otherwise this the repository root.
for modulePath in [moduleRootDir]:
if modulePath and not modulePath in modulePaths:
modulePaths.append(modulePath)
getLogger().debug(
"Resolving import %s from %s: searching in paths:\n%s" %
(dep, fromFilename, "\n".join(modulePaths)))
found = None
depSource = None
for path in modulePaths:
depFilename = '%s/%s.py' % (path, dep.replace('.', '/'))
try:
depSource = FileSystemManager.instance().read(depFilename)
except Exception:
pass
if depSource is not None:
found = depFilename
break
if not found:
getLogger().debug(
"Resolving import %s from %s: not available in repository, searched in paths:\n%s"
% (dep, fromFilename, "\n".join(modulePaths)))
try:
imp.find_module(dep, pythonPath)
except:
getLogger().debug(
"Resolving import %s from %s: not available in PYTHONPATH either, searched paths:\n%s"
% (dep, fromFilename, "\n".join(pythonPath)))
raise Exception(
'Missing module: %s (imported from %s) is not available in userland (repository, searched paths: %s) or in TE PYTHONPATH (searched paths: %s)'
% (dep, fromFilename, modulePaths, pythonPath))
getLogger().debug(
"Resolving import %s from %s: OK, found in PYTHONPATH, won't be included in userland dependencies"
)
if found:
# OK, we resolved a file.
resolvedSoFar[(dep, fromFilename)] = depFilename
getLogger().debug("Resolving import %s from %s: resolved as %s" %
(dep, fromFilename, depFilename))
if not depFilename in ret:
ret.append(depFilename)
getLogger().debug(
"Script %s is now using the following files:\n%s" %
(sourceFilename, "\n".join(ret)))
# Now, analyze the resolved file and add its own dependencies to the list to resolve,
# if not already resolved
if recursive:
importedModules = python_getImportedUserlandModules(
depSource, depFilename)
for im in importedModules:
if not (im, depFilename) in resolvedSoFar:
toResolve.append((im, depFilename))
else:
getLogger().debug(
"Resolving import %s from %s: already resolved" %
(im, depFilename))
return ret
def python_getDependencyFilenames(source, sourceFilename, recursive = True, moduleRootDir = '/repository'):
"""
Returns a list of userland module filenames
(including their own dependencies) the ATS/module depends on.
Only userland dependencies are searched, i.e. modules below $docroot/repository.
Non-userland dependencies, such as Testerman and Python std modules,
are not reported.
NB: this works only because user modules are files only, not packages.
@type source: utf-8 string
@param source: a Python source file (module or ATS)
@type sourceFilename: string
@param sourceFilename: the full docroot path to the source file.
For non-repository files, should be set to something like /repository/<anonymous.ats>
so that we can deduce the docroot "working dir" dependencies should be searched from.
@type recursive: bool
@param recursive: if True, search recursively for dependencies in imported modules.
@type moduleRootDir: string
@param moduleRootDir: the docroot directory we should start looking modules from.
For a normal file, this is /repository. For a file contained in a package, this is the package src dir.
@rtype: list of strings
@returns: a list of docroot-path to dependencies (no duplicate).
"""
getLogger().info("Resolving dependencies for file %s" % sourceFilename)
# This is the PYTHONPATH that will be used to run the TE.
# Will be used as a fallback if an import cannot be found in userland (ie in the repo)
pythonPath = '%(root)s/modules' % dict(
root = cm.get_transient("ts.server_root"))
additionalPythonPath = cm.get("testerman.te.python.additional_pythonpath")
if additionalPythonPath:
pythonPath += ':' + additionalPythonPath
pythonPath = pythonPath.split(':')
# Will only include userland dependencies. System dependencies found in PYTHONPATH won't be included.
ret = []
# Bootstrap the deps (stored as (list of imported modules, path of the importing file) )
toResolve = [ (d, sourceFilename) for d in python_getImportedUserlandModules(source, sourceFilename = sourceFilename) ]
# the list of resolved dependencies,
# as a map (import, fromFilename): resolvedFilename
resolvedSoFar = {}
# For each deps to resolve (a list of (import, fromFilename)),
# we need to resolve the filename that will provide this import for this file.
while len(toResolve):
getLogger().debug("List of imports to resolve for script %s:\n%s" % (sourceFilename, "\n".join(["%s (used in %s)" % x for x in toResolve])))
dep, fromFilename = toResolve.pop()
# Some non-userland files - not resolved to build the TE userland package
# How can we detect standard Python includes ?
# fromFilePath starts with the "python home" ? something else ?
getLogger().debug("Resolving import %s from %s..." % (dep, fromFilename))
# Skip some dependencies provided by the Testerman infrastructure
#if dep in [ ]:
# getLogger().info("Resolving import %s from %: skipped, not userland" % (dep, fromFilename))
# continue
# Skip already resolved dependencies
if (dep, fromFilename) in resolvedSoFar:
getLogger().debug("Resolving import %s from %s: already resolved as %s" % (dep, fromFilename, resolvedSoFar[(dep, fromFilename)]))
continue
# Ordered list of filenames within the docroot that could provide the dependency:
# (module path)
# - first search from the local file path, if provided,
# - then search from the userland module paths (limited to '/repository/' for now)
modulePaths = []
# First, try a local module (relative path) (same dir as the currently analyzed file)
sourcePath = os.path.split(fromFilename)[0]
modulePaths.append(sourcePath)
# Then fall back to standard "testerman userland paths"
# If the filename was in a package, look from the package's root dir
# Otherwise this the repository root.
for modulePath in [ moduleRootDir ]:
if modulePath and not modulePath in modulePaths:
modulePaths.append(modulePath)
getLogger().debug("Resolving import %s from %s: searching in paths:\n%s" % (dep, fromFilename, "\n".join(modulePaths)))
found = None
depSource = None
for path in modulePaths:
depFilename = '%s/%s.py' % (path, dep.replace('.', '/'))
try:
depSource = FileSystemManager.instance().read(depFilename)
except Exception:
pass
if depSource is not None:
found = depFilename
break
if not found:
getLogger().debug("Resolving import %s from %s: not available in repository, searched in paths:\n%s" % (dep, fromFilename, "\n".join(modulePaths)))
try:
imp.find_module(dep, pythonPath)
except:
getLogger().debug("Resolving import %s from %s: not available in PYTHONPATH either, searched paths:\n%s" % (dep, fromFilename, "\n".join(pythonPath)))
raise Exception('Missing module: %s (imported from %s) is not available in userland (repository, searched paths: %s) or in TE PYTHONPATH (searched paths: %s)' % (dep, fromFilename, modulePaths, pythonPath))
getLogger().debug("Resolving import %s from %s: OK, found in PYTHONPATH, won't be included in userland dependencies")
if found:
# OK, we resolved a file.
resolvedSoFar[(dep, fromFilename)] = depFilename
getLogger().debug("Resolving import %s from %s: resolved as %s" % (dep, fromFilename, depFilename))
if not depFilename in ret:
ret.append(depFilename)
getLogger().debug("Script %s is now using the following files:\n%s" % (sourceFilename, "\n".join(ret)))
# Now, analyze the resolved file and add its own dependencies to the list to resolve,
# if not already resolved
if recursive:
importedModules = python_getImportedUserlandModules(depSource, depFilename)
for im in importedModules:
if not (im, depFilename) in resolvedSoFar:
toResolve.append((im, depFilename))
else:
getLogger().debug("Resolving import %s from %s: already resolved" % (im, depFilename))
return ret
getLogger().info("Using %s = %s" % (str(k), cm.get(k)))
# Now we can daemonize if needed
if cm.get("ts.daemonize"):
if pidfile:
getLogger().info("Daemonizing, using pid file %s..." % pidfile)
else:
getLogger().info("Daemonizing...")
Tools.daemonize(pidFilename = pidfile, displayPid = True)
# Main start
cm.set_transient("ts.pid", os.getpid())
try:
serverThread = XmlRpcServerThread() # Ws server
FileSystemManager.initialize()
EventManager.initialize() # Xc server, Ih server [TSE:CH], Il server [TSE:TL]
ProbeManager.initialize() # Ia client
JobManager.initialize() # Job scheduler
serverThread.start()
getLogger().info("Started.")
while 1:
time.sleep(1)
except KeyboardInterrupt:
getLogger().info("Shutting down Testerman Server...")
except Exception, e:
sys.stderr.write("Unable to start server: %s\n" % str(e))
getLogger().critical("Unable to start server: " + str(e))
serverThread.stop()
JobManager.finalize()
for k in items:
getLogger().info("Using %s = %s" % (str(k), cm.get(k)))
# Now we can daemonize if needed
if cm.get("ts.daemonize"):
if pidfile:
getLogger().info("Daemonizing, using pid file %s..." % pidfile)
else:
getLogger().info("Daemonizing...")
Tools.daemonize(pidFilename=pidfile, displayPid=True)
# Main start
cm.set_transient("ts.pid", os.getpid())
try:
serverThread = XmlRpcServerThread() # Ws server
FileSystemManager.initialize()
EventManager.initialize(
) # Xc server, Ih server [TSE:CH], Il server [TSE:TL]
ProbeManager.initialize() # Ia client
JobManager.initialize() # Job scheduler
serverThread.start()
getLogger().info("Started.")
while 1:
time.sleep(1)
except KeyboardInterrupt:
getLogger().info("Shutting down Testerman Server...")
except Exception, e:
sys.stderr.write("Unable to start server: %s\n" % str(e))
getLogger().critical("Unable to start server: " + str(e))
serverThread.stop()
def main():
server_root = os.path.abspath(
os.path.dirname(sys.modules[globals()['__name__']].__file__))
testerman_home = os.path.abspath("%s/.." % server_root)
# Set transient values
cm.set_transient("testerman.testerman_home", testerman_home)
cm.set_transient("ts.server_root", server_root)
# standard paths within the document root.
cm.set_transient("constants.repository", "repository")
cm.set_transient("constants.archives", "archives")
cm.set_transient("constants.modules", "modules")
cm.set_transient("constants.components", "components")
cm.set_transient("ts.version", Versions.getServerVersion())
# Register persistent variables
expandPath = lambda x: x and os.path.abspath(
os.path.expandvars(os.path.expanduser(x)))
splitPaths = lambda paths: [expandPath(x) for x in paths.split(',')]
cm.register("interface.ws.ip", "0.0.0.0")
cm.register("interface.ws.port", 8080)
cm.register("interface.xc.ip", "0.0.0.0")
cm.register("interface.xc.port", 8081)
cm.register("interface.il.ip", "0.0.0.0")
cm.register("interface.il.port", 8082)
cm.register("interface.ih.ip", "0.0.0.0")
cm.register("interface.ih.port", 8083)
cm.register("interface.xa.ip", "0.0.0.0")
cm.register("interface.xa.port", 40000)
cm.register("tacs.ip", "127.0.0.1")
cm.register("tacs.port", 8087)
cm.register("ts.daemonize", False)
cm.register("ts.debug", False)
cm.register("ts.log_filename", "")
cm.register("ts.pid_filename", "")
cm.register("ts.name", socket.gethostname(), dynamic=True)
cm.register("ts.jobscheduler.interval", 1000, dynamic=True)
cm.register("testerman.document_root",
"/tmp",
xform=expandPath,
dynamic=True)
cm.register("testerman.var_root", "", xform=expandPath)
cm.register("testerman.web.document_root",
"%s/web" % testerman_home,
xform=expandPath,
dynamic=False)
cm.register("testerman.webclient.document_root",
"%s/webclient" % testerman_home,
xform=expandPath,
dynamic=False)
cm.register("testerman.administrator.name", "administrator", dynamic=True)
cm.register("testerman.administrator.email",
"testerman-admin@localhost",
dynamic=True)
# testerman.te.*: test executable-related variables
cm.register("testerman.te.codec_paths",
"%s/plugins/codecs" % testerman_home,
xform=splitPaths)
cm.register("testerman.te.probe_paths",
"%s/plugins/probes" % testerman_home,
xform=splitPaths)
cm.register("testerman.te.python.interpreter",
"/usr/bin/python",
dynamic=True)
cm.register(
"testerman.te.python.ttcn3module", "TestermanTTCN3", dynamic=True
) # TTCN3 adaptation lib (enable the easy use of previous versions to keep script compatibility)
cm.register(
"testerman.te.python.additional_pythonpath", "", dynamic=True
) # Additional search paths for system-wide modules (non-userland/in repository)
cm.register(
"testerman.te.log.max_payload_size", 64 * 1024, dynamic=True
) # the maximum dumpable payload in log (as a single value). Bigger payloads are truncated to this size, in bytes.
cm.register("ts.webui.theme", "default", dynamic=True)
cm.register("wcs.webui.theme", "default", dynamic=True)
parser = optparse.OptionParser(version=getVersion())
group = optparse.OptionGroup(parser, "Basic Options")
group.add_option("--debug",
dest="debug",
action="store_true",
help="turn debug traces on")
group.add_option("-d",
dest="daemonize",
action="store_true",
help="daemonize")
group.add_option("-r",
dest="docRoot",
metavar="PATH",
help="use PATH as document root (default: %s)" %
cm.get("testerman.document_root"))
group.add_option("--log-filename",
dest="logFilename",
metavar="FILENAME",
help="write logs to FILENAME instead of stdout")
group.add_option(
"--pid-filename",
dest="pidFilename",
metavar="FILENAME",
help=
"write the process PID to FILENAME when daemonizing (default: no pidfile)"
)
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "IPs and Ports Options")
group.add_option(
"--ws-ip",
dest="wsIp",
metavar="ADDRESS",
help=
"set listening Ws IP address to ADDRESS (default: listening on all interfaces)"
)
group.add_option("--ws-port",
dest="wsPort",
metavar="PORT",
help="set listening Ws port to PORT (default: %s)" %
cm.get("interface.ws.port"),
type="int")
group.add_option(
"--xc-ip",
dest="xcIp",
metavar="ADDRESS",
help=
"set Xc service IP address to ADDRESS (default: Ws IP if set, fallback to hostname resolution)"
)
group.add_option("--xc-port",
dest="xcPort",
metavar="PORT",
help="set Xc service port to PORT (default: %s)" %
cm.get("interface.xc.port"),
type="int")
group.add_option(
"--il-ip",
dest="ilIp",
metavar="ADDRESS",
help=
"set Il IP address to ADDRESS (default: listening on all interfaces)")
group.add_option("--il-port",
dest="ilPort",
metavar="PORT",
help="set Il port address to PORT (default: %s)" %
cm.get("interface.il.port"),
type="int")
group.add_option(
"--ih-ip",
dest="ihIp",
metavar="ADDRESS",
help=
"set Ih IP address to ADDRESS (default: listening o all interfaces)")
group.add_option("--ih-port",
dest="ihPort",
metavar="PORT",
help="set Ih port address to PORT (default: %s)" %
cm.get("interface.ih.port"),
type="int")
group.add_option(
"--tacs-ip",
dest="tacsIp",
metavar="ADDRESS",
help="set TACS Ia target IP address to ADDRESS (default: %s)" %
cm.get("tacs.ip"))
group.add_option(
"--tacs-port",
dest="tacsPort",
metavar="PORT",
help="set TACS Ia target port address to PORT (default: %s)" %
cm.get("tacs.port"),
type="int")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Advanced Options")
group.add_option(
"-V",
dest="varDir",
metavar="PATH",
help=
"use PATH to persist Testerman Server runtime variables, such as the job queue. If not provided, no persistence occurs between restarts."
)
group.add_option(
"-C",
"--conf-file",
dest="configurationFile",
metavar="FILENAME",
help=
"path to a configuration file. You may still use the command line options to override the values it contains."
)
group.add_option(
"-U",
"--users-file",
dest="usersFile",
metavar="FILENAME",
help=
"path to the configuration file that contains authorized webclient users."
)
group.add_option(
"-A",
"--apis-file",
dest="apisFile",
metavar="FILENAME",
help=
"path to the configuration file that contains supported language apis."
)
group.add_option(
"--codec-path",
dest="codecPaths",
metavar="PATHS",
help=
"search for codec modules in PATHS, which is a comma-separated list of paths"
)
group.add_option(
"--probe-path",
dest="probePaths",
metavar="PATHS",
help=
"search for probe modules in PATHS, which is a comma-separated list of paths"
)
group.add_option(
"--var",
dest="variables",
metavar="VARS",
help="set additional variables as VARS (format: key=value[,key=value]*)"
)
parser.add_option_group(group)
(options, args) = parser.parse_args()
# Configuration
# Read the settings from the saved configuration, if any
configFile = None
# Provided on the command line ?
if options.configurationFile is not None:
configFile = options.configurationFile
# No config file provided - fallback to $TESTERMAN_HOME/conf/testerman.conf if set and exists
elif Tools.fileExists("%s/conf/testerman.conf" % testerman_home):
configFile = "%s/conf/testerman.conf" % testerman_home
cm.set_transient("ts.configuration_filename", configFile)
usersFile = None
# Provided on the command line ?
if options.usersFile is not None:
usersFile = options.usersFile
# No config file provided - fallback to $TESTERMAN_HOME/conf/webclient-users.conf if set and exists
elif Tools.fileExists("%s/conf/webclient-users.conf" % testerman_home):
usersFile = "%s/conf/webclient-users.conf" % testerman_home
cm.set_transient("wcs.users_filename", usersFile)
apisFile = None
# Provided on the command line ?
if options.apisFile is not None:
apisFile = options.apisFile
# No config file provided - fallback to $TESTERMAN_HOME/conf/language-apis.conf if set and exists
elif Tools.fileExists("%s/conf/language-apis.conf" % testerman_home):
apisFile = "%s/conf/language-apis.conf" % testerman_home
cm.set_transient("ts.apis_filename", apisFile)
try:
if configFile: cm.read(configFile)
if usersFile: cm.read(usersFile, autoRegister=True)
if apisFile: cm.read(apisFile, autoRegister=True)
except Exception as e:
print(str(e))
return 1
# Now, override read settings with those set on explicit command line flags
# (or their default values inherited from the ConfigManager default values)
cm.set_user("interface.ws.ip", options.wsIp)
cm.set_user("interface.ws.port", options.wsPort)
cm.set_user("interface.xc.ip", options.xcIp)
cm.set_user("interface.xc.port", options.xcPort)
cm.set_user("interface.il.ip", options.ilIp)
cm.set_user("interface.il.port", options.ilPort)
cm.set_user("interface.ih.ip", options.ihIp)
cm.set_user("interface.ih.port", options.ihPort)
cm.set_user("tacs.ip", options.tacsIp)
cm.set_user("tacs.port", options.tacsPort)
cm.set_user("ts.daemonize", options.daemonize)
cm.set_user("ts.debug", options.debug)
cm.set_user("ts.log_filename", options.logFilename)
cm.set_user("ts.pid_filename", options.pidFilename)
cm.set_user("testerman.document_root", options.docRoot)
cm.set_user("testerman.te.codec_paths", options.codecPaths)
cm.set_user("testerman.te.probe_paths", options.probePaths)
cm.set_user("testerman.var_root", options.varDir)
if options.variables:
for var in options.variables.split(','):
try:
(key, val) = var.split('=')
cm.set_user(key, val)
except:
pass
# Commit all provided values (construct actual values via registered xforms)
cm.commit()
# Compute/adjust actual variables where applies
# Actual Xc IP address: if not explictly set, fallback to ws.ip, then to hostname().
ip = cm.get("interface.xc.ip")
if not ip or ip == '0.0.0.0':
ip = cm.get("interface.ws.ip")
cm.set_actual("interface.xc.ip", ip)
if not ip or ip == '0.0.0.0':
cm.set_actual(
"interface.xc.ip", socket.gethostbyname(socket.gethostname(
))) # Not fully qualified ? defaults to the hostname resolution.
# Set the TACS IP address that can be used by agents
# Normally, we should ask the TACS the server is connected to to get this value.
tacs = cm.get("interface.xa.ip")
if not tacs or tacs == '0.0.0.0':
# We'll publish the XC as XA IP address. If it was also set to localhost, it's unlikely agents are deployed outside localhost too.
cm.set_actual("interface.xa.ip", cm.get("interface.xc.ip"))
# If an explicit pid file was provided, use it. Otherwise, fallback to the var_root/ts.pid if possible.
pidfile = cm.get("ts.pid_filename")
if not pidfile and cm.get("testerman.var_root"):
# Set an actual value
pidfile = cm.get("testerman.var_root") + "/ts.pid"
cm.set_actual("ts.pid_filename", pidfile)
# print (Tools.formatTable([ ('key', 'Name'), ('format', 'Type'), ('dynamic', 'Dynamic'), ('default', 'Default value'), ('user', 'User value'), ('actual', 'Actual value')], cm.getVariables(), order = "key"))
# Logger initialization
if cm.get("ts.debug"):
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(
level=level,
format=
'%(asctime)s.%(msecs)03d %(thread)d %(levelname)-8s %(name)-20s %(message)s',
datefmt='%Y%m%d %H:%M:%S',
filename=cm.get("ts.log_filename"))
# Display startup info
getLogger().info("Starting Testerman Server %s" %
(Versions.getServerVersion()))
getLogger().info("Web Service (Ws) listening on tcp://%s:%s" %
(cm.get("interface.ws.ip"), cm.get("interface.ws.port")))
getLogger().info("Client events (Xc) listening on tcp://%s:%s" %
(cm.get("interface.xc.ip"), cm.get("interface.xc.port")))
getLogger().info("Log manager (Il) listening on tcp://%s:%s" %
(cm.get("interface.il.ip"), cm.get("interface.il.port")))
getLogger().info("Component manager (Ih) listening on tcp://%s:%s" %
(cm.get("interface.ih.ip"), cm.get("interface.ih.port")))
getLogger().info("Using TACS at tcp://%s:%s" %
(cm.get("tacs.ip"), cm.get("tacs.port")))
items = cm.getKeys()
items.sort()
for k in items:
getLogger().info("Using %s = %s" % (str(k), cm.get(k)))
# Now we can daemonize if needed
if cm.get("ts.daemonize"):
if pidfile:
getLogger().info("Daemonizing, using pid file %s..." % pidfile)
else:
getLogger().info("Daemonizing...")
Tools.daemonize(pidFilename=pidfile, displayPid=True)
# Main start
cm.set_transient("ts.pid", os.getpid())
try:
serverThread = XmlRpcServerThread() # Ws server
FileSystemManager.initialize()
EventManager.initialize(
) # Xc server, Ih server [TSE:CH], Il server [TSE:TL]
ProbeManager.initialize() # Ia client
JobManager.initialize() # Job scheduler
serverThread.start()
getLogger().info("Started.")
while 1:
time.sleep(1)
except KeyboardInterrupt:
getLogger().info("Shutting down Testerman Server...")
except Exception as e:
sys.stderr.write("Unable to start server: %s\n" % str(e))
getLogger().critical("Unable to start server: " + str(e))
serverThread.stop()
JobManager.finalize()
ProbeManager.finalize()
EventManager.finalize()
FileSystemManager.finalize()
getLogger().info("Shut down.")
logging.shutdown()
Tools.cleanup(cm.get("ts.pid_filename"))