def _describe_calling_test():
my_fname = os.path.basename(__file__)
stack = traceback.extract_stack()
i = 1
while i < len(stack):
fname, line, caller, code_at_line = stack[i]
fname = ioutil.norm_seps(fname)
if not fname.endswith('/' + my_fname):
if caller.startswith('test') or caller.startswith('setUp') or caller.startswith('tearDown'):
i = fname.find('/test/')
j = fname.rfind('/')
if i > -1:
i += 6
if j > -1:
j += 1
folder = fname[i:j]
fname = os.path.basename(fname)
return caller, folder, fname, line
i += 1
# If we get here, something's wrong. As a fall-back, just report line nums
# and file names for any stuff that looks interesting.
fnames = []
for fname, line, caller, code_at_line in stack:
if 'site-packages' not in fname and 'unittest2' not in fname:
fname = ioutil.norm_seps(fname)
if '/buildscripts/test.py' not in fname and '/unittest' not in fname:
fname = os.path.basename(fname)
if fname != my_fname:
fnames.append('%s(%d)' % (fname, line))
fnames = ','.join(fnames)
i = fnames.rfind('(')
if i > -1:
fnames = fnames[0:i]
return 'unknown_func', 'unknown_component', fnames, line
def find_component_from_within(path):
'''
Find the component that "owns" the data in a particular subtree, no matter
how deeply nested. Return the component name, or None if path doesn't look
like it's inside a component's stuff.
@param path A relative or absolute path, existing or proposed.
'''
path = ioutil.norm_seps(os.path.abspath(path), trailing=False)
sbroot = find_root_from_within(path)
if sbroot:
path = path[len(sbroot):]
if path:
segments = path.split('/')
if len(segments) > 1:
if segments[0].startswith(
component.BUILT_ASPECT_NAME +
'.') or (segments[0]
in _ASPECTS_THAT_APPEAR_IN_COMPONENT_PATH):
return segments[1]
if segments and segments[0] in component.TOP_COMPONENT_ONLY_ASPECTS:
segments = ioutil.norm_seps(sbroot, trailing=False).split('/')
sbname = segments[-1]
comp, br, task = split_name(sbname)
return comp
def __init__(self, root):
'''
Create a Sandbox object that's tied (in theory) to the specified folder.
The folder may or may not exist; it is not created by this call.
@param root An absolute or relative path to an existing folder that
constitutes a sandbox root.
'''
assert (not os.path.isfile(root))
# First get fq path without trailing slash
self._root = ioutil.norm_seps(os.path.abspath(root), trailing=False)
# Last segment is the sandbox name; it can be split into its pieces with
# a simple regex.
self._name = self._root[self._root.rfind('/') + 1:]
self._top, self._branch, self._variant = split_name(self._name)
# Now put root in normal form (trailing slash).
self._root += '/'
# Init config object.
self._cfg = ConfigParser.RawConfigParser()
self._cfg.add_section('settings')
self._cfg.defaults
self._last_config_size = None
self._last_config_time = None
self._load_config()
self._lock = None
self._sandboxtype = sandboxtype.SandboxType(self)
def __init__(self, root):
"""
Create a Sandbox object that's tied (in theory) to the specified folder.
The folder may or may not exist; it is not created by this call.
@param root An absolute or relative path to an existing folder that
constitutes a sandbox root.
"""
assert not os.path.isfile(root)
# First get fq path without trailing slash
self._root = ioutil.norm_seps(os.path.abspath(root), trailing=False)
# Last segment is the sandbox name; it can be split into its pieces with
# a simple regex.
self._name = self._root[self._root.rfind("/") + 1 :]
self._top, self._branch, self._variant = split_name(self._name)
# Now put root in normal form (trailing slash).
self._root += "/"
# Init config object.
self._cfg = ConfigParser.RawConfigParser()
self._cfg.add_section("settings")
self._cfg.defaults
self._last_config_size = None
self._last_config_time = None
self._load_config()
self._lock = None
self._sandboxtype = sandboxtype.SandboxType(self)
def save(path, txt, report=True):
if os.path.isfile(path):
os.remove(path)
f = open(path, 'w')
f.write(txt)
f.close()
if report:
print("Saved %s." % ioutil.norm_seps(path))
def _get_variant_from_path(path):
# Should only be called on str or unicode objects.
assert (hasattr(path, 'lower'))
name = ioutil.norm_seps(path, trailing=False)
i = name.rfind('/')
if i > -1:
name = name[i + 1:]
m = _STANDARD_NAME_PAT.match(name)
if m:
return m.group(3)
raise SandboxNameError(path)
def _get_variant_from_path(path):
# Should only be called on str or unicode objects.
assert(hasattr(path, 'lower'))
name = ioutil.norm_seps(path, trailing=False)
i = name.rfind('/')
if i > -1:
name = name[i + 1:]
m = _STANDARD_NAME_PAT.match(name)
if m:
return m.group(3)
raise SandboxNameError(path)
def find_component_from_within(path):
"""
Find the component that "owns" the data in a particular subtree, no matter
how deeply nested. Return the component name, or None if path doesn't look
like it's inside a component's stuff.
@param path A relative or absolute path, existing or proposed.
"""
path = ioutil.norm_seps(os.path.abspath(path), trailing=False)
sbroot = find_root_from_within(path)
if sbroot:
path = path[len(sbroot) :]
if path:
segments = path.split("/")
if len(segments) > 1:
if segments[0].startswith(component.BUILT_ASPECT_NAME + ".") or (
segments[0] in _ASPECTS_THAT_APPEAR_IN_COMPONENT_PATH
):
return segments[1]
if segments and segments[0] in component.TOP_COMPONENT_ONLY_ASPECTS:
segments = ioutil.norm_seps(sbroot, trailing=False).split("/")
sbname = segments[-1]
comp, br, task = split_name(sbname)
return comp
def find_root_from_within(path):
"""
Find the absolute path to the root of a sandbox from any file or folder
inside a sandbox, no matter how deeply nested. Return the path prefix that
identifies the root, or None if path doesn't look like it's inside a sandbox.
@param path A relative or absolute path, existing or proposed.
"""
path = ioutil.norm_seps(os.path.abspath(path), trailing=False)
segments = path.split("/")
i = len(segments) - 1
while i >= 0:
m = _STANDARD_NAME_PAT.match(segments[i])
if m:
return "/".join(segments[0 : i + 1]) + "/"
i -= 1
def find_root_from_within(path):
'''
Find the absolute path to the root of a sandbox from any file or folder
inside a sandbox, no matter how deeply nested. Return the path prefix that
identifies the root, or None if path doesn't look like it's inside a sandbox.
@param path A relative or absolute path, existing or proposed.
'''
path = ioutil.norm_seps(os.path.abspath(path), trailing=False)
segments = path.split('/')
i = len(segments) - 1
while i >= 0:
m = _STANDARD_NAME_PAT.match(segments[i])
if m:
return '/'.join(segments[0:i + 1]) + '/'
i -= 1
def _discover(self, recurse_filter, visit_filter):
for root, dirs, files in os.walk(self.root):
root = ioutil.norm_folder(root)
relative_root = root[len(self.root):]
items = []
for d in dirs[:]:
if recurse_filter and (not recurse_filter(relative_root + d)):
dirs.remove(d)
else:
items.append(ioutil.norm_seps(relative_root + d, trailing=True))
for f in files:
if C_EXTS_PAT.match(f):
if (not visit_filter) or visit_filter(f):
fname, ext = os.path.splitext(f)
if not ext in self.by_ext:
self.by_ext[ext] = []
self.by_ext[ext].append(relative_root + f)
items.append(relative_root + f)
self.by_folder[relative_root] = items
def find_aspect_from_within(path, with_suffix=False):
'''
Find an aspect name from any file or folder inside a sandbox, no matter how
deeply nested. Return the aspect name or None if path doesn't look like
it's inside an aspect.
@param path A relative or absolute path, existing or proposed.
@param with_suffix If True, the built aspect is returned with its tpv suffix
(e.g., "built.linux_x86-64" instead of "built")
'''
path = ioutil.norm_seps(os.path.abspath(path), trailing=False)
sbroot = find_root_from_within(path)
if sbroot:
path = path[len(sbroot):]
if path:
segments = path.split('/')
if segments:
if segments[0].startswith(component.BUILT_ASPECT_NAME + '.'):
if with_suffix:
return segments[0]
else:
return component.BUILT_ASPECT_NAME
elif segments[0] in _ASPECTS_THAT_APPEAR_IN_ASPECT_PATH:
return segments[0]
def find_aspect_from_within(path, with_suffix=False):
"""
Find an aspect name from any file or folder inside a sandbox, no matter how
deeply nested. Return the aspect name or None if path doesn't look like
it's inside an aspect.
@param path A relative or absolute path, existing or proposed.
@param with_suffix If True, the built aspect is returned with its tpv suffix
(e.g., "built.linux_x86-64" instead of "built")
"""
path = ioutil.norm_seps(os.path.abspath(path), trailing=False)
sbroot = find_root_from_within(path)
if sbroot:
path = path[len(sbroot) :]
if path:
segments = path.split("/")
if segments:
if segments[0].startswith(component.BUILT_ASPECT_NAME + "."):
if with_suffix:
return segments[0]
else:
return component.BUILT_ASPECT_NAME
elif segments[0] in _ASPECTS_THAT_APPEAR_IN_ASPECT_PATH:
return segments[0]
def final_folder(folder):
folder = ioutil.norm_seps(folder)
if folder.endswith('/'):
folder = folder[0:-1]
return folder.split('/')[-1]
def get_deps(rough_sources, rough_dependent_items, rough_source_filter=None, rough_dependent_filter=None):
'''
Get direct and indirect dependencies of all enumerated items on python
modules in sources folders.
@param rough_sources A folder, or a list of folders, that contain modules of
interest (that might or might not be depended *on*).
@param rough_dependent_items One or more items that have dependencies on the
source. These items can be specific files, folders, or python module names
within the source. They serve as the starting point of the dependency
analysis. Can be either a sequence or a string.
@param rough_source_filter A callable that takes a path to a python module.
The module is a potential source module, and will be included in our analysis
(though not necessarily in the final dependency graph) unless the callable
returns False. Example of use: Suppose a folder of python code contains a
subdirectory of unit tests that should be ignored in transitive dependency
analysis. You could exclude this subdirectory with rough_source_filter.
@param rough_dependent_filter A callable that takes a path to a python module.
The module is a candidate for analysis to decide if it depends on any source
modules. The callable returns True if the file should be included in our
analysis and False if not. Example of use: suppose python folder D (dependent)
depends somewhat on files in python folder S (source). Suppose further that
most dependencies are run-time dependencies, but 2 scripts in D have build-
time dependencies instead. If you only wanted a picture of run-time
dependencies, you could pass D as one of the rough_dependent_items to this
function, and use rough_dependent_filter to exclude the 2 scripts with
build-time dependencies.
Return a DependencyInfo named tuple:
.module_info_by_module_name = dict of depended-on-module-name -->
ModuleInfo named tuple:
.full_path = path to depended-on-module
.dependent_files = full paths of files dependent on the module
.module_names_by_importer = dict of full-path-of-dependent-file -->
depended-on module name
Note that although DependencyInfo is a read-only, named tuple, it has
a number of useful methods; it is not just a raw data container. See
top of module for details.
'''
sources = rough_sources
# Allow a single string/unicode as sources as well as a list.
if hasattr(sources, 'lower'): #string or unicode
sources = [sources]
if not sources:
raise Exception('Must specify at least one folder of python source.')
dependent_items = rough_dependent_items
# Allow a single string/unicode as dependent_items as well as a list.
if hasattr(sources, 'lower'): #string or unicode
dependent_items = [dependent_items]
if rough_dependent_filter:
dependent_items = [di for di in dependent_items if rough_dependent_filter(di)]
if not dependent_items:
raise Exception('Must specify at least one folder, file, or module name to start the dependency analysis.')
if _debug:
print('sources = %s\n' % str(sources))
print('dependent_items = %s\n' % str(dependent_items))
sources = [os.path.abspath(src) for src in sources]
bad = []
for src in sources:
if not os.path.isdir(src):
bad.append(src)
if bad:
raise Exception('The following source items are not folders:\n ' + '\n '.join(bad))
sources = [ioutil.norm_seps(src, trailing=True) for src in sources]
possible_modules = find_all_modules(sources)
if _debug:
print('possible modules = %s\n' % ', '.join(sorted(possible_modules.keys())))
if rough_source_filter:
for key in possible_modules.keys()[:]:
if not rough_source_filter(possible_modules[key]):
del(possible_modules[key])
start_folders = [di for di in dependent_items if os.path.isdir(di)]
start_files = [di for di in dependent_items if os.path.isfile(di)]
start_modules = [di for di in dependent_items if di not in start_folders and di not in start_files]
bad = [x for x in start_modules if x not in possible_modules]
if bad:
raise Exception('The following start dependencies are neither folders, files, nor python modules:\n ' + '\n '.join(bad))
start_folders = [ioutil.norm_folder(sf) for sf in start_folders]
start_files = [ioutil.norm_seps(os.path.abspath(sf)) for sf in start_files]
# Guarantee uniqueness. Shouldn't be a problem unless someone was careless
# on cmdline -- but just in case...
modules = list(set(start_modules[:]))
module_names_by_importer = {}
dependent_files_by_module_name = {}
for m in start_modules:
expand(modules, possible_modules[m], possible_modules, module_names_by_importer, dependent_files_by_module_name)
for sf in start_folders:
for folder, dirs, files in os.walk(sf):
for f in files:
if f.endswith('.py'):
start_files.append(ioutil.norm_seps(os.path.abspath(os.path.join(folder, f))))
for sf in start_files:
if (not rough_dependent_filter) or rough_dependent_filter(sf):
# If we haven't already analyzed a particular start file because we
# saw it while expanding dependencies of something in start_modules...
if sf not in module_names_by_importer:
expand(modules, sf, possible_modules, module_names_by_importer, dependent_files_by_module_name)
# If this file is in one of the sources directories, then include it
# as a depended-on file. Otherwise, we just treat the file as a
# source of dependencies, but not a depended on file itself.
folder, fname = os.path.split(sf)
#print('split yielded %s, %s' % (folder, fname))
#print('sources = %s' % sources)
if is_in_sources(folder, sources):
module, ext = os.path.splitext(fname)
if module not in dependent_files_by_module_name:
modules.append(module)
dependent_files_by_module_name[module] = []
if '' not in module_names_by_importer:
module_names_by_importer[''] = []
module_names_by_importer[''].append(module)
# Convert data to output format.
mibmn = {}
for name in modules:
mi = ModuleInfo(possible_modules[name], dependent_files_by_module_name.get(name, []))
mibmn[name] = mi
return DependencyInfo(mibmn, module_names_by_importer)
def join_path(*args):
return ioutil.norm_seps(os.path.join(*args))
def is_vcs_or_test_folder(folder):
folder = ioutil.norm_seps(folder)
if folder.endswith('/'):
folder = folder[0:-1]
folder = folder.split('/')[-1]
return bool(VCS_FOLDER_PAT.match(folder)) or bool(TEST_FOLDER_PAT.match(folder))
def _makeroot(path):
if os.name == 'nt':
return ioutil.norm_seps(os.path.abspath(path), trailing=True)
return path