def find_file(obj):
"""Find the absolute path to the file where an object was defined.
This is essentially a robust wrapper around `inspect.getabsfile`.
Returns None if no file can be found.
Parameters
----------
obj : any Python object
Returns
-------
fname : str
The absolute path to the file where the object was defined.
"""
obj = _get_wrapped(obj)
fname = None
try:
fname = inspect.getabsfile(obj)
except TypeError:
# For an instance, the file that matters is where its class was
# declared.
if hasattr(obj, '__class__'):
try:
fname = inspect.getabsfile(obj.__class__)
except TypeError:
# Can happen for builtins
pass
except:
pass
return cast_unicode(fname)
def reloadTypeModule(self, module_name, path=None):
"""Loads/reloads the given module name"""
#path = path or [ os.path.dirname(__file__) ]
m = None
try:
m = ModuleManager().reloadModule(module_name, path)
except:
pass
if m is None:
if module_name in self._modules:
self._modules.pop(module_name)
return
self._modules[module_name] = {}
abs_file = inspect.getabsfile(m)
ms = self.macro_server
for name in dir(m):
if name.startswith("_"):
continue
klass = getattr(m, name)
try:
if not issubclass(klass, ParamType):
continue
except:
continue
if klass in AbstractParamTypes:
continue
if inspect.getabsfile(klass) != abs_file:
continue
t = klass(ms, name)
self.addType(t)
def sage_getfile(obj):
r"""
Get the full file name associated to ``obj`` as a string.
INPUT: ``obj``, a Sage object, module, etc.
EXAMPLES::
sage: from sagenb.misc.sageinspect import sage_getfile
sage: sage_getfile(sage.rings.rational)[-23:]
'sage/rings/rational.pyx'
sage: sage_getfile(Sq)[-41:]
'sage/algebras/steenrod_algebra_element.py'
AUTHOR:
- Nick Alexander
"""
# We try to extract from docstrings, because Python's inspect
# will happily report compiled .so files
d = inspect.getdoc(obj)
pos = _extract_embedded_position(d)
if pos is not None:
(_, filename, _) = pos
return filename
# The instance case
if isclassinstance(obj):
return inspect.getabsfile(obj.__class__)
# No go? fall back to inspect.
return inspect.getabsfile(obj)
def getmodule(object):
"""Return the module an object was defined in, or None if not found."""
if ismodule(object):
return object
if hasattr(object, '__module__'):
return sys.modules.get(object.__module__)
try:
file = getabsfile(object)
except TypeError:
return None
if file in modulesbyfile:
return sys.modules.get(modulesbyfile[file])
for module in sys.modules.values():
if hasattr(module, '__file__'):
modulesbyfile[
os.path.realpath(
getabsfile(module))] = module.__name__
if file in modulesbyfile:
return sys.modules.get(modulesbyfile[file])
main = sys.modules['__main__']
if not hasattr(object, '__name__'):
return None
if hasattr(main, object.__name__):
mainobject = getattr(main, object.__name__)
if mainobject is object:
return main
builtin = sys.modules['__builtin__']
if hasattr(builtin, object.__name__):
builtinobject = getattr(builtin, object.__name__)
if builtinobject is object:
return builtin
def find_file(obj):
"""Find the absolute path to the file where an object was defined.
This is essentially a robust wrapper around `inspect.getabsfile`.
Returns None if no file can be found.
Parameters
----------
obj : any Python object
Returns
-------
fname : str
The absolute path to the file where the object was defined.
"""
# get source if obj was decorated with @decorator
if safe_hasattr(obj, "__wrapped__"):
obj = obj.__wrapped__
fname = None
try:
fname = inspect.getabsfile(obj)
except TypeError:
# For an instance, the file that matters is where its class was
# declared.
if hasattr(obj, "__class__"):
try:
fname = inspect.getabsfile(obj.__class__)
except TypeError:
# Can happen for builtins
pass
except: # pylint:disable=bare-except
pass
return cast_unicode(fname)
def __init__(self, *args, **kwargs):
# These can be overridden by self.get_opts
self.f77 = ""
self.cc = ""
self.pplist = ""
self.usr_lflags = ""
self.ifmpi = True
self.source_root = os.path.dirname(os.path.dirname(inspect.getabsfile(self.__class__)))
self.examples_root = os.path.dirname(inspect.getabsfile(self.__class__))
self.makenek = os.path.join(self.source_root, 'bin', 'makenek')
self.tools_root = os.path.join(self.source_root, 'tools')
self.tools_bin = os.path.join(self.source_root, 'bin')
self.log_root = ""
self.verbose = True
self.serial_procs = 1
self.parallel_procs = 2
self.size_params = {}
# These are overridden by method decorators (pn_pn_serial, pn_pn_parallel,
# pn_pn_2_serial, and pn_pn_2_parallel)
self.log_suffix = ""
self.mpi_procs = None
# Empy list of delayed fails
self._delayed_failures = []
self.get_opts()
unittest.TestCase.__init__(self, *args, **kwargs)
def storeISubProviderImplementations(startModule):
""" This functions gets a loaded module as input, and stores all the
ISubProvider implementation under it. The function uses the inspect
module functions in order to do most of the work. If the object is
module, we call ourself again, with the object as parameter. If the
object is class, it will be stored in the AllAvaliableProviders list
if:
1. Is a SubClass of ISubProvider
2. Is not ISubProvider itself
3. His PROVIDER_NAME value doesn't starts in "Base - "
On any failure, we proceed to the next module.
"""
# The directory in which the module file (py) seats
start_module_path = os.path.split(inspect.getabsfile(startModule))[0]
# Lambda to check if the current object is placed under the start module
_isUnderPathOfStartModule = lambda current: \
start_module_path in os.path.split(inspect.getabsfile(current))[0]
# inspect.getmembers gives us a (key, value) tuple for each object
for name, obj in inspect.getmembers(startModule):
try:
# In order to avoid checking python infrastracture modules, we
# check if the current module is located under the start module
# directory.
if inspect.ismodule(obj) and _isUnderPathOfStartModule(obj):
storeISubProviderImplementations(obj)
elif inspect.isclass(obj):
if issubclass(obj, ISubProvider) and obj is not ISubProvider:
if ('Base - ' not in obj.PROVIDER_NAME and
obj not in AllAvaliableProviders):
AllAvaliableProviders.append(obj)
except Exception as eX:
WriteDebug( 'Provider load Failure => %s' % eX )
def save_timestamped_config_and_mw_links_list(config, mw_link_OID_listing):
""" Save pySNMPdaq config and MW link OID listing in a unified file """
import inspect
# Build filename for unified config file
filename = "config_" + config.FILENAME_PREFIX + "_" + datetime.utcnow().strftime("%Y%m%d_%H%M") + ".py"
# Build absolute filename for unified config file
absfilename = path.join(config.DATA_DIR, config.CONFIG_ARCHIVE_DIR, filename)
# Get absolute filename from module 'config'
fn_config = inspect.getabsfile(config)
logging.debug("Config filename to read from %s", fn_config)
# Get absolute filename from module 'mw_link_OID_listing'
fn_mw_link_OID_listing = inspect.getabsfile(mw_link_OID_listing)
logging.debug("mw_link_OID_listing filename to read from %s", fn_mw_link_OID_listing)
# Concatenate the content of the two files into one unified config file
with open(absfilename, "wb") as outfile:
for f in [fn_config, fn_mw_link_OID_listing]:
with open(f, "rb") as infile:
outfile.write(infile.read())
outfile.write("\n\n") # add some vertical space
logging.info("Config written to %s", absfilename)
# Put a copy of the config file to the data outbox dir (if desired)
if config.PUT_DATA_TO_OUT_DIR:
import shutil
shutil.copy(absfilename, path.join(config.DATA_DIR, config.DATA_OUT_DIR))
# Return the filename of the unified config file
return filename
def mygetmodule(object):
"""Return the module an object was defined in, or None if not found."""
import os.path
if inspect.ismodule(object):
return object
if hasattr(object, "__module__"):
return sys.modules.get(object.__module__)
try:
file = inspect.getabsfile(object)
except TypeError:
return None
if modulesbyfile.has_key(file):
return sys.modules.get(modulesbyfile[file])
for module in sys.modules.values():
if inspect.ismodule(module) and hasattr(module, "__file__"): # check if value is indeed a module
modulesbyfile[os.path.realpath(inspect.getabsfile(module))] = module.__name__
if modulesbyfile.has_key(file):
return sys.modules.get(modulesbyfile[file])
main = sys.modules["__main__"]
if not hasattr(object, "__name__"):
return None
if hasattr(main, object.__name__):
mainobject = getattr(main, object.__name__)
if mainobject is object:
return main
builtin = sys.modules["__builtin__"]
if hasattr(builtin, object.__name__):
builtinobject = getattr(builtin, object.__name__)
if builtinobject is object:
return builtin
def _patched_getmodule(object, _filename=None):
"""Return the module an object was defined in, or None if not found.
This replicates the functionality of the stdlib `inspect.getmodule`
function but includes a fix for a bug present in Python 3.1 and 3.2.
"""
#these imports mock up what would otherwise have been in inspect
import sys
import os
from inspect import modulesbyfile, _filesbymodname, getabsfile, ismodule
if ismodule(object):
return object
if hasattr(object, '__module__'):
return sys.modules.get(object.__module__)
# Try the filename to modulename cache
if _filename is not None and _filename in modulesbyfile:
return sys.modules.get(modulesbyfile[_filename])
# Try the cache again with the absolute file name
try:
file = getabsfile(object, _filename)
except TypeError:
return None
if file in modulesbyfile:
return sys.modules.get(modulesbyfile[file])
# Update the filename to module name cache and check yet again
# Copy sys.modules in order to cope with changes while iterating
# This is where the fix is made - the adding of the "list" call:
for modname, module in list(sys.modules.items()):
if ismodule(module) and hasattr(module, '__file__'):
f = module.__file__
if f == _filesbymodname.get(modname, None):
# Have already mapped this module, so skip it
continue
_filesbymodname[modname] = f
f = getabsfile(module)
# Always map to the name the module knows itself by
modulesbyfile[f] = modulesbyfile[
os.path.realpath(f)] = module.__name__
if file in modulesbyfile:
return sys.modules.get(modulesbyfile[file])
# Check the main module
main = sys.modules['__main__']
if not hasattr(object, '__name__'):
return None
if hasattr(main, object.__name__):
mainobject = getattr(main, object.__name__)
if mainobject is object:
return main
# Check builtins
builtin = sys.modules['builtins']
if hasattr(builtin, object.__name__):
builtinobject = getattr(builtin, object.__name__)
if builtinobject is object:
return builtin
def _getlocalmembernames(self, module, predicate=None):
ret = []
modulepath, tail = os.path.split(inspect.getabsfile(module))
for n, v in inspect.getmembers(module, predicate):
if inspect.isbuiltin(v):
continue # ignore builtin functions
try:
memberpath, tail = os.path.split(inspect.getabsfile(v))
except TypeError:
continue # ignore builtin modules
if memberpath == modulepath:
ret.append(n)
return ret
def getmodule_2_5(object, _filename=None):
"""Return the module an object was defined in, or None if not found."""
global modulesbyfile
global _filesbymodname
if inspect.ismodule(object):
return object
if hasattr(object, '__module__'):
return sys.modules.get(object.__module__)
# Try the filename to modulename cache
if _filename is not None and _filename in modulesbyfile:
return sys.modules.get(modulesbyfile[_filename])
# Try the cache again with the absolute file name
try:
file = inspect.getabsfile(object)
except TypeError:
return None
if file in modulesbyfile:
return sys.modules.get(modulesbyfile[file])
# Update the filename to module name cache and check yet again
# Copy sys.modules in order to cope with changes while iterating
for modname, module in sys.modules.items():
if inspect.ismodule(module) and hasattr(module, '__file__'):
f = module.__file__
if f == _filesbymodname.get(modname, None):
# Have already mapped this module, so skip it
continue
_filesbymodname[modname] = f
f = inspect.getabsfile(module)
# Always map to the name the module knows itself by
modulesbyfile[f] = modulesbyfile[
os.path.realpath(f)] = module.__name__
if file in modulesbyfile:
return sys.modules.get(modulesbyfile[file])
# Check the main module
main = sys.modules['__main__']
if not hasattr(object, '__name__'):
return None
if hasattr(main, object.__name__):
mainobject = getattr(main, object.__name__)
if mainobject is object:
return main
# Check builtins
builtin = sys.modules['__builtin__']
if hasattr(builtin, object.__name__):
builtinobject = getattr(builtin, object.__name__)
if builtinobject is object:
return builtin
def sys_path():
"""Detect installation path of ecpy.
Automtically called, DOES NOT use directly. Use ecpy_path to get the path
to the ecpy directory.
"""
import ecpy
# Hiding current app_directory.ini to avoid losing user choice.
path = os.path.dirname(getabsfile(ecpy))
pref_path = os.path.join(path, APP_PREFERENCES)
app_dir = os.path.join(pref_path, APP_DIR_CONFIG)
new = os.path.join(pref_path, '_' + APP_DIR_CONFIG)
# If a hidden file exists already assume it is because previous test
# failed and do nothing.
if os.path.isfile(app_dir) and not os.path.isfile(new):
os.rename(app_dir, new)
global ECPY
ECPY = path
yield
# Remove created app_directory.ini and put hold one back in place.
app_dir = os.path.join(pref_path, APP_DIR_CONFIG)
if os.path.isfile(app_dir):
os.remove(app_dir)
# Put user file back in place.
protected = os.path.join(pref_path, '_' + APP_DIR_CONFIG)
if os.path.isfile(protected):
os.rename(protected, app_dir)
def __init__(self, name, args=None, return_type=None, callback=None,
subworkflow=None, symmetric=False, wizard=None):
self.name = name
self.package_identifier = None
self.parameters = args
self.return_type = return_type
self.callback = self.subworkflow = None
self.usable_in_command = True
if callback is not None and subworkflow is not None:
raise ValueError("VariableOperation() got both callback and "
"subworkflow parameters")
elif callback is not None:
self.callback = callback
elif subworkflow is not None:
caller = inspect.currentframe().f_back
package = os.path.dirname(inspect.getabsfile(caller))
self.subworkflow = subworkflow.format(package_dir=package)
elif wizard is None:
raise ValueError("VariableOperation() got neither callback nor "
"subworkflow parameters")
else:
self.usable_in_command = False
if self.usable_in_command:
if self.parameters is None:
raise TypeError("missing parameter 'args'")
if self.return_type is None:
raise TypeError("missing parameter 'return_type")
self.symmetric = symmetric
self.wizard = wizard
def test_text_doc(self):
result, doc_loc = get_pydoc_text(pydoc_mod)
expected_text = expected_text_pattern % (
(doc_loc,) +
expected_text_data_docstrings +
(inspect.getabsfile(pydoc_mod),))
self.assertEqual(expected_text, result)
def warnAboutFunction(offender, warningString):
"""
Issue a warning string, identifying C{offender} as the responsible code.
This function is used to deprecate some behavior of a function. It differs
from L{warnings.warn} in that it is not limited to deprecating the behavior
of a function currently on the call stack.
@param function: The function that is being deprecated.
@param warningString: The string that should be emitted by this warning.
@type warningString: C{str}
@since: 11.0
"""
# inspect.getmodule() is attractive, but somewhat
# broken in Python < 2.6. See Python bug 4845.
offenderModule = sys.modules[offender.__module__]
filename = inspect.getabsfile(offenderModule)
lineStarts = list(findlinestarts(offender.func_code))
lastLineNo = lineStarts[-1][1]
globals = offender.func_globals
kwargs = dict(
category=DeprecationWarning,
filename=filename,
lineno=lastLineNo,
module=offenderModule.__name__,
registry=globals.setdefault("__warningregistry__", {}),
module_globals=None)
if sys.version_info[:2] < (2, 5):
kwargs.pop('module_globals')
warn_explicit(warningString, **kwargs)
def test_help_output_redirect(self):
# issue 940286, if output is set in Helper, then all output from
# Helper.help should be redirected
old_pattern = expected_text_pattern
getpager_old = pydoc.getpager
getpager_new = lambda: (lambda x: x)
self.maxDiff = None
buf = StringIO()
helper = pydoc.Helper(output=buf)
unused, doc_loc = get_pydoc_text(pydoc_mod)
module = "test.pydoc_mod"
help_header = """
Help on module test.pydoc_mod in test:
""".lstrip()
help_header = textwrap.dedent(help_header)
expected_help_pattern = help_header + expected_text_pattern
pydoc.getpager = getpager_new
try:
with captured_output('stdout') as output, \
captured_output('stderr') as err:
helper.help(module)
result = buf.getvalue().strip()
expected_text = expected_help_pattern % (
(doc_loc,) +
expected_text_data_docstrings +
(inspect.getabsfile(pydoc_mod),))
self.assertEqual('', output.getvalue())
self.assertEqual('', err.getvalue())
self.assertEqual(expected_text, result)
finally:
pydoc.getpager = getpager_old
def pfile(self,obj,oname=''):
"""Show the whole file where an object was defined."""
try:
try:
lineno = inspect.getsourcelines(obj)[1]
except TypeError:
# For instances, try the class object like getsource() does
if hasattr(obj,'__class__'):
lineno = inspect.getsourcelines(obj.__class__)[1]
# Adjust the inspected object so getabsfile() below works
obj = obj.__class__
except:
self.noinfo('file',oname)
return
# We only reach this point if object was successfully queried
# run contents of file through pager starting at line
# where the object is defined
ofile = inspect.getabsfile(obj)
if ofile.endswith(('.so', '.dll', '.pyd')):
print 'File %r is binary, not printing.' % ofile
elif not os.path.isfile(ofile):
print 'File %r does not exist, not printing.' % ofile
else:
# Print only text files, not extension binaries. Note that
# getsourcelines returns lineno with 1-offset and page() uses
# 0-offset, so we must adjust.
page.page(self.format(open(ofile).read()),lineno-1)
def getdocloc(self, object):
"""Return the location of module docs or None"""
try:
file = inspect.getabsfile(object)
except TypeError:
file = '(built-in)'
docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS)
basedir = os.path.join(sys.base_exec_prefix, "lib",
"python%d.%d" % sys.version_info[:2])
if (isinstance(object, type(os)) and
(object.__name__ in ('errno', 'exceptions', 'gc', 'imp',
'marshal', 'posix', 'signal', 'sys',
'_thread', 'zipimport') or
(file.startswith(basedir) and
not file.startswith(os.path.join(basedir, 'site-packages')))) and
object.__name__ not in ('xml.etree', 'test.pydoc_mod')):
if docloc.startswith("http://"):
docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__)
else:
docloc = os.path.join(docloc, object.__name__ + ".html")
else:
docloc = None
return docloc
def source_file(self):
if self._source_file is not Lazy: return self._source
try:
self._source_file = inspect.getabsfile(self._obj)
except (TypeError, OSError):
self._source_file = None
return self._source_file
def test_text_doc(self):
result, doc_loc = get_pydoc_text(pydoc_mod)
expected_text = expected_text_pattern % \
(inspect.getabsfile(pydoc_mod), doc_loc)
if result != expected_text:
print_diffs(expected_text, result)
self.fail("outputs are not equal, see diff above")
def addDefinition(self,d):
''' initialize definition objects'''
self.definition = d.__name__
self.cwd = os.path.dirname(inspect.getabsfile(d))
d = update(d)
self.d = d
log.debug("reading %s" % self.definition)
self.compdata = self.fixPropertyId(self.d.componentData['properties'])
self.dsdata = self.fixPropertyId(self.d.datasourceData['properties'])
self.zenpackroot = self.d.zenpackroot
self.zenpackbase = self.d.zenpackbase
self.zenpackname = "%s.%s" % (self.zenpackroot, self.zenpackbase)
self.componentClass = self.d.component
self.singular = self.d.componentData['singular']
self.plural = self.d.componentData['plural']
self.manual = self.d.addManual
self.datasourceClass = '%sDataSource' % d.component
if self.zenpackname not in self.packs.keys():
log.debug("adding %s to packs" % self.zenpackname)
self.packs[self.zenpackname] = {
'constructs' : {},
'builder' : ZenPackBuilder(self.cwd, self.zenpackname, self.indent),
'properties' : {},
'zenpack': None,
}
self.packs[self.zenpackname]['constructs'][self.definition] = {'component': None, 'datasource': None}
self.packs[self.zenpackname]['properties'][self.definition] = {'component': self.compdata, 'datasource': self.dsdata }
self.packs[self.zenpackname]['builder'].addHelper(self.d.component, self.definition)
self.buildComponent()
if self.d.createDS == True:
self.buildDataSource()
def get_caller(self):
"""Return name of calling ZenPack"""
frm = inspect.stack()[-1]
file_name = inspect.getabsfile(frm[0])
first_part = file_name[file_name.find('ZenPacks.'):]
zp_name = first_part[:first_part.find('/')]
return zp_name
def __init__(self, cache_engine=Configurator.CACHE_ENGINE_IN_MEMORY, base_dir=None, cls=None, logger=None):
super(Topology, self).__init__()
if logger:
self.logger = logger
# TODO: read the cache engines from the configuration and allow passing parameters if needed
#Toplogy is support self discovering of its needed modules but it can be supplied in init
if not base_dir:
base_dir = self.__module__
if not cls:
cls = self.__class__
self.cache_engines = {Configurator.CACHE_ENGINE_IN_MEMORY: InMemoryCache, Configurator.CACHE_ENGINE_MYSQL_CACHE: MySQLCache}
self.transformers = Transformers()
self.base_dir = '.'.join(base_dir.split('.')[:-1])
self._modules_dir = self.base_dir
self._templates_dir = os.path.dirname(inspect.getabsfile(cls))
cache_engine_params = None
if type(cache_engine) == dict:
cache_engine_name = cache_engine['cache_engine_name']
cache_engine_params = cache_engine['params']
else:
cache_engine_name = cache_engine
cache_engine_class = self.cache_engines.get(cache_engine_name, InMemoryCache)
self.cache_engine = cache_engine_class(cache_engine_params)
self._execution_plan = ExecutionPlan()
self.query_engine = QueryEngineFactory.get_query_engine(self._modules_dir, self.cache_engine, self._execution_plan, self.logger)
self.query_engine.set_templates_dir(self._templates_dir)
self.logger.debug("Topology {0} was instantiated, modules_dir: {1}, templates_dir: {2}".
format(type(self).__name__, self._modules_dir, self._templates_dir))
self._topology_cache_ttl = Configurator.CACHE_DB_KEY_EXPIRE
def __init__(self, *args, **kwargs):
# These can be overridden by self.get_opts
self.f77 = "gfortran"
self.cc = "gcc"
self.ifmpi = False
self.verbose = True
self.source_root = ''
#self.examples_root = os.path.join(os.path.dirname(inspect.getabsfile(self.__class__)), 'examples')
self.examples_root = os.path.join(os.path.dirname(inspect.getabsfile(self.__class__)))
self.tools_root = ''
self.tools_bin = ''
self.log_root = ''
self.makenek = ''
self.serial_procs = 1
self.parallel_procs = 4
# These are overridden by method decorators (pn_pn_serial, pn_pn_parallel,
# pn_pn_2_serial, and pn_pn_2_parallel)
self.log_suffix = ""
self.mpi_procs = None
# Empy list of delayed warnings and fails
self._delayed_warnings = []
self._delayed_failures = []
self.get_opts()
unittest.TestCase.__init__(self, *args, **kwargs)
def get_script_dir(follow_symlinks=True):
global script_dir
if script_dir is not None:
return script_dir
if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = os.path.abspath(sys.executable)
else:
path = inspect.getabsfile(get_script_dir)
if follow_symlinks:
path = os.path.realpath(path)
script_dir = os.path.dirname(path)
return script_dir
def get_script_dir(follow_symlinks=True):
"""Checks where the script is being executed from to verify the imports
will work properly."""
if getattr(sys, 'frozen', False):
path = os.path.abspath(sys.executable)
else:
path = inspect.getabsfile(get_script_dir)
if follow_symlinks:
path = os.path.realpath(path)
return os.path.dirname(path)
def test_html_doc(self):
result, doc_loc = get_pydoc_html(pydoc_mod)
mod_file = inspect.getabsfile(pydoc_mod)
if sys.platform == 'win32':
import nturl2path
mod_url = nturl2path.pathname2url(mod_file)
else:
mod_url = mod_file
expected_html = expected_html_pattern % (mod_url, mod_file, doc_loc)
if result != expected_html:
print_diffs(expected_html, result)
self.fail("outputs are not equal, see diff above")
def _get_custom_packages_filelist(self, custom_packages):
'''This method returns all filenames where the given custom packages reside on disk.'''
folders = set()
for custom_package in custom_packages:
package = importlib.import_module(custom_package)
filename = inspect.getabsfile(package)
folders.add(filename[:filename.rfind(os.path.sep)])
return folders
def get_script_dir(follow_symlinks=True):
"""https://stackoverflow.com/questions/3718657/how-to-properly-determine-current-script-directory/22881871#22881871"""
import inspect
import os
import sys
if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = os.path.abspath(sys.executable)
else:
path = inspect.getabsfile(get_script_dir)
if follow_symlinks:
path = os.path.realpath(path)
return os.path.dirname(path)
def get_script_dir(self, follow_symlinks=True):
if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = os.path.abspath(sys.executable)
else:
path = inspect.getabsfile(self.get_script_dir)
if follow_symlinks:
path = os.path.realpath(path)
path = os.path.dirname(path)
if os.path.isfile(path):
return path
else:
return ""
def DumpAllObjects(cls, tFindFilter=[], oStream=None):
'''For example, call:
# with open('/mydir/mydump.log','wb') as f:
# sc_mspysys.DumpAllObjects(tFindFilter=['edis','sc_', 'c_clients', 'Connection', 'Lock', 'Thread', 'Queue', 'Event',],oStream=f)
'''
exclude = [
"function",
"type",
"list",
"dict",
"tuple",
"wrapper_descriptor",
"module",
"method_descriptor",
"member_descriptor",
"instancemethod",
"builtin_function_or_method",
"frame",
"classmethod",
"classmethod_descriptor",
"_Environ",
"MemoryError",
"_Printer",
"_Helper",
"getset_descriptor",
]
gc.collect()
oo = gc.get_objects()
if oStream is None: oStream = sys.stdout
for o in oo:
try:
if getattr(o, "__class__", None):
name = o.__class__.__name__
if name not in exclude:
filename = ''
try:
filename = inspect.getabsfile(o.__class__)
except:
filename = repr(o.__class__)
bFiltered = False
if len(tFindFilter) > 0:
bFiltered = True
for item in tFindFilter:
if name.find(item) >= 0:
bFiltered = False
break
if not bFiltered:
oStream.write(
"Object of class: {0} ... defined in file: {1}\n"
.format(name, filename))
oStream.flush()
except:
pass
def __init__(self, classobj, location):
self.location = location
self.methods = {}
self.comments = inspect.getdoc(classobj)
module = inspect.getmodule(classobj)
self.path = inspect.getabsfile(module)
self.errors = ""
self.properties = []
for key, val in classobj.__dict__.items():
if key.startswith("_"):
continue
self.properties.append(key)
def getsafeabsfile(obj):
r"""SUMMARY
@Arguments:
- `obj`:
@Return:
"""
try:
return _inspect.getabsfile(obj)
except TypeError:
return ''
def load_app(model_name='app'):
instance = import_module(model_name)
model_path = getabsfile(instance)
if model_path.endswith('__init__.py'):
dir_path = os.path.dirname(model_path)
for _dir in os.listdir(dir_path):
if '__pycache__' == _dir or '__init__.py' == _dir:
continue
if _dir.find('.html') > 0:
continue
_model_name = _dir.rstrip('.py').rstrip('.pyc')
load_app('.'.join([model_name, _model_name]))
def wrap(f, autodeclare, *args, **kwargs):
"""
Wrapper function returned by the cython_compile and cython_compile_autodeclare decorators
:param f: Function to compile
:type f: function
:param py2pyx_func: py2pyx or py2pyx_autodeclare
:type py2pyx_func: function
"""
# Generate name: "c:\documents\project\mymodule.py" -> mymodule_myfunction
# When called from ipython notebooks, filename is an object e.g: "<ipython-input-12-e897f9fefc0c>"
# therefore <,>,- is removed to make it a legal python module name
name = os.path.relpath(
inspect.getabsfile(f),
os.getcwd()).replace(".py",
"")
name = name.replace("<", "").replace(">", "").replace("-", "")
name = "%s_%s" % (name, f.func_name)
module = name.replace(os.path.sep, ".")
# import compiled module if exists, otherwise compile and import
try:
cmodule = __import__(module)
except ImportError:
# Generate pyrex code lines
if autodeclare:
pyx_lines = py2pyx_autodeclare(f, args, kwargs.copy())
else:
# source lines except '@cython_compile'
source_lines = inspect.getsourcelines(f)[0][1:]
pyx_lines = py2pyx(source_lines)
# Write pyrex code lines to .pyx file
pyx_filename = name + ".pyx"
with open(pyx_filename, 'w') as fid:
fid.writelines(pyx_lines)
# compile, import compiled module and delete temporary files
cmodule = compile_and_cleanup(module, pyx_filename)
try:
cf = getattr(cmodule, f.func_name)
if kwargs == {}:
return cf(*args)
else:
return cf(*args, **kwargs)
except AttributeError:
warnings.warn(
"Compilation or import of %s failed. Python function used instead" %
f)
return f(*args, **kwargs)
def __init__(self,
cache_engine=Configurator.CACHE_ENGINE_IN_MEMORY,
base_dir=None,
cls=None,
logger=None):
super(Topology, self).__init__()
# Initializing django here in case Hydro is not used within a django project
if 'DJANGO_SETTINGS_MODULE' not in os.environ:
if not settings.configured:
settings.configure()
django.setup()
if logger:
self.logger = logger
# TODO: read the cache engines from the configuration and allow passing parameters if needed
# Topology is support self discovering of its needed modules but it can be supplied in init
if not base_dir:
base_dir = self.__module__
if not cls:
cls = self.__class__
self.cache_engines = CacheEnginesFactory.get_cache_engines()
self.transformers = Transformers()
self.base_dir = '.'.join(base_dir.split('.')[:-1])
self._modules_dir = self.base_dir
self._templates_dir = os.path.dirname(inspect.getabsfile(cls))
cache_engine_params = None
if type(cache_engine) == dict:
cache_engine_name = cache_engine['cache_engine_name']
cache_engine_params = cache_engine['params']
else:
cache_engine_name = cache_engine
cache_engine_class = self.cache_engines.get(cache_engine_name)
if not cache_engine_class:
cache_engine_class = self.cache_engines.get(
Configurator.CACHE_ENGINE_IN_MEMORY)
self.cache_engine = cache_engine_class(cache_engine_params)
self._execution_plan = ExecutionPlan()
self.query_engine = QueryEngineFactory.get_query_engine(
self._modules_dir, self.cache_engine, self._execution_plan,
self.logger)
self.query_engine.set_templates_dir(self._templates_dir)
self.logger.debug(
"Topology {0} was instantiated, modules_dir: {1}, templates_dir: {2}"
.format(
type(self).__name__, self._modules_dir, self._templates_dir))
self._topology_cache_ttl = Configurator.CACHE_DB_KEY_EXPIRE
def main():
explicit = False
if len(sys.argv) == 2:
explicit = True
srctree = os.path.abspath(sys.argv[1])
else:
srctree = os.path.dirname(inspect.getabsfile(c7n))
update_headers(srctree)
if not explicit:
update_headers(os.path.abspath('tests'))
update_headers(os.path.abspath('ftests'))
def testRequest(self):
runner.run(self._func, 'p', ('foo', 'bar'), host=_HOST, port=_PORT)
response = urllib.request.urlopen('http://%s:%s/profile' %
(_HOST, _PORT))
response_data = gzip.decompress(response.read())
stats = json.loads(response_data.decode('utf-8'))
curr_filename = inspect.getabsfile(inspect.currentframe())
self.assertEqual(stats['p']['objectName'],
'_func @ %s (function)' % curr_filename)
self.assertTrue(len(stats['p']['callStats']) > 0)
self.assertTrue(stats['p']['totalTime'] > 0)
self.assertTrue(stats['p']['primitiveCalls'] > 0)
self.assertTrue(stats['p']['totalCalls'] > 0)
def run(self):
"""Run command."""
try:
import gt4py
except ImportError:
return
target_dir = "{install_dir}/_external_src/gridtools".format(
install_dir=os.path.dirname(inspect.getabsfile(gt4py)))
rm_cmd = "rm -Rf {target_dir}".format(target_dir=target_dir)
print("Deleting sources...\n$ {}".format(rm_cmd))
subprocess.run(rm_cmd.split())
def scan(base_dir: str = "", regx: str = r"", project_dir: str = "") -> None:
if project_dir:
work_dir = project_dir
else:
ft = inspect.currentframe()
fts = inspect.getouterframes(ft)
entrance = fts[-1]
work_dir = os.path.dirname(inspect.getabsfile(entrance[0]))
modules = _load_all_modules(work_dir, base_dir, regx)
for mname in modules:
__logger.info(f"Import controllers from module: {mname}")
_import_module(mname)
def rel_path(relative_filename):
"""Returns the full path of the file relative to the caller module.
:param relative_filename: target filename relative to the caller's containing folder.
:return: the full path of the target relative file.
"""
# Extract the filename of the caller's stack frame.
caller_frame = inspect.stack()[1]
try:
caller_filepath = inspect.getabsfile(caller_frame[0])
finally:
del caller_frame # Force remove frame reference to prevent garbage-collection issues.
return os.path.join(os.path.dirname(os.path.realpath(caller_filepath)),
relative_filename)
def _get_novaclient_version():
"""Read version from versioninfo file."""
mod_abspath = inspect.getabsfile(inspect.currentframe())
novaclient_path = os.path.dirname(mod_abspath)
version_path = os.path.join(novaclient_path, 'versioninfo')
if os.path.exists(version_path):
version = open(version_path).read().strip()
else:
version = "Unknown, couldn't find versioninfo file at %s"\
% version_path
return version
def __new__(meta, name, bases, dictionary):
T = slotmachine.SchemaMetaMachine.__new__(meta, name, bases,
dictionary)
if T.__name__ == 'Item' and T.__module__ == __name__:
return T
T.__already_inherited__ += 1
if T.__already_inherited__ >= 2:
raise NoInheritance("already inherited from item once: "
"in-database inheritance not yet supported")
if T.typeName is None:
T.typeName = normalize(qual(T))
if T.schemaVersion is None:
T.schemaVersion = 1
if not T.__legacy__ and T.typeName in _typeNameToMostRecentClass:
# Let's try not to gc.collect() every time.
gc.collect()
if T.typeName in _typeNameToMostRecentClass:
if T.__legacy__:
return T
otherT = _typeNameToMostRecentClass[T.typeName]
if (otherT.__name__ == T.__name__
and getabsfile(T) == getabsfile(otherT)
and T.__module__ != otherT.__module__):
if len(T.__module__) < len(otherT.__module__):
relmod = T.__module__
else:
relmod = otherT.__module__
raise RuntimeError("Use absolute imports; relative import"
" detected for type %r (imported from %r)" %
(T.typeName, relmod))
raise RuntimeError(
"2 definitions of axiom typename %r: %r %r" %
(T.typeName, T, _typeNameToMostRecentClass[T.typeName]))
_typeNameToMostRecentClass[T.typeName] = T
return T
def coverage_includes(self):
if self.covers is not None:
return self.covers
test_file = inspect.getabsfile(self.func_call.func)
covers = [test_file]
match = Test.TEST_COVERS_MATCH.match(test_file)
if match:
covers.append(
os.path.join(os.path.dirname(os.path.dirname(test_file)),
match.group(1) + '.py'))
return covers
def class_path(cls):
'''Return the path of the class attribute.
Args:
class (cls): Class.
Returns:
str: Absolute path to class source file.
'''
path = inspect.getabsfile(cls)
return path
def wrapper():
try:
func_file = inspect.getfile(func)
except Exception:
try:
func_file = inspect.getabsfile(func)
except Exception:
func_file = 'may be local func: {}'.format(func.__name__)
try:
func()
print("OK", func_file, func.__name__)
except Exception:
print(traceback.format_exc())
def object_path(obj):
'''Return the path of the class object.
Args:
object (obj): Class object.
Returns:
str: Absolute path to class source file.
'''
path = inspect.getabsfile(obj.__class__)
return path
def is_local_code(obj,
*,
module=None,
path: str = None,
name: str = None) -> bool:
import os
import sys
s = sum([module is not None, path is not None, name is not None])
if s == 0:
raise Exception('module, file or name must be given')
elif s > 1:
raise Exception(
'only one of the parameters can be used at the same time')
else:
if module:
return is_local_code(obj, path=inspect.getabsfile(module))
elif name:
return is_local_code(obj, module=sys.modules[name])
elif path:
topmost_dir = os.path.dirname(os.path.realpath(path))
if any([
inspect.isfunction(obj),
inspect.ismodule(obj),
inspect.ismethod(obj)
]):
module_path = os.path.realpath(inspect.getabsfile(obj))
return topmost_dir in module_path
else:
raise NotImplementedError
else:
raise Exception
def _get_position_of_arg(self, arg):
try:
obj = self._getval(arg)
except:
return None, None, None
if isinstance(obj, str):
return obj, 1, None
try:
filename = inspect.getabsfile(obj)
lines, lineno = inspect.getsourcelines(obj)
except (IOError, TypeError) as e:
print('** Error: %s **' % e, file=self.stdout)
return None, None, None
return filename, lineno, lines
def _child_modules(module):
package_path = os.path.dirname(inspect.getabsfile(module))
# Import all child modules
for module_path in filesystem.find_modules(package_path):
_import_from_path(module_path, module.__package__ or module.__name__)
# Import all sub packages
for package_path in filesystem.find_packages(package_path):
_import_from_path(package_path, module.__package__ or module.__name__)
for name, obj in inspect.getmembers(module):
if inspect.ismodule(obj) and is_child_of_module(obj, module):
yield obj
def get_script_dir(follow_symlinks=True):
if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = abspath(executable)
else:
path = getabsfile(get_script_dir)
if follow_symlinks:
path = realpath(path)
dir_ = list(dirname(path))
for _ in range(8):
del dir_[-1]
true_dir = ''
for elm in dir_:
true_dir += elm
return true_dir
def get_script_dir(follow_symlinks=True):
if (sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = os.path.abspath(sys.executable)
print('1',path)
else:
path = inspect.getabsfile(get_script_dir)
print('2',path)
if follow_symlinks:
path = os.path.realpath(path)
print('3',path)
return os.path.dirname(path)
def is_report_handler(report, abs_file=None):
"""Helper function to determine if a certain python object is a valid
report handler"""
import inspect
if not inspect.isclass(report):
return False
if not issubclass(report, TaurusMessageReportHandler):
return False
try:
if inspect.getabsfile(report) != abs_file:
return False
except:
return False
return True
def get_function(session_factory,
name,
role,
log_groups,
project,
account_name,
account_id,
pattern="Traceback"):
"""Lambda function provisioning.
Self contained within the component, to allow for easier reuse.
"""
# Lazy import to avoid runtime dependency
import inspect
import os
import c7n
from c7n.mu import (LambdaFunction, PythonPackageArchive,
CloudWatchLogSubscription)
config = dict(name=name,
runtime='python2.7',
memory_size=512,
timeout=15,
role=role,
description='Custodian Sentry Relay',
events=[
CloudWatchLogSubscription(session_factory, log_groups,
pattern)
])
archive = PythonPackageArchive(
# Directory to lambda file
os.path.join(os.path.dirname(inspect.getabsfile(c7n)), 'ufuncs',
'cwl2sentry.py'),
# Don't include virtualenv deps
lib_filter=lambda x, y, z: ([], []))
archive.create()
archive.add_contents(
'config.json',
json.dumps({
'project': project,
'account_name': account_name,
'account_id': account_id
}))
archive.close()
return LambdaFunction(config, archive)