def test_sys_path_none_builtins():
prevPath = sys.path
#import some builtin modules not previously imported
try:
sys.path = [None] + prevPath
Assert('datetime' not in sys.modules.keys())
import datetime
Assert('datetime' in sys.modules.keys())
sys.path = prevPath + [None]
if not imp.is_builtin('copy_reg'):
Assert('copy_reg' not in sys.modules.keys())
import datetime
import copy_reg
Assert('datetime' in sys.modules.keys())
Assert('copy_reg' in sys.modules.keys())
sys.path = [None]
if not imp.is_builtin('binascii'):
Assert('binascii' not in sys.modules.keys())
import datetime
import copy_reg
import binascii
Assert('datetime' in sys.modules.keys())
Assert('copy_reg' in sys.modules.keys())
Assert('binascii' in sys.modules.keys())
finally:
sys.path = prevPath
def test_sys_path_none_builtins():
prevPath = sys.path
#import some builtin modules not previously imported
try:
sys.path = [None] + prevPath
Assert('datetime' not in list(sys.modules.keys()))
import datetime
Assert('datetime' in list(sys.modules.keys()))
sys.path = prevPath + [None]
if not imp.is_builtin('copy_reg'):
Assert('copy_reg' not in list(sys.modules.keys()))
import datetime
import copyreg
Assert('datetime' in list(sys.modules.keys()))
Assert('copy_reg' in list(sys.modules.keys()))
sys.path = [None]
if not imp.is_builtin('binascii'):
Assert('binascii' not in list(sys.modules.keys()))
import datetime
import copyreg
import binascii
Assert('datetime' in list(sys.modules.keys()))
Assert('copy_reg' in list(sys.modules.keys()))
Assert('binascii' in list(sys.modules.keys()))
finally:
sys.path = prevPath
def test_sys_path_none_builtins(self):
prevPath = sys.path
#import some builtin modules not previously imported
try:
sys.path = prevPath + [None]
if not imp.is_builtin('copy_reg'):
self.assertTrue('copy_reg' not in sys.modules.keys())
import datetime
import copy_reg
self.assertTrue('datetime' in sys.modules.keys())
self.assertTrue('copy_reg' in sys.modules.keys())
sys.path = [None]
if not imp.is_builtin('binascii'):
self.assertTrue('binascii' not in sys.modules.keys())
import datetime
import copy_reg
import binascii
self.assertTrue('datetime' in sys.modules.keys())
self.assertTrue('copy_reg' in sys.modules.keys())
self.assertTrue('binascii' in sys.modules.keys())
finally:
sys.path = prevPath
def test_obscure_functions(self):
import imp
mod = imp.new_module('hi')
assert mod.__name__ == 'hi'
mod = imp.init_builtin(
'hello.world.this.is.never.a.builtin.module.name')
assert mod is None
mod = imp.init_frozen('hello.world.this.is.never.a.frozen.module.name')
assert mod is None
assert imp.is_builtin('sys')
assert not imp.is_builtin(
'hello.world.this.is.never.a.builtin.module.name')
assert not imp.is_frozen(
'hello.world.this.is.never.a.frozen.module.name')
def searchingImport(name, globals=None, locals=None, fromlist=None):
"""Replacement for Python default import
Arguments:
name -- Name of the module to be imported (No default)
Keyword arguments:
globals -- Dictionary containing the globally defined names
(Default: None)
locals -- Dictionary containing the locally defined names
(Default: None)
fromlist -- List of classes to be imported (Default: None)
Returns:
m -- The module object imported
Exceptions:
ImportError is thrown if the module is not found.
"""
try:
m = _original_import(name, globals, locals, fromlist)
if inspect.ismodule(m) and not imp.is_builtin(
m.__name__) and not hasattr(m, '__file__'):
reload(m)
except ImportError:
parent = _determineParent(globals)
q, tail = _findHeadPackage(parent, name)
m = _loadTail(q, tail)
if not fromlist:
return q
if hasattr(m, "__path__"):
_ensureFromList(m, fromlist)
return m
def find_module(self, name, paths=None):
'''
@see: http://www.python.org/dev/peps/pep-0302/
'''
if is_builtin(
name
) == 0 and name not in self.__loading and name not in sys.modules:
self.__loading.add(name)
loader = find_loader(name)
self.__loading.remove(name)
if self._isUnextended(name):
if loader is None:
# If there is no loader but the parent package is extend able we might try to refresh the paths of
# the parent because maybe meanwhile the python path has been updated.
k = name.rfind('.')
if k > 0: inPackage = name[:k]
else: inPackage = None
if not self._isUnextended(inPackage):
_extendPackagePaths(sys.modules[inPackage])
loader = find_loader(name)
elif loader is not None:
if loader.is_package(name): loader = PackageLoader(loader)
elif self.__unittest: loader = ModuleLoader(loader)
return loader
def is_standard_library(self, name):
"""Determine if a module name refers to a module in the python standard library.
Parameters
----------
name : string
Name of the module to check.
Returns
-------
bool
True iff the module is in the standard library.
"""
# Name must be defined.
if name is None:
raise Exception('Name cannot be none')
# Attempt to use python import tools to discover facts about the module.
# If we get an import error, it was definitely not part of the standard library, so return false.
# If we do find the module, check to make sure it's not not a builtin or part of python extras or site-packages.
try:
path = imp.find_module(name)[1]
return bool(imp.is_builtin(name) or ('site-packages' not in path and 'Extras' not in path))
except ImportError:
return False
def test_is_builtin(self):
import sys, imp
for name in sys.builtin_module_names:
assert imp.is_builtin(name)
mod = imp.init_builtin(name)
assert mod
assert mod.__spec__
def get(self, fullname):
if isinstance(fullname, str) and is_builtin(fullname) != 0:
raise ImportError()
imp = self._raw_get(fullname)
if not imp.exists:
raise ImportError()
return imp
def find_builtin_module(self, name):
# XXX frozen packages?
if imp.is_builtin(name):
return None, '', ('', '', BUILTIN_MODULE)
if imp.is_frozen(name):
return None, '', ('', '', FROZEN_MODULE)
return None
def searchingImport(name, globals=None, locals=None, fromlist=None):
"""Replacement for Python default import
Arguments:
name -- Name of the module to be imported (No default)
Keyword arguments:
globals -- Dictionary containing the globally defined names
(Default: None)
locals -- Dictionary containing the locally defined names
(Default: None)
fromlist -- List of classes to be imported (Default: None)
Returns:
m -- The module object imported
Exceptions:
ImportError is thrown if the module is not found.
"""
try:
m = _original_import(name, globals, locals, fromlist)
if inspect.ismodule(m) and not imp.is_builtin(m.__name__) and not hasattr(m,'__file__'):
reload(m)
except ImportError:
parent = _determineParent(globals)
q, tail = _findHeadPackage(parent, name)
m = _loadTail(q, tail)
if not fromlist:
return q
if hasattr(m, "__path__"):
_ensureFromList(m, fromlist)
return m
def find_builtin_module(self, name):
if imp.is_builtin(name):
return (None, '', ('', '', BUILTIN_MODULE))
elif imp.is_frozen(name):
return (None, '', ('', '', FROZEN_MODULE))
else:
return None
def _findModuleInPath(module_name, package_name):
if _debug_module_finding:
print("_findModuleInPath: Enter", module_name, "in", package_name)
# Free pass for built-in modules, the need not exist.
if package_name is None and imp.is_builtin(module_name):
return None, module_name
search_path = getPackageSearchPath(package_name)
if _debug_module_finding:
print("_findModuleInPath: Using search path", search_path)
try:
module_filename = _findModuleInPath2(
module_name = module_name,
search_path = search_path
)
except SyntaxError:
# Warn user, as this is kind of unusual.
warning(
"%s: Module cannot be imported due to syntax errors.",
module_name
if package_name is None else
package_name + '.' + module_name,
)
return None
if _debug_module_finding:
print("_findModuleInPath: _findModuleInPath2 gave", module_filename)
return module_filename
def find_builtin_module(self, name):
if imp.is_builtin(name):
return (None, '', ('', '', BUILTIN_MODULE))
else:
if imp.is_frozen(name):
return (None, '', ('', '', FROZEN_MODULE))
return None
def stdlib_import(name):
"Work around undesired behavior of Python's relative import feature."
try:
return sys.modules[name]
except KeyError:
pass
import imp
if imp.is_builtin(name):
try:
return imp.init_builtin(name)
except Exception:
pass
sys_path = sys.path[:]
sys_path.reverse()
for path in sys_path:
try:
fp, pathname, description = imp.find_module(name, [path])
except ImportError:
pass
else:
try:
return imp.load_module(name, fp, pathname, description)
finally:
if fp is not None:
fp.close()
raise RuntimeError("Cannot import %s module." % name)
def find_builtin_module(self, name):
# XXX frozen packages?
if imp.is_builtin(name):
return None, '', ('', '', BUILTIN_MODULE)
if imp.is_frozen(name):
return None, '', ('', '', FROZEN_MODULE)
return None
def _findModuleInPath(module_name, package_name):
if _debug_module_finding:
print("_findModuleInPath: Enter", module_name, "in", package_name)
# Free pass for built-in modules, the need not exist.
if package_name is None and imp.is_builtin(module_name):
return None, module_name
search_path = getPackageSearchPath(package_name)
if _debug_module_finding:
print("_findModuleInPath: Using search path", search_path)
try:
module_filename = _findModuleInPath2(module_name=module_name,
search_path=search_path)
except SyntaxError:
# Warn user, as this is kind of unusual.
warning(
"%s: Module cannot be imported due to syntax errors.",
module_name if package_name is None else package_name + '.' +
module_name,
)
return None
if _debug_module_finding:
print("_findModuleInPath: _findModuleInPath2 gave", module_filename)
return module_filename
def find_module(self, fullname, path=None):
# Python's default import implementation doesn't handle builtins with '.' in them well, so we handle them here
# as well.
# We allow freeze_overrides to give prefixes that although may be frozen we'll skip so they can be found in
# the PYTHONPATH - this is good for development.
if (not any(fullname.startswith(override) for override in freeze_overrides)) and \
(imp.is_frozen('freezer_package.' + fullname) or imp.is_frozen('freezer.' + fullname) or
(fullname.find('.') != -1 and imp.is_builtin(fullname))):
return self
else:
return None
def addModule(self, name, moduleDir=None) :
module = getPCModule(name, moduleDir)
if module is None :
self.modules[name] = module = PyCheckerModule(name, 0)
if imp.is_builtin(name) == 0:
module.load()
else :
globalModule = globals().get(name)
if globalModule :
module.attributes.extend(dir(globalModule))
else :
self.modules[name] = module
def addModule(self, name) :
module = _allModules.get(name, None)
if module is None :
self.modules[name] = module = Module(name, 0)
if imp.is_builtin(name) == 0 :
module.load()
else :
globalModule = globals().get(name)
if globalModule :
module.attributes.extend(dir(globalModule))
else :
self.modules[name] = module
def addModule(self, name):
module = _allModules.get(name, None)
if module is None:
self.modules[name] = module = Module(name, 0)
if imp.is_builtin(name) == 0:
module.load()
else:
globalModule = globals().get(name)
if globalModule:
module.attributes.extend(dir(globalModule))
else:
self.modules[name] = module
def addModule(self, name, moduleDir=None):
module = pcmodules.getPCModule(name, moduleDir)
if module is None:
self.modules[name] = module = PyCheckerModule(name, 0)
if imp.is_builtin(name) == 0:
module.load()
else:
globalModule = globals().get(name)
if globalModule:
module.attributes.extend(dir(globalModule))
else:
self.modules[name] = module
def get_code(self, parent, modname, fqname):
if parent:
return
else:
if imp.is_builtin(modname):
type = imp.C_BUILTIN
elif imp.is_frozen(modname):
type = imp.PY_FROZEN
else:
return
module = imp.load_module(modname, None, modname, ('', '', type))
return (0, module, {})
def get_code(self, parent, modname, fqname):
if parent:
return
else:
if imp.is_builtin(modname):
type = imp.C_BUILTIN
elif imp.is_frozen(modname):
type = imp.PY_FROZEN
else:
return
module = imp.load_module(modname, None, modname, ('', '', type))
return (0, module, {})
def load_module(self,fullname):
try:
return sys.modules[fullname]
except KeyError:
pass
if imp.is_builtin(fullname):
mod = imp.init_builtin(fullname)
elif imp.is_frozen(fullname):
mod = imp.init_frozen(fullname)
else:
raise ImportError(fullname + " is not builtin or frozen")
sys.modules[fullname] = mod
return mod
def is_stdlib_name(modname):
"""Return :data:`True` if `modname` appears to come from the standard
library."""
if imp.is_builtin(modname) != 0:
return True
module = sys.modules.get(modname)
if module is None:
return False
# six installs crap with no __file__
modpath = os.path.abspath(getattr(module, '__file__', ''))
return is_stdlib_path(modpath)
def show_module_path(module):
import imp, sys, inspect
if (hasattr(module, '__name__') is False):
print('Error: ' + str(module) + ' is not a module object.')
return None
moduleName = module.__name__
modulePath = None
if imp.is_builtin(moduleName):
modulePath = sys.modules[moduleName]
else:
modulePath = inspect.getsourcefile(module)
modulePath = '<module \'' + moduleName + '\' from \'' + modulePath + 'c\'>'
print(modulePath)
return modulePath
def get_code(self, parent, modname, fqname):
if parent:
# these modules definitely do not occur within a package context
return None
# look for the module
if imp.is_builtin(modname):
type = imp.C_BUILTIN
elif imp.is_frozen(modname):
type = imp.PY_FROZEN
else:
# not found
return None
# got it. now load and return it.
module = imp.load_module(modname, None, modname, ('', '', type))
return 0, module, {}
def get_code(self, parent, modname, fqname):
if parent:
# these modules definitely do not occur within a package context
return None
# look for the module
if imp.is_builtin(modname):
type = imp.C_BUILTIN
elif imp.is_frozen(modname):
type = imp.PY_FROZEN
else:
# not found
return None
# got it. now load and return it.
module = imp.load_module(modname, None, modname, ('', '', type))
return 0, module, { }
def watch_import(name, globals=None, *args, **kwargs):
"""
When a module is asked to be imported, check if we have previously imported
it. If so, check if the time stamp of it, a companion yaml file, or any
modules it imports have changed. If so, reimport the module.
:params: see __builtin__.__import__
"""
# Don"t monitor builtin modules. types seem special, so don"t monitor it
# either.
monitor = not imp.is_builtin(name) and name not in ("types", )
# Don"t monitor modules if we don"t know where they came from
monitor = monitor and isinstance(globals, dict) and globals.get("__name__")
if not monitor:
return builtin_import(name, globals, *args, **kwargs)
# This will be the dotted module name except for service modules where it
# will be the absolute file path.
parent = globals["__name__"]
key = parent + "." + name
module_reload_changed(key)
try:
module = builtin_import(name, globals, *args, **kwargs)
except ImportError:
raise
if getattr(module, "__file__", None):
if key not in WatchList:
tangelo.log_info("WATCH", "Monitoring import %s from %s" % (name, parent))
imp.acquire_lock()
try:
if key not in WatchList:
filemtime = module_getmtime(module.__file__) or 0
filemtime = latest_submodule_time(key, filemtime)
WatchList[key] = {
"time": filemtime
}
WatchList[key].update({
"parent": parent,
"name": name,
"file": module.__file__
})
finally:
imp.release_lock()
return module
def _findModuleInPath(module_name):
package_name, module_name = module_name.splitModuleBasename()
if _debug_module_finding:
print("_findModuleInPath: Enter", module_name, "in", package_name)
# The "site" module must be located based on PYTHONPATH before it was
# executed, while we normally search in PYTHONPATH after it was executed,
# and on some systems, that fails.
if package_name is None and module_name == "site":
candidate = os.environ.get("NUITKA_SITE_FILENAME", "")
if candidate:
return candidate
# Free pass for built-in modules, the need not exist.
if package_name is None and imp.is_builtin(module_name):
return None
search_path = getPackageSearchPath(package_name)
if _debug_module_finding:
print("_findModuleInPath: Using search path", search_path, "for",
package_name)
try:
module_filename = _findModuleInPath2(package_name=package_name,
module_name=module_name,
search_path=search_path)
except SyntaxError:
# Warn user, as this is kind of unusual.
recursion_logger.warning(
"%s: Module cannot be imported due to syntax errors.",
module_name if package_name is None else package_name + "." +
module_name,
)
return None
if _debug_module_finding:
print("_findModuleInPath: _findModuleInPath2 gave", module_filename)
return module_filename
def _findModuleInPath(module_name, package_name):
if _debug_module_finding:
print("_findModuleInPath: Enter", module_name, "in", package_name)
# The "site" module must be located based on PYTHONPATH before it was
# executed, while we normally search in PYTHONPATH after it was executed,
# and on some systems, that fails.
if package_name is None and module_name == "site":
candidate = os.environ.get("NUITKA_SITE_FILENAME", "")
if candidate:
return candidate
# Free pass for built-in modules, the need not exist.
if package_name is None and imp.is_builtin(module_name):
return None, module_name
search_path = getPackageSearchPath(package_name)
if _debug_module_finding:
print("_findModuleInPath: Using search path", search_path)
try:
module_filename = _findModuleInPath2(
module_name = module_name,
search_path = search_path
)
except SyntaxError:
# Warn user, as this is kind of unusual.
warning(
"%s: Module cannot be imported due to syntax errors.",
module_name
if package_name is None else
package_name + '.' + module_name,
)
return None
if _debug_module_finding:
print("_findModuleInPath: _findModuleInPath2 gave", module_filename)
return module_filename
def addModule(self, name, alias, moduleDir=None):
"""
@type name: str
@param alias: the name of the token in the module;
for example import gettext as g gives alias g
@type alias: str
"""
module = getPCModule(name, moduleDir)
if module is None:
# not yet loaded, so load
self.modules[alias] = module = PyCheckerModule(name, 0)
if imp.is_builtin(name) == 0:
module.load()
else:
# FIXME: probably should be alias ?
globalModule = globals().get(name)
if globalModule:
module.attributes.extend(dir(globalModule))
else:
self.modules[alias] = module
def addModule(self, name, alias, moduleDir=None) :
"""
@type name: str
@param alias: the name of the token in the module;
for example import gettext as g gives alias g
@type alias: str
"""
module = getPCModule(name, moduleDir)
if module is None :
# not yet loaded, so load
self.modules[alias] = module = PyCheckerModule(name, 0)
if imp.is_builtin(name) == 0:
module.load()
else :
# FIXME: probably should be alias ?
globalModule = globals().get(name)
if globalModule :
module.attributes.extend(dir(globalModule))
else :
self.modules[alias] = module
def is_stdlib_name(self, modname):
"""Return ``True`` if `modname` appears to come from the standard
library."""
if imp.is_builtin(modname) != 0:
return True
module = sys.modules.get(modname)
if module is None:
return False
# six installs crap with no __file__
modpath = getattr(module, '__file__', '')
if 'site-packages' in modpath:
return False
for dirname in self._STDLIB_PATHS:
if os.path.commonprefix((dirname, modpath)) == dirname:
return True
return False
def is_stdlib_name(modname):
"""
Return :data:`True` if `modname` appears to come from the standard library.
"""
# `imp.is_builtin()` isn't a documented as part of Python's stdlib API.
#
# """
# Main is a little special - imp.is_builtin("__main__") will return False,
# but BuiltinImporter is still the most appropriate initial setting for
# its __loader__ attribute.
# """ -- comment in CPython pylifecycle.c:add_main_module()
if imp.is_builtin(modname) != 0:
return True
module = sys.modules.get(modname)
if module is None:
return False
# six installs crap with no __file__
modpath = os.path.abspath(getattr(module, '__file__', ''))
return is_stdlib_path(modpath)
def is_stdlib(mod_name: str) -> bool:
python_path = os.path.dirname(sys.executable)
if imp.is_builtin(mod_name):
return True
try:
module_path = imp.find_module(mod_name)[1]
print(python_path)
print(module_path)
if "site-packages" in module_path:
return True
if python_path in module_path:
return True
return False
except ImportError:
return False
def watch_module(self, name):
"""Load a module, determine which files it uses, and watch them"""
if imp.is_builtin(name) in [-1, 1]:
# Pretty pointless to watch built-in modules
return
f, pathname, description = imp.find_module(name)
try:
mod = imp.load_module(name, f, pathname, description)
if f:
self._watch_file(f.name, name, mod)
else:
for root, dirs, files in os.walk(pathname):
for filename in files:
fpath = os.path.join(root, filename)
if fpath.endswith(".py"):
self._watch_file(fpath, name, mod)
finally:
if f:
f.close()
def _find_loader(self, dirobj, fullname):
_, _, module = fullname.rpartition('.')
if imp.is_builtin(fullname):
return None
for suffix, mode, kind in self.SUFFIXES:
for additional, is_package in [((), False),
(('__init__', ), True)]:
relpath = os.path.join(module, *additional) + suffix
if dirobj.exists(relpath):
break
else:
continue
if kind == imp.C_EXTENSION:
loader = RTLDOpenatLoader
elif kind == imp.PY_SOURCE:
loader = TryPycThenPyOpenatLoader
elif kind == imp.PY_COMPILED:
loader = PyCompiledOpenatLoader
return loader(dirobj, relpath, is_package)
def watch_module(self, name):
"Load a module, determine which files it uses, and watch them"
if imp.is_builtin(name) != 0:
# Pretty pointless to watch built-in modules
return
(fd, pathname, description) = imp.find_module(name)
try:
mod = imp.load_module(name, fd, pathname, description)
if fd:
self._watch_file(fd.name, name)
else:
for root, dirs, files in os.walk(pathname):
for filename in files:
fpath = os.path.join(root, filename)
if fpath.endswith('.py'):
self._watch_file(fpath, name)
finally:
if fd:
fd.close()
def _is_builtin(pkgname):
if not pkgname:
return True
pkgtop = pkgname.split('.')[0]
try:
path = imp.find_module(pkgtop)[1]
except ImportError:
return True
global _builtin_prefixes
if _builtin_prefixes is None:
_builtin_prefixes = []
if sys.platform == 'win32':
join_prefixes = (('DLLs',), ('Lib',), ('lib',), ('lib', 'lib-tk'))
else:
join_prefixes = (('local', 'lib'), ('lib',))
for prefix in join_prefixes:
for site_prefix in set(site.PREFIXES):
_builtin_prefixes.append(os.path.join(site_prefix, *prefix))
if imp.is_builtin(path):
return True
elif os.path.dirname(path) in _builtin_prefixes:
return True
return False
def stdlib_import(name):
"Work around undesired behavior of Python's relative import feature."
try:
return sys.modules[name]
except KeyError:
pass
import imp
if (imp.is_builtin(name)):
try: return imp.init_builtin(name)
except Exception: pass
sys_path = sys.path[:]
sys_path.reverse()
for path in sys_path:
try:
fp, pathname, description = imp.find_module(name, [path])
except ImportError:
pass
else:
try:
return imp.load_module(name, fp, pathname, description)
finally:
if (fp is not None): fp.close()
raise RuntimeError("Cannot import %s module." % name)
def load_module(self, fullname):
try:
return sys.modules[fullname]
except KeyError:
pass
if imp.is_builtin(fullname):
import freeze_external_modules
try:
imp.acquire_lock()
py_package_context = freeze_external_modules.get_py_package_context()
freeze_external_modules.set_py_package_context(fullname)
return imp.init_builtin(fullname)
finally:
freeze_external_modules.set_py_package_context(py_package_context)
imp.release_lock()
else:
if imp.is_frozen('freezer_package.' + fullname):
co = imp.get_frozen_object('freezer_package.' + fullname)
is_package = True
else:
co = imp.get_frozen_object("freezer." + fullname) # This may throw ImportError if not found.
is_package = False
try:
imp.acquire_lock()
mod = imp.new_module(fullname)
if '__builtins__' not in mod.__dict__:
mod.__dict__['__builtins__'] = __builtins__
mod.__file__ = "frozen/" + "/".join(fullname.split('.'))
if is_package:
mod.__path__ = [fullname]
mod.__file__ = mod.__file__ + "/__init__"
sys.modules[fullname] = mod
eval(co, mod.__dict__, mod.__dict__)
return mod
finally:
imp.release_lock()
def find_module(self, fullname, path=None):
if '.' in fullname:
# Caching deals only with top level packages
rdot = fullname.rindex('.')
head = fullname[:rdot] # 'logging'
tail = fullname[rdot + 1:] # 'os'
#self.debug("ignoring call for " + fullname + " (" + str(path) + ")")
self.debug("trying to predict location of " + fullname +
" within " + str(path))
for directory in path:
solution = self.find_in_directory(tail, fullname,
directory)
if solution:
return solution
self.debug("no such luck")
raise ImportError('No module named ' + fullname)
return None
self.debug("trying to predict location of " + fullname)
for directory in sys.path:
solution = self.find_in_directory(fullname, fullname,
directory)
if solution:
return solution
# No match found in paths. Check if this is a builtin module
if imp.is_builtin(fullname):
self.debug("identified " + fullname + " as builtin")
solution = (None, fullname, ('', '', imp.C_BUILTIN))
return pkgutil.ImpLoader(fullname, *solution)
# TODO: Frozen?
# Still no match. This might not exist
self.debug("Import " + fullname + " failed: not found")
raise ImportError('No module named ' + fullname)
def find_module(self, name, paths=None):
'''
@see: http://www.python.org/dev/peps/pep-0302/
'''
if is_builtin(name) == 0 and name not in self.__loading and name not in sys.modules:
self.__loading.add(name)
loader = find_loader(name)
self.__loading.remove(name)
if self._isUnextended(name):
if loader is None:
# If there is no loader but the parent package is extend able we might try to refresh the paths of
# the parent because maybe meanwhile the python path has been updated.
k = name.rfind('.')
if k > 0: inPackage = name[:k]
else: inPackage = None
if not self._isUnextended(inPackage):
_extendPackagePaths(sys.modules[inPackage])
loader = find_loader(name)
elif loader is not None:
if loader.is_package(name): loader = PackageLoader(loader)
elif self.__unittest: loader = ModuleLoader(loader)
return loader
def test_is_builtin(self):
self.assertEqual(imp.is_builtin("xxx"),0)
self.assertEqual(imp.is_builtin("12324"),0)
self.assertEqual(imp.is_builtin("&*^^"),0)
self.assertEqual(imp.is_builtin("dir"),0)
self.assertEqual(imp.is_builtin("__doc__"),0)
self.assertEqual(imp.is_builtin("__name__"),0)
self.assertEqual(imp.is_builtin("_locle"),0)
self.assertEqual(imp.is_builtin("cPickle"),1)
self.assertEqual(imp.is_builtin("_random"),1)
if is_posix:
self.assertEqual(imp.is_builtin("posix"),1)
else:
self.assertEqual(imp.is_builtin("nt"),1)
self.assertEqual(imp.is_builtin("thread"),1)
# there are a several differences between ironpython and cpython
if is_cli:
self.assertEqual(imp.is_builtin("copy_reg"),1)
else:
self.assertEqual(imp.is_builtin("copy_reg"),0)
# supposedly you can't re-init these
self.assertEqual(imp.is_builtin("sys"), -1)
self.assertEqual(imp.is_builtin("__builtin__"), -1)
self.assertEqual(imp.is_builtin("exceptions"), -1)
imp.init_builtin("sys")
imp.init_builtin("__builtin__")
imp.init_builtin("exceptions")
def test_imp_is_builtin(self):
self.assertTrue(all(imp.is_builtin(mod)
for mod in ['sys', '__builtin__']))
self.assertFalse(imp.is_builtin('os'))
def test_is_builtin():
AreEqual(imp.is_builtin("xxx"),0)
AreEqual(imp.is_builtin("12324"),0)
AreEqual(imp.is_builtin("&*^^"),0)
AreEqual(imp.is_builtin("dir"),0)
AreEqual(imp.is_builtin("__doc__"),0)
AreEqual(imp.is_builtin("__name__"),0)
AreEqual(imp.is_builtin("_locle"),0)
AreEqual(imp.is_builtin("cPickle"),1)
AreEqual(imp.is_builtin("_random"),1)
# nt module disabled in Silverlight
if not is_silverlight:
AreEqual(imp.is_builtin("nt"),1)
AreEqual(imp.is_builtin("thread"),1)
# there are a several differences between ironpython and cpython
if is_cli or is_silverlight:
AreEqual(imp.is_builtin("copy_reg"),1)
else:
AreEqual(imp.is_builtin("copy_reg"),0)
# supposedly you can't re-init these
AreEqual(imp.is_builtin("sys"), -1)
AreEqual(imp.is_builtin("__builtin__"), -1)
AreEqual(imp.is_builtin("exceptions"), -1)
imp.init_builtin("sys")
imp.init_builtin("__builtin__")
imp.init_builtin("exceptions")
"""Import hook support.
def is_builtin(self, name): return imp.is_builtin(name) def init_builtin(self, name): return imp.init_builtin(name)
except Exception as e:
print e
imp.acquire_lock()
imp.release_lock()
import os
print "first load_source():"
m1 = imp.load_source("import_target", os.path.join(os.path.dirname(__file__), "import_target.py"))
print "second load_source():"
m2 = imp.load_source("import_target", os.path.join(os.path.dirname(__file__), "import_target.py"))
print m1 is m2
m = imp.new_module("My new module")
print type(m), m, hasattr(m, "__file__")
print imp.is_builtin("sys"), imp.is_frozen("sys")
print imp.is_builtin("io"), imp.is_frozen("io")
e = imp.find_module("1")
m = imp.load_module("test_1", e[0], e[1], e[2])
def n(s):
return str(s).replace(".pyc", ".py")
print n(m), n(m.__name__), n(m.__file__), hasattr(m, "__path__")
import sys, types
name = "json"
m = sys.modules[name] = types.ModuleType(name)
print sorted(dir(m))
s = imp.find_module(name)
def rproc(ProcName, P1, Mem=None, options=None, runtime=None, callfile=None, resubmission=False):
# [jobinfo]=rproc(ProcName, P1, Mem, options, time)
#
# time in minutes
# mem in mb
global SCHED_JOB_ID_SPLIT, SCHED_GET_JOB_NUMBER, SCHED_SUBMIT_CMD
global SCHED_MIN_OPTIONS
_set_scheduler()
environment = '' # TODO
if callfile is None:
### check if ProcName is defined in calling function
callframe = sys._getframe(1)
if not ProcName in callframe.f_locals:
if not ProcName in callframe.f_globals:
print >> sys.stderr, 'ERROR: Could find no definition for %s in local or global context of calling function. Use kword callfile to specify file where %s is defined. Use the relative path to the location of the calling function!' % (ProcName, ProcName)
return
else:
callfile = (callframe.f_globals[ProcName].__module__, inspect.getfile(callframe.f_globals[ProcName]))
else:
callfile = (callframe.f_locals[ProcName].__module__, inspect.getfile(callframe.f_locals[ProcName]))
### detect path of this script
this_frame = sys._getframe(0)
rproc_path = os.path.abspath(inspect.getfile(this_frame))
if runtime is None:
runtime = 24
if Mem is None:
Mem = 300
if Mem < 100:
print >> sys.stderr, 'WARNING: You specified to allocate less than 100Mb memory for your job. This might not be enough to start. Re-setting to 100Mb'
Mem = 100
if options is None:
options = dict()
### get module list of caller to re-create environment
if not 'imports' in options:
options['imports'] = dict()
if not resubmission:
callframe = sys._getframe(1)
#options['package'] = os.path.dirname(os.path.abspath(callframe.f_globals['__file__']))
for l in callframe.f_globals:
if (len(l) < 2 or l[:2] != '__'):
if isinstance(callframe.f_globals[l], types.ModuleType):
if not l in options['imports']:
if imp.is_builtin(callframe.f_globals[l].__name__) != 0:
options['imports'][l] = (callframe.f_globals[l].__name__, 'builtin')
else:
options['imports'][l] = (callframe.f_globals[l].__name__, callframe.f_globals[l].__file__)
if not callfile[0] in options['imports']:
options['imports'][callfile[0]] = callfile
home_str = os.environ['HOME']
use_reservation = False
### TODO this is only relevant for SGE
if 'ncpus' in options and options['ncpus'] > 1:
use_reservation = 1 ;
if not 'verbosity' in options:
options['verbosity'] = True
if not 'maxjobs' in options:
options['maxjobs'] = 5000
if not 'waitonfull' in options:
options['waitonfull'] = True
if not 'immediately' in options:
options['immediately'] = False
if not 'immediately_bg' in options:
options['immediately_bg'] = False
if not 'submit_now' in options:
options['submit_now'] = True
if not 'nicetohave' in options:
options['nicetohave'] = False
if not 'ncpus' in options:
options['ncpus'] = 1
if not 'start_dir' in options:
dirctry = os.getcwd()
else:
dirctry = options['start_dir']
if not 'resubmit' in options:
options['resubmit'] = False
options['time_req_resubmit'] = []
options['mem_req_resubmit'] = []
if not 'data_size' in options:
options['data_size'] = []
if not 'hard_time_limit' in options:
options['hard_time_limit'] = 1000000
jobinfo = rproc_empty()
jobinfo.ProcName = ProcName
jobinfo.P1 = P1
jobinfo.Mem = Mem
jobinfo.options = options
jobinfo.time = runtime
jobinfo.created = True
jobinfo.resubmit = options['resubmit']
jobinfo.mem_req_resubmit = options['mem_req_resubmit']
jobinfo.time_req_resubmit = options['time_req_resubmit']
jobinfo.data_size = options['data_size']
jobinfo.hard_time_limit = options['hard_time_limit']
if not os.path.exists(os.path.join(os.environ['HOME'], 'tmp', '.sge')):
username = os.environ['USER']
base_dir = os.path.join(home_str, '.sge.', 'tmp', username)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
tmp_dir = os.path.join(home_str, '.sge', 'tmp', username, 'tmp')
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
sge_dir = os.path.join(home_str, '.sge', 'tmp', username, 'sge')
if not os.path.exists(sge_dir):
os.makedirs(sge_dir)
if not os.path.exists(os.path.join(os.environ['HOME'], 'tmp')):
os.makedirs(os.path.join(os.environ['HOME'], 'tmp'))
if not os.path.exists(os.path.join(os.environ['HOME'], 'tmp', '.sge')):
### TODO this does not exist anywhere: sge_tmp_dir
os.symlink(sge_tmp_dir, os.path.join(os.environ['HOME'], 'tmp', '.sge'))
assert(os.path.exists(os.path.join(os.environ['HOME'],'tmp', '.sge')))
if not os.path.exists(os.path.join(dirctry, '.sge')):
username = os.environ['USER']
### TODO make this configurable
sge_base_dir = dirctry.replace(os.path.join('cluster', 'home', username), os.path.join(home_str, '.sge', 'tmp', username))
if not os.path.exists(sge_base_dir):
os.makedirs(sge_base_dir)
sge_dir = os.path.join(sge_base_dir, 'sge')
os.makedirs(sge_dir)
os.symlink(sge_dir, os.path.join(dirctry, '.sge'))
### assembly option string
if use_reservation:
option_str = ' -R y'
else:
option_str = ''
#option_str += ' -l h_vmem=%iM -l s_vmem=%iM -soft -l h_cpu=%1.0f -hard ' % (Mem, Mem, max(60, runtime*60))
# TORQUE
#option_str += '-l nodes=1:ppn=%i -l mem=%imb,vmem=%imb,pmem=%imb -l walltime=%1.0f' % (options['ncpus'], Mem, Mem, Mem, max(60, runtime*60))
#option_str += '-n %i -M %i -R "rusage[mem=%i]" -W %i' % (options['ncpus'], Mem, math.ceil(Mem / float(options['ncpus'])), max(60, runtime*60))
option_str += SCHED_MIN_OPTIONS.substitute(cores=str(options['ncpus']), mem=str(Mem), coremem=str(math.ceil(Mem / float(options['ncpus']))), time=str(max(60, runtime)))
if environment == 'galaxy':
option_str += ' -l parent=0.0 '
if 'hold' in options:
if options['hold']:
option_str += ' -h u'
if 'queue' in options:
option_str += ' -q "%s" ' % options['queue']
if 'nicetohave' in options and options['nicetohave']:
option_str += ' -l nicetohave=1'
if 'priority' in options:
option_str += ' -p %i' % options['priority']
if 'express' in options and options['express']:
option_str += ' -l express'
if 'hostname' in options:
option_str += '%s -l hostname=%s' % (option_str, options['hostname'])
### TODO make this configurable
# use same pthon that the one it was called with
bin_str = sys.executable
### request cplex license
if 'cplex' in options and options['cplex']:
option_str += ' -l cplex=1'
### request several cpus
#if 'ncpus' in options and options['ncpus'] > 1:
# option_str += ' -pe "*" %i ' % options['ncpus']
if 'identifier' in options:
identifier = options['identifier']
else:
identifier = 'RP' ;
cc = random.randint(0, 100000)
prefix = '%s%i-%1.10f' % (identifier, cc, time.time())
rproc_dir = '%s/tmp/.sge' % os.environ['HOME']
mat_fname = os.path.join(rproc_dir, '%s.pickle' % prefix)
data_fname = os.path.join(rproc_dir, '%s_data.pickle' % prefix)
result_fname = os.path.join(rproc_dir, '%s_result.pickle' % prefix)
m_fname = os.path.join(rproc_dir, '%s.sh' % prefix)
while os.path.exists(mat_fname) or os.path.exists(result_fname) or os.path.exists(m_fname):
cc = random.randint(0, 100000)
prefix = '%s%i-%1.10f' % (identifier, cc, time.time())
mat_fname = os.path.join(rproc_dir, '%s.pickle' % prefix)
data_fname = os.path.join(rproc_dir, '%s_data.pickle' % prefix)
result_fname = os.path.join(rproc_dir, '%s_result.pickle' % prefix)
m_fname = os.path.join(rproc_dir, '%s.sh' % prefix)
if 'log_fname' in options:
log_fname = options['log_fname']
else:
log_fname = os.path.join(dirctry, '.sge', '%s_%s.rproc' % (prefix, time.strftime('%d-%b-%Y_%H_%M')))
qsublog_fname = '%s.qsubout' % log_fname
jobinfo.prefix = prefix
jobinfo.mat_fname = mat_fname
jobinfo.data_fname = data_fname
jobinfo.result_fname = result_fname
jobinfo.m_fname = m_fname
jobinfo.log_fname = log_fname
jobinfo.qsublog_fname = qsublog_fname
jobinfo.callfile = callfile
### save the call information
cPickle.dump((ProcName, dirctry, options, callfile), open(mat_fname, 'wb'), -1)
cPickle.dump(P1, open(data_fname, 'wb'), -1)
evalstring = '%s %s %s %s' % (bin_str, rproc_path, mat_fname, data_fname)
evalstring = 'cd %s; %s; exit' % (dirctry, evalstring)
fd = open(m_fname, 'w')
print >> fd, '%s' % evalstring
fd.close()
if 'envstr' in options:
envstr = options['envstr']
if len(envstr) > 0:
envstr += ';'
else:
envstr = ''
if options['immediately']:
callstr = '%s bash %s >> %s' % (envstr, m_fname, log_fname)
elif options['immediately_bg']:
callstr = '%s bash %s >> %s &' % (envstr, m_fname, log_fname)
else:
#str = 'echo \'%s hostname; bash %s >> %s\' | qsub -o %s -j y -r y %s -N %s >> %s 2>&1' % (envstr, m_fname, log_fname, qsublog_fname, option_str, prefix, log_fname)
# TORQUE
#str = 'echo \'%s hostname; bash %s >> %s\' | qsub -o %s -j oe -r y %s -N %s >> %s 2>&1' % (envstr, m_fname, log_fname, qsublog_fname, option_str, prefix, log_fname)
# LSF
#str = 'echo \'%s hostname; bash %s >> %s\' | bsub -o %s -e %s %s -J %s >> %s 2>&1' % (envstr, m_fname, log_fname, qsublog_fname, qsublog_fname, option_str, prefix, log_fname)
callstr = SCHED_SUBMIT_CMD.substitute(env=envstr, script=m_fname, log=log_fname, qsub_log=qsublog_fname, options=option_str, name=prefix)
#print >> sys.stderr, callstr
### too verbose
#if options['submit_now'] and options['verbosity']:
# print callstr
# wait until we are allowed to submit again, i.e. #jobs < maxjobs
if not options['immediately'] and not options['immediately_bg'] and options['waitonfull']:
while True:
try:
#num_queued = int(subprocess.check_output('qstat -u' + os.environ['USER'] + '2> /dev/null | grep ' + os.environ['USER'] + '| wc -l | tr -d " "', shell=True).strip())
#num_queued = int(subprocess.check_output('bjobs -u' + os.environ['USER'] + '2> /dev/null | grep ' + os.environ['USER'] + '| wc -l | tr -d " "', shell=True).strip())
num_queued = int(subprocess.check_output(SCHED_GET_JOB_NUMBER.substitute(user=os.environ['USER']), shell=True).strip())
except:
print >> sys.stderr, 'WARNING: could not determine how many jobs are scheduled'
break
# keep 50 spare jobs if multiple rprocs are scheduling...
if (num_queued < options['maxjobs']):
break
else:
if options['verbosity']:
print >> sys.stdout, 'queue full, sleeping 60 seconds (%i/%i)' %(num_queued, options['maxjobs'])
time.sleep(60)
if options['submit_now']:
if options['immediately'] and options['verbosity']:
print >> sys.stdout, 'immediatedly starting job on local machine'
if options['immediately_bg'] and options['verbosity']:
print >> sys.stdout, 'immediatedly starting job on local machine in background'
if options['immediately_bg']:
while True:
str_ = subprocess.check_output('uptime').strip()
float(str_[re.search('average:', str_).start()+8:].split(',')[0])
hit = re.search('average:', str_)
while hit is None:
hit = re.search('average:', str_)
idx = hit.start()
cpu_load = float(str_[idx+8:].split(',')[0])
if cpu_load > 13:
if options['verbosity']:
print 'load too high: %1.2f' % cpu_load
time.sleep(10)
else:
break
time.sleep(2)
p1 = subprocess.Popen(['echo', callstr], stdout=subprocess.PIPE)
p2 = subprocess.Popen(['bash'], stdin=p1.stdout, stdout=subprocess.PIPE)
p2.communicate()
ret = p2.returncode
if ret != 0:
print >> sys.stderr, 'submission failed:\n\tsubmission string: %s\n\treturn code: %i' % (callstr, ret)
jobinfo.submission_time = time.time()
### grab job ID from submission log file
if not options['immediately'] and not options['immediately_bg']:
fd = open(log_fname, 'r')
jobinfo.jobid = -1
if fd:
s = fd.read().strip()
items = eval(SCHED_JOB_ID_SPLIT.substitute(var='s'))
try:
jobinfo.jobid = int(items[0])
except:
print >> sys.stderr, callstr
print >> sys.stderr, 'ERROR: submission failed: %s' % s
sys.exit(1)
fd.close()
rproc_register('submit', jobinfo)
else:
print >> sys.stderr, '%s does not exist' % log_fname
else:
jobinfo.jobid = 0
else:
jobinfo.jobid = 0
return jobinfo
def find_module(self,fullname,path=None):
if imp.is_builtin(fullname):
return self
if imp.is_frozen(fullname):
return self
return None
def getmodule_type(mod):
"""Type for loaded module as defined by module 'imp.*'"""
if not V3K:
try:
ret = find_module(getmodule_oid(mod))
if ret and ret[2]:
if type(ret[2]) is int: # MT_SOURCE,MT_COMPILED,MT_EXTENSION
return ret[2]
elif type(ret[2]) is tuple: # MT_DIRECTORY,MT_BUILTIN
return ret[2][2]
# TODO: PY_FROZEN
except:
if ismodule(mod) or type(mod) is ModuleType:
try:
_n = getmodule_oid(mod)
if _n in sys.builtin_module_names or is_builtin(_n):
return MT_BUILTIN
else:
bn = sys.modules[_n].__file__
if bn:
bn = os.path.basename(bn)
if bn.startswith('__init__'):
return MT_DIRECTORY
elif bn[-2:] == 'py':
return MT_SOURCE
elif bn[-3:] in ('pyc',):
return MT_COMPILED_DEBUG
elif bn[-3:] in ('pyo'):
return MT_COMPILED_OPT # either O1 or O2
elif bn[-3:] in ('so',):
return MT_EXTENSION
elif is_frozen(_n):
return MT_FROZEN
except ImportError:
pass
except:
raise
return MT_UNKNOWN
else:
# PEP 3147 -- PYC Repository Directories
ret = find_loader(getmodule_oid(mod))
if ret and hasattr(ret, 'path'):
# check python native
cdir = os.path.dirname(ret.path) + os.path.sep + '__pycache__'
if os.path.exists(cdir):
v = '{0}{1}'.format(*sys.version_info[:2])
mname = re.sub(r"^.*[.]", '', ret.name) + '.' + sys.implementation.cache_tag # @UndefinedVariable
for xf in os.walk(cdir):
for xi in xf[2]:
if xi.startswith(mname):
#
# for now supports: Python3, Pypy3
#
if xi[-3:]== "pyc":
return MT_COMPILED_DEBUG
elif xi[-3:]== "pyo":
if xi[-9:-4]== "opt-1":
return MT_COMPILED_OPT1
if xi[-9:-4]== "opt-2":
return MT_COMPILED_OPT2
else:
raise PySourceInfoError("Extension unknown:" + str(xi))
else:
return MT_SOURCE
elif type(ret) in (BuiltinFunctionType, BuiltinMethodType,):
return MT_BUILTIN
else:
_n = None
if ret:
if not hasattr(ret, 'name'):
if type(ret) == type:
# return MT_FROZEN
return MT_BUILTIN
else:
return MT_UNKNOWN
else:
_n = ret.name
else:
try:
_n = getmodule_oid(mod)
except ImportError:
pass
except:
raise
if _n in sys.builtin_module_names or is_builtin(_n):
return MT_BUILTIN
else:
bn = sys.modules[_n].__file__
if bn:
bn = os.path.basename(bn)
if bn.startswith('__init__'):
return MT_DIRECTORY
elif bn[-2:] == 'py':
return MT_SOURCE
elif bn[-3:] in ('pyc', 'pyo'):
return MT_COMPILED
elif bn[-3:] in ('so',):
return MT_EXTENSION
elif is_frozen(_n):
return MT_FROZEN
return MT_UNKNOWN
def start_proc(fname, data_fname, rm_flag=True):
# start_proc(fname, data_fname, rm_flag)
global THIS_IS_A_RPROC_PROCESS
THIS_IS_A_RPROC_PROCESS = True
### load and create environment
(ProcName, dirctry, options, callfile) = cPickle.load(open(fname, 'r'))
os.chdir(dirctry)
print '%s on %s started (in %s; from %s %s)' % (ProcName, os.environ['HOSTNAME'], dirctry, fname, data_fname)
print '### job started %s' % time.strftime('%Y-%m-%d %H:%S')
if 'rmpaths' in options:
for i in range(len(options['rmpaths'])):
print 'removing path %s' % options['rmpaths'][i]
while options['rmpaths'][i] in sys.path:
r_idx = sys.path.index(options['rmpaths'][i])
del sys.path[r_idx]
if 'addpaths' in options:
for i in range(len(options['addpaths'])):
if not options['addpaths'][i] in sys.path:
print 'adding path %s' % options['addpaths'][i]
sys.path.append(options['addpaths'][i])
if 'rm_flag' in options:
rm_flag = options['rm_flag']
### create environment
import_list = []
for mod in options['imports']:
module = options['imports'][mod]
if module[1] == 'builtin':
if imp.is_builtin(module[0]) == 1:
exec('import %s' % module[0])
else:
mod_sl = module[0].split('.')
subpaths = get_subpaths(os.path.dirname(module[1]).split('/'))
imported = True
for m in range(len(mod_sl)):
exec('exists = \'%s\' in globals().keys()' % '.'.join(mod_sl[:m+1]))
if not exists and not '.'.join(mod_sl[:m+1]) in import_list and not 'rproc' in mod_sl[:m+1]:
try:
(f, fn, des) = imp.find_module(mod_sl[m], subpaths)
try:
### TODO: This is a bit hacky, but the only way that linalg can be loaded right now
if fn.endswith('scipy'):
import scipy
import_list.append('scipy')
continue
exec('%s = imp.load_module(\'%s\', f, fn, des)' % ('.'.join(mod_sl[:m+1]), '.'.join(mod_sl[:m+1])))
import_list.append('.'.join(mod_sl[:m+1]))
except:
imported = False
finally:
if f is not None:
f.close()
except ImportError:
print >> sys.stderr, 'Module %s could not be found' % '.'.join(mod_sl[:m+1])
imported = False
else:
imported = False
if mod != module[0] and imported:
exec('%s = %s' % (mod, module[0]))
### load data into environment
P1 = cPickle.load(open(data_fname, 'r'))
retval1 = []
retval2 = []
try:
if callfile[0] == '__main__':
exec('from %s import %s' % (re.sub(r'.py$', '', callfile[1]), ProcName))
else:
exec('from %s import %s' % (callfile[0], ProcName))
if len(P1) > 0:
retval = eval('%s(P1)' % ProcName)
else:
retval = eval('%s()' % ProcName)
if retval is None:
pass
elif isinstance(retval, tuple):
retval1 = retval[0]
retval2 = retval[1]
else:
retval1 = retval
if not ('no_result_file' in options and options['no_result_file']):
print 'saving results to %s_result.pickle' % os.path.splitext(fname)[0]
cPickle.dump((retval1, retval2), open('%s_result.pickle' % os.path.splitext(fname)[0], 'wb'), -1)
except (NameError, TypeError) as e:
print >> sys.stderr, 'execution of %s failed' % ProcName
print >> sys.stderr, '%s' % str(e)
global MATLAB_RETURN_VALUE
MATLAB_RETURN_VALUE = -1
rm_flag = False
except RprocRerun as e:
# if we rerun, then we should not cleanup
print >> sys.stderr, 'job is marked for rerunning. exiting without finished computations'
else:
if rm_flag:
os.remove(fname) # data file
os.remove('%ssh' % fname.strip('pickle')) # script file
print '### job finished %s' % time.strftime('%Y-%m-%d %H:%S')
def test_is_builtin():
AreEqual(imp.is_builtin("xxx"),0)
AreEqual(imp.is_builtin("12324"),0)
AreEqual(imp.is_builtin("&*^^"),0)
AreEqual(imp.is_builtin("dir"),0)
AreEqual(imp.is_builtin("__doc__"),0)
AreEqual(imp.is_builtin("__name__"),0)
AreEqual(imp.is_builtin("_locle"),0)
AreEqual(imp.is_builtin("cPickle"),1)
AreEqual(imp.is_builtin("_random"),1)
# nt module disabled in Silverlight
if not is_silverlight:
AreEqual(imp.is_builtin("nt"),1)
AreEqual(imp.is_builtin("thread"),1)
# there are a several differences between ironpython and cpython
if is_cli or is_silverlight:
AreEqual(imp.is_builtin("copy_reg"),1)
else:
AreEqual(imp.is_builtin("copy_reg"),0)
# supposedly you can't re-init these
AreEqual(imp.is_builtin("sys"), -1)
AreEqual(imp.is_builtin("__builtin__"), -1)
AreEqual(imp.is_builtin("exceptions"), -1)
imp.init_builtin("sys")
imp.init_builtin("__builtin__")
imp.init_builtin("exceptions")
def is_package(self,fullname):
if imp.is_builtin(fullname+".__init__"):
return True
if imp.is_frozen(fullname+".__init__"):
return True
return False
