def testZipImporterMethods(self):
packdir = TESTPACK + os.sep
packdir2 = packdir + TESTPACK2 + os.sep
files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc),
packdir2 + "__init__" + pyc_ext: (NOW, test_pyc),
packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc),
"spam" + pyc_ext: (NOW, test_pyc)}
z = ZipFile(TEMP_ZIP, "w")
try:
for name, (mtime, data) in files.items():
zinfo = ZipInfo(name, time.localtime(mtime))
zinfo.compress_type = self.compression
zinfo.comment = b"spam"
z.writestr(zinfo, data)
z.close()
zi = zipimport.zipimporter(TEMP_ZIP)
self.assertEqual(zi.archive, TEMP_ZIP)
self.assertEqual(zi.is_package(TESTPACK), True)
find_mod = zi.find_module('spam')
self.assertIsNotNone(find_mod)
self.assertIsInstance(find_mod, zipimport.zipimporter)
self.assertFalse(find_mod.is_package('spam'))
load_mod = find_mod.load_module('spam')
self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__)
mod = zi.load_module(TESTPACK)
self.assertEqual(zi.get_filename(TESTPACK), mod.__file__)
existing_pack_path = importlib.import_module(TESTPACK).__path__[0]
expected_path_path = os.path.join(TEMP_ZIP, TESTPACK)
self.assertEqual(existing_pack_path, expected_path_path)
self.assertEqual(zi.is_package(packdir + '__init__'), False)
self.assertEqual(zi.is_package(packdir + TESTPACK2), True)
self.assertEqual(zi.is_package(packdir2 + TESTMOD), False)
mod_path = packdir2 + TESTMOD
mod_name = module_path_to_dotted_name(mod_path)
mod = importlib.import_module(mod_name)
self.assertTrue(mod_name in sys.modules)
self.assertEqual(zi.get_source(TESTPACK), None)
self.assertEqual(zi.get_source(mod_path), None)
self.assertEqual(zi.get_filename(mod_path), mod.__file__)
# To pass in the module name instead of the path, we must use the
# right importer
loader = mod.__loader__
self.assertEqual(loader.get_source(mod_name), None)
self.assertEqual(loader.get_filename(mod_name), mod.__file__)
# test prefix and archivepath members
zi2 = zipimport.zipimporter(TEMP_ZIP + os.sep + TESTPACK)
self.assertEqual(zi2.archive, TEMP_ZIP)
self.assertEqual(zi2.prefix, TESTPACK + os.sep)
finally:
z.close()
os.remove(TEMP_ZIP)
def execfilename(filename):
if filename.endswith('.py'):
execfile(filename)
elif filename.endswith('.zip'):
sys.path.insert(0, filename)
exec zipimport.zipimporter(filename).get_code('__main__')
else:
raise ValueError('filename=%r is not a valid python file' % filename)
def test_cache(self):
self.writefile('x.py', 'y')
from zipimport import _zip_directory_cache, zipimporter
new_importer = zipimporter(self.zipfile)
try:
assert zipimporter(self.zipfile) is not new_importer
finally:
del _zip_directory_cache[self.zipfile]
def testUnencodable(self):
filename = support.TESTFN_UNENCODABLE + ".zip"
self.addCleanup(support.unlink, filename)
with ZipFile(filename, "w") as z:
zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW))
zinfo.compress_type = self.compression
z.writestr(zinfo, test_src)
zipimport.zipimporter(filename).load_module(TESTMOD)
def _tool_module(self):
# TODO: Interchange zipimport with normal initilization for better error reporting
oldpythonpath = sys.path
sys.path = self.toolpath + sys.path
try:
try:
file, path, desc = imp.find_module(self.name, self.toolpath)
try:
return imp.load_module(self.name, file, path, desc)
finally:
if file:
file.close()
except ImportError as e:
if str(e)!="No module named %s"%self.name:
raise SCons.Errors.EnvironmentError(e)
try:
import zipimport
except ImportError:
pass
else:
for aPath in self.toolpath:
try:
importer = zipimport.zipimporter(aPath)
return importer.load_module(self.name)
except ImportError as e:
pass
finally:
sys.path = oldpythonpath
full_name = 'SCons.Tool.' + self.name
try:
return sys.modules[full_name]
except KeyError:
try:
smpath = sys.modules['SCons.Tool'].__path__
try:
file, path, desc = imp.find_module(self.name, smpath)
module = imp.load_module(full_name, file, path, desc)
setattr(SCons.Tool, self.name, module)
if file:
file.close()
return module
except ImportError as e:
if str(e)!="No module named %s"%self.name:
raise SCons.Errors.EnvironmentError(e)
try:
import zipimport
importer = zipimport.zipimporter( sys.modules['SCons.Tool'].__path__[0] )
module = importer.load_module(full_name)
setattr(SCons.Tool, self.name, module)
return module
except ImportError as e:
m = "No tool named '%s': %s" % (self.name, e)
raise SCons.Errors.EnvironmentError(m)
except ImportError as e:
m = "No tool named '%s': %s" % (self.name, e)
raise SCons.Errors.EnvironmentError(m)
def testUnencodable(self):
filename = support.TESTFN_UNENCODABLE + ".zip"
z = ZipFile(filename, "w")
zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW))
zinfo.compress_type = self.compression
z.writestr(zinfo, test_src)
z.close()
try:
zipimport.zipimporter(filename)
finally:
os.remove(filename)
def test_cache_subdir(self):
import os
self.writefile('x.py', '')
self.writefile('sub/__init__.py', '')
self.writefile('sub/yy.py', '')
from zipimport import _zip_directory_cache, zipimporter
sub_importer = zipimporter(self.zipfile + os.path.sep + 'sub')
main_importer = zipimporter(self.zipfile)
assert main_importer is not sub_importer
assert main_importer.prefix == ""
assert sub_importer.prefix == "sub" + os.path.sep
def testZipImporterMethodsInSubDirectory(self):
packdir = TESTPACK + os.sep
packdir2 = packdir + TESTPACK2 + os.sep
files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc),
packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)}
z = ZipFile(TEMP_ZIP, "w")
try:
for name, (mtime, data) in files.items():
zinfo = ZipInfo(name, time.localtime(mtime))
zinfo.compress_type = self.compression
zinfo.comment = b"eggs"
z.writestr(zinfo, data)
z.close()
zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir)
self.assertEqual(zi.archive, TEMP_ZIP)
self.assertEqual(zi.prefix, packdir)
self.assertEqual(zi.is_package(TESTPACK2), True)
mod = zi.load_module(TESTPACK2)
self.assertEqual(zi.get_filename(TESTPACK2), mod.__file__)
self.assertEqual(
zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
self.assertEqual(
zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2
zi2 = zipimport.zipimporter(pkg_path)
find_mod_dotted = zi2.find_module(TESTMOD)
self.assertIsNotNone(find_mod_dotted)
self.assertIsInstance(find_mod_dotted, zipimport.zipimporter)
self.assertFalse(zi2.is_package(TESTMOD))
load_mod = find_mod_dotted.load_module(TESTMOD)
self.assertEqual(
find_mod_dotted.get_filename(TESTMOD), load_mod.__file__)
mod_path = TESTPACK2 + os.sep + TESTMOD
mod_name = module_path_to_dotted_name(mod_path)
mod = importlib.import_module(mod_name)
self.assertTrue(mod_name in sys.modules)
self.assertEqual(zi.get_source(TESTPACK2), None)
self.assertEqual(zi.get_source(mod_path), None)
self.assertEqual(zi.get_filename(mod_path), mod.__file__)
# To pass in the module name instead of the path, we must use the
# right importer.
loader = mod.__loader__
self.assertEqual(loader.get_source(mod_name), None)
self.assertEqual(loader.get_filename(mod_name), mod.__file__)
finally:
z.close()
os.remove(TEMP_ZIP)
def testFileUnreadable(self):
support.unlink(TESTMOD)
fd = os.open(TESTMOD, os.O_CREAT, 000)
try:
os.close(fd)
with self.assertRaises(zipimport.ZipImportError) as cm:
zipimport.zipimporter(TESTMOD)
finally:
# If we leave "the read-only bit" set on Windows, nothing can
# delete TESTMOD, and later tests suffer bogus failures.
os.chmod(TESTMOD, 0o666)
support.unlink(TESTMOD)
def test_good_bad_arguments(self):
from zipimport import zipimporter
import os
self.writefile("x.py", "y")
zipimporter(self.zipfile) # should work
raises(ImportError, "zipimporter(os.path.dirname(self.zipfile))")
raises(ImportError, 'zipimporter("fsafdetrssffdsagadfsafdssadasa")')
name = os.path.join(os.path.dirname(self.zipfile), "x.zip")
f = open(name, "w")
f.write("zzz")
f.close()
raises(ImportError, 'zipimporter(name)')
# this should work as well :-/
zipimporter(os.path.join(self.zipfile, 'x'))
def get_zip_data(self, filename):
"""Get data from `filename` if it is a zip file path.
Returns the string data read from the zip file, or None if no zip file
could be found or `filename` isn't in it. The data returned will be
an empty string if the file is empty.
"""
import zipimport
markers = ['.zip'+os.sep, '.egg'+os.sep]
for marker in markers:
if marker in filename:
parts = filename.split(marker)
try:
zi = zipimport.zipimporter(parts[0]+marker[:-1])
except zipimport.ZipImportError:
continue
try:
data = zi.get_data(parts[1])
except IOError:
continue
if sys.version_info >= (3, 0):
data = data.decode('utf8') # TODO: How to do this properly?
return data
return None
def module_list(path):
"""
Return the list containing the names of the modules available in the given
folder.
"""
# sys.path has the cwd as an empty string, but isdir/listdir need it as '.'
if path == '':
path = '.'
if os.path.isdir(path):
folder_list = os.listdir(path)
elif path.endswith('.egg'):
try:
folder_list = [f for f in zipimporter(path)._files]
except:
folder_list = []
else:
folder_list = []
if not folder_list:
return []
# A few local constants to be used in loops below
isfile = os.path.isfile
pjoin = os.path.join
basename = os.path.basename
# Now find actual path matches for packages or modules
folder_list = [p for p in folder_list
if isfile(pjoin(path, p,'__init__.py'))
or import_re.match(p) ]
return [basename(p).split('.')[0] for p in folder_list]
def dist_from_egg(egg_path):
if os.path.isdir(egg_path):
metadata = PathMetadata(egg_path, os.path.join(egg_path, 'EGG-INFO'))
else:
# Assume it's a file or an internal egg
metadata = EggMetadata(zipimporter(egg_path))
return Distribution.from_filename(egg_path, metadata=metadata)
def register_module_classes(m):
""" Register module named m, if not already registered.
This will load the module and register all of its models. (Actually, the
explicit constructor call of each of the models inside the module will
register them.)
"""
def log(e):
mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or ''
msg = "Couldn't load %smodule %s" % (mt, m)
logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg)
logger.notifyChannel('init', netsvc.LOG_CRITICAL, e)
global loaded
if m in loaded:
return
logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m)
mod_path = get_module_path(m)
initialize_sys_path()
try:
zip_mod_path = mod_path + '.zip'
if not os.path.isfile(zip_mod_path):
load_module(m)
else:
zimp = zipimport.zipimporter(zip_mod_path)
zimp.load_module(m)
except Exception, e:
log(e)
raise
def PythonEgg(glob, name=None):
"""Refers to pre-built Python eggs in the file system. (To instead fetch
eggs in a ``pip``/``easy_install`` way, use ``python_requirement``)
E.g., ``egg(name='foo', glob='foo-0.1-py2.6.egg')`` would pick up the
file ``foo-0.1-py2.6.egg`` from the ``BUILD`` file's directory; targets
could depend on it by name ``foo``.
:param string glob: File glob pattern.
:param string name: Target name; by default uses the egg's project name.
"""
# TODO(John Sirois): Rationalize with globs handling in ParseContext
eggs = fsglob(ParseContext.path(glob))
requirements = set()
for egg in eggs:
if os.path.isdir(egg):
metadata = PathMetadata(egg, os.path.join(egg, 'EGG-INFO'))
else:
metadata = EggMetadata(zipimporter(egg))
dist = Distribution.from_filename(egg, metadata=metadata)
requirements.add(dist.as_requirement())
if len(requirements) > 1:
raise ValueError('Got multiple egg versions! => %s' % requirements)
return PythonRequirement(str(requirements.pop()), name=name)
def register_class(m):
"""
Register module named m, if not already registered
"""
def log(e):
mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or ''
msg = "Couldn't load %smodule %s" % (mt, m)
logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg)
logger.notifyChannel('init', netsvc.LOG_CRITICAL, e)
global loaded
if m in loaded:
return
logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m)
mod_path = get_module_path(m)
try:
zip_mod_path = mod_path + '.zip'
if not os.path.isfile(zip_mod_path):
fm = imp.find_module(m, ad_paths)
try:
imp.load_module(m, *fm)
finally:
if fm[0]:
fm[0].close()
else:
zimp = zipimport.zipimporter(zip_mod_path)
zimp.load_module(m)
except Exception, e:
log(e)
raise
def download(self, cr, uid, ids, download=True, context=None):
res = []
for mod in self.browse(cr, uid, ids, context=context):
if not mod.url:
continue
match = re.search('-([a-zA-Z0-9\._-]+)(\.zip)', mod.url, re.I)
version = '0'
if match:
version = match.group(1)
if parse_version(mod.installed_version or '0') >= parse_version(version):
continue
res.append(mod.url)
if not download:
continue
zip_content = urllib.urlopen(mod.url).read()
fname = addons.get_module_path(str(mod.name)+'.zip', downloaded=True)
try:
with open(fname, 'wb') as fp:
fp.write(zip_content)
except Exception:
_logger.exception('Error when trying to create module '
'file %s', fname)
raise orm.except_orm(_('Error'), _('Can not create the module file:\n %s') % (fname,))
terp = self.get_module_info(mod.name)
self.write(cr, uid, mod.id, self.get_values_from_terp(terp))
cr.execute('DELETE FROM ir_module_module_dependency ' \
'WHERE module_id = %s', (mod.id,))
self._update_dependencies(cr, uid, mod, terp.get('depends',
[]))
self._update_category(cr, uid, mod, terp.get('category',
'Uncategorized'))
# Import module
zimp = zipimport.zipimporter(fname)
zimp.load_module(mod.name)
return res
def load_openerp_module(module_name):
""" Load an PengERP module, if not already loaded.
This loads the module and register all of its models, thanks to either
the MetaModel metaclass, or the explicit instantiation of the model.
This is also used to load server-wide module (i.e. it is also used
when there is no model to register).
"""
global loaded
if module_name in loaded:
return
initialize_sys_path()
try:
mod_path = get_module_path(module_name)
zip_mod_path = '' if not mod_path else mod_path + '.zip'
if not os.path.isfile(zip_mod_path):
__import__('openerp.addons.' + module_name)
else:
zimp = zipimport.zipimporter(zip_mod_path)
zimp.load_module(module_name)
# Call the module's post-load hook. This can done before any model or
# data has been initialized. This is ok as the post-load hook is for
# server-wide (instead of registry-specific) functionalities.
info = load_information_from_description_file(module_name)
if info['post_load']:
getattr(sys.modules['openerp.addons.' + module_name], info['post_load'])()
except Exception, e:
mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or ''
msg = "Couldn't load %smodule %s" % (mt, module_name)
_logger.critical(msg)
_logger.critical(e)
raise
def Load_Module(fileName): """ Load module without load_module from importer. \ In this way, we can change the name of module in the built-in. """ ### import zipfile model if zipfile.is_zipfile(fileName): importer = zipimport.zipimporter(fileName) ### change module name old_plugin_name = 'plugins' new_plugin_name = '%s.%s'%(os.path.basename(os.path.splitext(fileName)[0]), old_plugin_name) ### get code of plug-ins code = importer.get_code(old_plugin_name) # Create the new 'temp' module. temp = imp.new_module(new_plugin_name) sys.modules[new_plugin_name] = temp ### there is syntax error ? try: exec code in temp.__dict__ except Exception, info: return info return sys.modules[new_plugin_name]
def platform_module(name = platform_default()):
"""Return the imported module for the platform.
This looks for a module name that matches the specified argument.
If the name is unspecified, we fetch the appropriate default for
our execution environment.
"""
full_name = 'SCons.Platform.' + name
if full_name not in sys.modules:
if os.name == 'java':
eval(full_name)
else:
try:
file, path, desc = imp.find_module(name,
sys.modules['SCons.Platform'].__path__)
try:
mod = imp.load_module(full_name, file, path, desc)
finally:
if file:
file.close()
except ImportError:
try:
import zipimport
importer = zipimport.zipimporter( sys.modules['SCons.Platform'].__path__[0] )
mod = importer.load_module(full_name)
except ImportError:
raise SCons.Errors.UserError("No platform named '%s'" % name)
setattr(SCons.Platform, name, mod)
return sys.modules[full_name]
def ziploader(*paths):
"""Obtain a zipimporter for a directory under the main zip."""
path = os.path.join(loader.archive, *paths)
zl = sys.path_importer_cache.get(path)
if not zl:
zl = zipimport.zipimporter(path)
return zl
def test_functions(self):
import os
import zipimport
data = "saddsadsa"
pyc_data = self.test_pyc
self.now_in_the_future(+5) # write the zipfile 5 secs after the .pyc
self.writefile("xxx", data)
self.writefile("xx/__init__.py", "5")
self.writefile("yy.py", "3")
self.writefile('uu.pyc', pyc_data)
z = zipimport.zipimporter(self.zipfile)
assert z.get_data(self.zipfile + os.sep + "xxx") == data
assert z.is_package("xx")
assert not z.is_package("yy")
assert z.get_source("yy") == '3'
assert z.get_source('uu') is None
raises(ImportError, "z.get_source('zz')")
#assert z.get_code('yy') == py.code.Source('3').compile()
#assert z.get_code('uu') == self.co
assert z.get_code('uu')
assert z.get_code('xx')
assert z.get_source('xx') == "5"
assert z.archive == self.zipfile
mod = z.load_module('xx')
assert z.get_filename('xx') == mod.__file__
def module_list(path):
"""
Return the list containing the names of the modules available in
the given folder.
:param path: folder path
:type path: str
:returns: modules
:rtype: list
"""
if os.path.isdir(path):
folder_list = os.listdir(path)
elif path.endswith('.egg'):
from zipimport import zipimporter
try:
folder_list = [f for f in zipimporter(path)._files]
except:
folder_list = []
else:
folder_list = []
#folder_list = glob.glob(os.path.join(path,'*'))
folder_list = [
p for p in folder_list
if (os.path.exists(os.path.join(path, p, '__init__.py')) or
p[-3:] in {'.py', '.so'} or
p[-4:] in {'.pyc', '.pyo', '.pyd'})]
folder_list = [os.path.basename(p).split('.')[0] for p in folder_list]
return folder_list
def load_from_file(self, path, dbname, key):
class_inst = None
expected_class = 'Parser'
try:
ad = os.path.abspath(os.path.join(tools.ustr(config['root_path']), u'addons'))
mod_path_list = map(lambda m: os.path.abspath(tools.ustr(m.strip())), config['addons_path'].split(','))
mod_path_list.append(ad)
mod_path_list = list(set(mod_path_list))
for mod_path in mod_path_list:
if os.path.lexists(mod_path+os.path.sep+path.split(os.path.sep)[0]):
filepath=mod_path+os.path.sep+path
filepath = os.path.normpath(filepath)
sys.path.append(os.path.dirname(filepath))
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
mod_name = '%s_%s_%s' % (dbname,mod_name,key)
if file_ext.lower() == '.py':
py_mod = imp.load_source(mod_name, filepath)
elif file_ext.lower() == '.pyc':
py_mod = imp.load_compiled(mod_name, filepath)
if expected_class in dir(py_mod):
class_inst = py_mod.Parser
return class_inst
elif os.path.lexists(mod_path+os.path.sep+path.split(os.path.sep)[0]+'.zip'):
zimp = zipimport.zipimporter(mod_path+os.path.sep+path.split(os.path.sep)[0]+'.zip')
return zimp.load_module(path.split(os.path.sep)[0]).parser.Parser
except SyntaxError, e:
raise osv.except_osv(_('Syntax Error !'), e)
def _testBogusZipFile(self):
support.unlink(TESTMOD)
fp = open(TESTMOD, 'w+')
fp.write(struct.pack('=I', 0x06054B50))
fp.write('a' * 18)
fp.close()
z = zipimport.zipimporter(TESTMOD)
try:
self.assertRaises(TypeError, z.find_module, None)
self.assertRaises(TypeError, z.load_module, None)
self.assertRaises(TypeError, z.is_package, None)
self.assertRaises(TypeError, z.get_code, None)
self.assertRaises(TypeError, z.get_data, None)
self.assertRaises(TypeError, z.get_source, None)
error = zipimport.ZipImportError
self.assertEqual(z.find_module('abc'), None)
self.assertRaises(error, z.load_module, 'abc')
self.assertRaises(error, z.get_code, 'abc')
self.assertRaises(IOError, z.get_data, 'abc')
self.assertRaises(error, z.get_source, 'abc')
self.assertRaises(error, z.is_package, 'abc')
finally:
zipimport._zip_directory_cache.clear()
def testZipImporterMethodsInSubDirectory(self):
packdir = TESTPACK + os.sep
packdir2 = packdir + TESTPACK2 + os.sep
files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc),
packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)}
z = ZipFile(TEMP_ZIP, "w")
try:
for name, (mtime, data) in files.items():
zinfo = ZipInfo(name, time.localtime(mtime))
zinfo.compress_type = self.compression
z.writestr(zinfo, data)
z.close()
zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir)
self.assertEquals(zi.archive, TEMP_ZIP)
self.assertEquals(zi.prefix, packdir)
self.assertEquals(zi.is_package(TESTPACK2), True)
zi.load_module(TESTPACK2)
self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
mod_name = TESTPACK2 + os.sep + TESTMOD
mod = __import__(module_path_to_dotted_name(mod_name))
self.assertEquals(zi.get_source(TESTPACK2), None)
self.assertEquals(zi.get_source(mod_name), None)
finally:
z.close()
os.remove(TEMP_ZIP)
def instance(self, parameters, parent=None, address=None):
# if no absolute path is given im mod-meta -> take it relative to the
# meta-file
mod_archive = self.getArchive()
if not os.path.isabs(mod_archive):
mod_archive = os.path.join( os.path.dirname(self._d_file_path), mod_archive)
# try to find module:
zipimp = zipimporter(mod_archive);
mod = zipimp.load_module(self.getClass());
#load class from module
try:
cls = mod.__dict__[self.getClass()]
except:
raise ItemNotFound("Can't find class %s in %s [%s]"%
(self.getClass(), mod_archive, mod.__dict__.keys()))
#instance class:
if not issubclass(cls, (CInnerModule,CDisposableModule)):
self._d_logger.debug("Module %s instance with params %s"%(self.getClass(),parameters))
return cls(parameters)
else:
self._d_logger.debug("InnerModule %s instance with addr %s and params %s"%(self.getClass(), address, parameters))
return cls(parent, address, parameters)
def LoadPlugins(self, obj, fileName):
""" Method which load plugins from zip
Used for define or redefine method of amd. and .cmd model
The name of plugin file must be "plugins.py"
"""
### if list of activated plugins is not empty
if obj.plugins != []:
### import zipfile model
if zipfile.is_zipfile(fileName):
importer = zipimport.zipimporter(fileName)
if importer.find_module('plugins'):
try:
module = importer.load_module('plugins')
except ImportError, info:
sys.stdout.write("%s\n"%info)
return info
for m in [e for e in map(module.__dict__.get, dir(module)) if not inspect.ismodule(e) and inspect.getmodule(e) is module]:
name = m.__name__
### import only plugins in plugins list (dynamic attribute)
if name in obj.plugins:
try:
### new object to assaign
new = eval("module.%s"%name)
if inspect.isfunction(new):
setattr(obj, name, types.MethodType(new, obj))
elif inspect.isclass(new):
### TODO: monkey patchin !!! (most simple is to change python file for override class)
pass
except Exception, info:
sys.stdout.write(_('plugins %s not loaded : %s\n'%(name,info)))
return info
def find_module(self, name, path):
# Ignore anything not in the form kaa.foo, or if kaa.foo is an egg in sys.path.
self.discover_kaa_eggs()
if not name.startswith('kaa.') or name.count('.') > 1 or name in self.kaa_eggs:
return
kaa_base_path = __file__.rsplit('/', 1)[0]
name = name[4:].replace('.', '/')
imp.acquire_lock()
try:
# Scenario 1: kaa.base is an on-disk tree (egg or otherwise)
if not self.zip:
try:
return KaaLoader(imp.find_module(name, [kaa_base_path]))
except ImportError:
pass
# Scenario 2: kaa.base is a zipped egg
if '.egg/' in kaa_base_path and self.zip is not False:
try:
if not self.zip:
self.zip = zipimport.zipimporter(kaa_base_path)
except ImportError:
# Not a valid zipped egg, cache the result so we don't try again.
self.zip = False
else:
if self.zip.find_module('kaa.base.' + name):
# kaa.base.foo found inside egg.
return KaaLoader(self.zip)
finally:
imp.release_lock()
def _tool_module(self):
# TODO: Interchange zipimport with normal initilization for better error reporting
oldpythonpath = sys.path
sys.path = self.toolpath + sys.path
try:
try:
file, path, desc = imp.find_module(self.name, self.toolpath)
try:
return imp.load_module(self.name, file, path, desc)
finally:
if file:
file.close()
except ImportError, e:
if str(e)!="No module named %s"%self.name:
raise SCons.Errors.EnvironmentError, e
try:
import zipimport
except ImportError:
pass
else:
for aPath in self.toolpath:
try:
importer = zipimport.zipimporter(aPath)
return importer.load_module(self.name)
except ImportError, e:
pass
class MyModuleGraph(modulegraph.ModuleGraph):
def _find_single_path(self, name, p, parent=None):
"""find module or zip module in directory or zipfile p"""
if parent is not None:
# assert path is not None
fullname = parent.identifier + '.' + name
else:
fullname = name
try:
return modulegraph.ModuleGraph.find_module(self, name, [p], parent)
except ImportError, err:
pass
if not os.path.isfile(p):
raise err
zi = zipimport.zipimporter(p)
m = zi.find_module(fullname.replace(".", "/"))
if m:
code = zi.get_code(fullname.replace(".", "/"))
return zi, p, ('', '', 314)
raise err
def get_zip_bytes(filename):
"""Get data from `filename` if it is a zip file path.
Returns the bytestring data read from the zip file, or None if no zip file
could be found or `filename` isn't in it. The data returned will be
an empty string if the file is empty.
"""
markers = ['.zip'+os.sep, '.egg'+os.sep]
for marker in markers:
if marker in filename:
parts = filename.split(marker)
try:
zi = zipimport.zipimporter(parts[0]+marker[:-1])
except zipimport.ZipImportError:
continue
try:
data = zi.get_data(parts[1])
except IOError:
continue
assert isinstance(data, bytes)
return data
return None
def projects(self, cls=False):
projects = []
for _, _, files in os.walk(self.full_path):
for file in files:
if file.endswith('.zip'):
project_name = file.split('.')[0]
module = zipimport.zipimporter(
os.path.join(self.full_path, file))
module = module.load_module(project_name)
_project = {
'package': project_name,
'name': module.__name__,
'author': module.__author__,
'create_date': module.__date__,
'description': module.__description__,
'version': module.__version__,
'spiders': self.spiders(project_name, cls),
'doc': module.__doc__,
'from': 'zip',
'path': os.path.join(self.full_path, file)
}
projects.append(_project)
return projects
def zip_import_data(self, filepath):
if zipimport is None:
return None
from astroid.builder import AstroidBuilder
builder = AstroidBuilder(self)
for ext in ZIP_IMPORT_EXTS:
try:
eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
except ValueError:
continue
try:
importer = zipimport.zipimporter(eggpath + ext)
zmodname = resource.replace(os.path.sep, ".")
if importer.is_package(resource):
zmodname = zmodname + ".__init__"
module = builder.string_build(
importer.get_source(resource), zmodname, filepath
)
return module
except Exception: # pylint: disable=broad-except
continue
return None
def moduleList(path):
"""
Return the list containing the names of the modules available in the given
folder.
"""
if os.path.isdir(path):
folder_list = os.listdir(path)
elif path.endswith('.egg'):
try:
folder_list = [f for f in zipimporter(path)._files]
except:
folder_list = []
else:
folder_list = []
#folder_list = glob.glob(os.path.join(path,'*'))
folder_list = [p for p in folder_list \
if os.path.exists(os.path.join(path, p,'__init__.py'))\
or p[-3:] in ('.py','.so')\
or p[-4:] in ('.pyc','.pyo','.pyd')]
folder_list = [os.path.basename(p).split('.')[0] for p in folder_list]
return folder_list
def _testBogusZipFile(self):
support.unlink(TESTMOD)
fp = open(TESTMOD, 'w+')
fp.write(struct.pack('=I', 101010256))
fp.write('a' * 18)
fp.close()
z = zipimport.zipimporter(TESTMOD)
try:
self.assertRaises(TypeError, z.find_module, None)
self.assertRaises(TypeError, z.load_module, None)
self.assertRaises(TypeError, z.is_package, None)
self.assertRaises(TypeError, z.get_code, None)
self.assertRaises(TypeError, z.get_data, None)
self.assertRaises(TypeError, z.get_source, None)
error = zipimport.ZipImportError
self.assertEqual(z.find_module('abc'), None)
self.assertRaises(error, z.load_module, 'abc')
self.assertRaises(error, z.get_code, 'abc')
self.assertRaises(OSError, z.get_data, 'abc')
self.assertRaises(error, z.get_source, 'abc')
self.assertRaises(error, z.is_package, 'abc')
finally:
zipimport._zip_directory_cache.clear()
def load_external_plugin(self, name):
if name in self.external_plugins:
return self.external_plugins[name]
# If we do not have the metadata, it was not detected by `load_external_plugins`
# on startup, or added by manual user installation after that point.
metadata = self.external_plugin_metadata.get(name, None)
if metadata is None:
self.print_error("attempted to load unknown external plugin %s" % name)
return
plugin_file_path = metadata["__file__"]
try:
zipfile = zipimport.zipimporter(plugin_file_path)
except zipimport.ZipImportError:
self.print_error("unable to load zip plugin '%s'" % plugin_file_path)
return
try:
module = zipfile.load_module(name)
except zipimport.ZipImportError as e:
self.print_error("unable to load zip plugin '%s' package '%s'" % (plugin_file_path, name), str(e))
return
sys.modules['electroncash_external_plugins.'+ name] = module
full_name = 'electroncash_external_plugins.' + name + '.' + self.gui_name
loader = pkgutil.find_loader(full_name)
if not loader:
raise RuntimeError("%s implementation for %s plugin not found"
% (self.gui_name, name))
p = loader.load_module(full_name)
plugin = p.Plugin(self, self.config, name)
plugin.set_enabled_prefix(EXTERNAL_USE_PREFIX)
self.add_jobs(plugin.thread_jobs())
self.external_plugins[name] = plugin
self.print_error("loaded external plugin", name)
return plugin
def load_from_file(self, path, key):
class_inst = None
expected_class = 'Parser'
try:
for mod_path in module.ad_paths:
if os.path.lexists(mod_path + os.path.sep +
path.split(os.path.sep)[0]):
filepath = mod_path + os.path.sep + path
filepath = os.path.normpath(filepath)
sys.path.append(os.path.dirname(filepath))
mod_name, file_ext = os.path.splitext(
os.path.split(filepath)[-1])
mod_name = '%s_%s_%s' % (self.env.cr.dbname, mod_name, key)
if file_ext.lower() == '.py':
py_mod = imp.load_source(mod_name, filepath)
elif file_ext.lower() == '.pyc':
py_mod = imp.load_compiled(mod_name, filepath)
if expected_class in dir(py_mod):
class_inst = py_mod.Parser
return class_inst
elif os.path.lexists(mod_path + os.path.sep +
path.split(os.path.sep)[0] + '.zip'):
zimp = zipimport.zipimporter(mod_path + os.path.sep +
path.split(os.path.sep)[0] +
'.zip')
return zimp.load_module(path.split(
os.path.sep)[0]).parser.Parser
except SyntaxError as e:
raise UserError(_('Syntax Error !'), e)
except Exception as e:
_logger.error(
'Error loading report parser: %s' +
(filepath and ' "%s"' % filepath or ''), e)
return None
def module_list(path):
"""
Return the list containing the names of the modules available in the given
folder.
"""
# sys.path has the cwd as an empty string, but isdir/listdir need it as '.'
if path == '':
path = '.'
# A few local constants to be used in loops below
pjoin = os.path.join
if os.path.isdir(path):
# Build a list of all files in the directory and all files
# in its subdirectories. For performance reasons, do not
# recurse more than one level into subdirectories.
files = []
for root, dirs, nondirs in os.walk(path):
subdir = root[len(path)+1:]
if subdir:
files.extend(pjoin(subdir, f) for f in nondirs)
dirs[:] = [] # Do not recurse into additional subdirectories.
else:
files.extend(nondirs)
else:
try:
files = list(zipimporter(path)._files.keys())
except:
files = []
# Build a list of modules which match the import_re regex.
modules = []
for f in files:
m = import_re.match(f)
if m:
modules.append(m.group('name'))
return list(set(modules))
def _tool_module(self):
oldpythonpath = sys.path
sys.path = self.toolpath + sys.path
try:
try:
file, path, desc = imp.find_module(self.name, self.toolpath)
try:
return imp.load_module(self.name, file, path, desc)
finally:
if file:
file.close()
except ImportError, e:
try:
import zipimport
except ImportError:
pass
else:
for aPath in self.toolpath:
try:
importer = zipimport.zipimporter(aPath)
return importer.load_module(self.name)
except ImportError, e:
pass
def load_openerp_module(module_name):
""" Load an OpenERP module, if not already loaded.
This loads the module and register all of its models, thanks to either
the MetaModel metaclass, or the explicit instantiation of the model.
This is also used to load server-wide module (i.e. it is also used
when there is no model to register).
"""
global loaded
if module_name in loaded:
return
initialize_sys_path()
try:
mod_path = get_module_path(module_name)
zip_mod_path = '' if not mod_path else mod_path + '.zip'
if not os.path.isfile(zip_mod_path):
__import__('openerp.addons.' + module_name)
else:
zimp = zipimport.zipimporter(zip_mod_path)
zimp.load_module(module_name)
# Call the module's post-load hook. This can done before any model or
# data has been initialized. This is ok as the post-load hook is for
# server-wide (instead of registry-specific) functionalities.
info = load_information_from_description_file(module_name)
if info['post_load']:
getattr(sys.modules['openerp.addons.' + module_name],
info['post_load'])()
except Exception, e:
mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or ''
msg = "Couldn't load %smodule %s" % (mt, module_name)
_logger.critical(msg)
_logger.critical(e)
raise
def download(self, cr, uid, ids, download=True, context=None):
res = []
for mod in self.browse(cr, uid, ids, context=context):
if not mod.url:
continue
match = re.search('-([a-zA-Z0-9\._-]+)(\.zip)', mod.url, re.I)
version = '0'
if match:
version = match.group(1)
if parse_version(mod.installed_version
or '0') >= parse_version(version):
continue
res.append(mod.url)
if not download:
continue
zipfile = urllib.urlopen(mod.url).read()
fname = addons.get_module_path(str(mod.name) + '.zip',
downloaded=True)
try:
fp = file(fname, 'wb')
fp.write(zipfile)
fp.close()
except Exception, e:
raise orm.except_orm(
_('Error'),
_('Can not create the module file:\n %s') % (fname, ))
terp = self.get_module_info(mod.name)
self.write(cr, uid, mod.id, self.get_values_from_terp(terp))
cr.execute('DELETE FROM ir_module_module_dependency ' \
'WHERE module_id = %s', (mod.id,))
self._update_dependencies(cr, uid, mod.id, terp.get('depends', []))
self._update_category(cr, uid, mod.id,
terp.get('category', 'Uncategorized'))
# Import module
zimp = zipimport.zipimporter(fname)
zimp.load_module(mod.name)
def platform_module(name = platform_default()):
"""Return the imported module for the platform.
This looks for a module name that matches the specified argument.
If the name is unspecified, we fetch the appropriate default for
our execution environment.
"""
full_name = 'SCons.Platform.' + name
if full_name not in sys.modules:
if os.name == 'java':
eval(full_name)
else:
try:
# the specific platform module is a relative import
mod = importlib.import_module("." + name, __name__)
except ImportError:
try:
import zipimport
importer = zipimport.zipimporter( sys.modules['SCons.Platform'].__path__[0] )
mod = importer.load_module(full_name)
except ImportError:
raise SCons.Errors.UserError("No platform named '%s'" % name)
setattr(SCons.Platform, name, mod)
return sys.modules[full_name]
def rl_get_module(name,dir):
if name in sys.modules:
om = sys.modules[name]
del sys.modules[name]
else:
om = None
try:
f = None
try:
f, p, desc= imp.find_module(name,[dir])
return imp.load_module(name,f,p,desc)
except:
if isCompactDistro():
#attempt a load from inside the zip archive
import zipimport
dir = _startswith_rl(dir)
dir = (dir=='.' or not dir) and _archive or os.path.join(_archive,dir.replace('/',os.sep))
zi = zipimport.zipimporter(dir)
return zi.load_module(name)
raise ImportError('%s[%s]' % (name,dir))
finally:
if om: sys.modules[name] = om
del om
if f: f.close()
def read_plugin(pathzipfile):
''' process uploaded plugin. '''
#test if valid zipfile
if not zipfile.is_zipfile(pathzipfile):
raise botslib.PluginError(_('Plugin is not a valid file.'))
#read index file
try:
myzipimport = zipimport.zipimporter(pathzipfile)
importedbotsindex = myzipimport.load_module('botsindex')
pluglist = importedbotsindex.plugins[:]
if 'botsindex' in sys.modules:
del sys.modules['botsindex']
except:
txt = botslib.txtexc()
raise botslib.PluginError(_('Error in plugin. Nothing is written. Error:\n%(txt)s'), {'txt': txt})
else:
botsglobal.logger.info(_('Plugin is OK.'))
botsglobal.logger.info(_('Start writing to database.'))
#write content of index file to the bots database
try:
read_index2database(pluglist)
except:
txt = botslib.txtexc()
raise botslib.PluginError(
_('Error writing plugin to database. Nothing is written. Error:\n%(txt)s'), {'txt': txt})
else:
botsglobal.logger.info(_('Writing to database is OK.'))
#write files to the file system.
botsglobal.logger.info(_('Start writing to files'))
try:
warnrenamed = False # to report in GUI files have been overwritten.
myzip = zipfile.ZipFile(pathzipfile, mode='r')
orgtargetpath = botsglobal.ini.get('directories', 'botspath')
if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
orgtargetpath = orgtargetpath[:-1]
for zipfileobject in myzip.infolist():
if zipfileobject.filename not in ['botsindex.py', 'README', 'botssys/sqlitedb/botsdb', 'config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo', '.pyc']:
#~ botsglobal.logger.info('Filename in zip "%s".',zipfileobject.filename)
if zipfileobject.filename[0] == '/':
targetpath = zipfileobject.filename[1:]
else:
targetpath = zipfileobject.filename
#convert for correct environment: repacle botssys, config, usersys in filenames
if targetpath.startswith('usersys'):
targetpath = targetpath.replace('usersys', botsglobal.ini.get('directories', 'usersysabs'), 1)
elif targetpath.startswith('botssys'):
targetpath = targetpath.replace('botssys', botsglobal.ini.get('directories', 'botssys'), 1)
elif targetpath.startswith('config'):
targetpath = targetpath.replace('config', botsglobal.ini.get('directories', 'config'), 1)
targetpath = botslib.join(orgtargetpath, targetpath)
#targetpath is OK now.
botsglobal.logger.info(_(' Start writing file: "%(targetpath)s".'), {'targetpath': targetpath})
if botslib.dirshouldbethere(os.path.dirname(targetpath)):
botsglobal.logger.info(_(' Create directory "%(directory)s".'),
{'directory': os.path.dirname(targetpath)})
if zipfileobject.filename[-1] == '/': # check if this is a dir; if so continue
continue
if os.path.isfile(targetpath): # check if file already exists
try: # this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
warnrenamed = True
except:
pass
source = myzip.read(zipfileobject.filename)
target = open(targetpath, 'wb')
target.write(source)
target.close()
botsglobal.logger.info(_(' File written: "%(targetpath)s".'), {'targetpath': targetpath})
except:
txt = botslib.txtexc()
myzip.close()
raise botslib.PluginError(
_('Error writing files to system. Nothing is written to database. Error:\n%(txt)s'), {'txt': txt})
else:
myzip.close()
botsglobal.logger.info(_('Writing files to filesystem is OK.'))
return warnrenamed
def add(priority, name, code=None):
"""
Adds a dependency to the loader
:param priority:
A two-digit string. If a dep has no dependencies, this can be
something like '01'. If it does, use something like '10' leaving
space for other entries
:param name:
The name of the dependency as a unicode string
:param code:
Any special loader code, otherwise the default will be used
"""
if not code:
code = _default_loader(name)
loader_filename = '%s-%s.py' % (priority, name)
just_created_loader = False
loader_metadata = {
"version": "1.0.0",
"sublime_text": "*",
# Tie the loader to the platform so we can detect
# people syncing packages incorrectly.
"platforms": [sublime.platform()],
"url": "https://github.com/wbond/package_control/issues",
"description": "Package Control dependency loader"
}
loader_metadata_enc = json.dumps(loader_metadata).encode('utf-8')
if sys.version_info < (3, ):
if not path.exists(loader_package_path):
just_created_loader = True
os.mkdir(loader_package_path, 0o755)
with open(
path.join(loader_package_path, 'dependency-metadata.json'),
'wb') as f:
f.write(loader_metadata_enc)
loader_path = path.join(loader_package_path, loader_filename)
with open(loader_path, 'wb') as f:
f.write(code.encode('utf-8'))
else:
# Make sure Python doesn't use the old file listing for the loader
# when trying to import modules
if loader_package_path in zipimport._zip_directory_cache:
del zipimport._zip_directory_cache[loader_package_path]
try:
loader_lock.acquire()
# If a swap of the loader .sublime-package was queued because of a
# file being removed, we need to add the new loader code the the
# .sublime-package that will be swapped into place shortly.
if swap_event.in_process() and os.path.exists(
new_loader_package_path):
package_to_update = new_loader_package_path
else:
package_to_update = loader_package_path
mode = 'w'
just_created_loader = True
# Only append if the file exists and is a valid zip file
if os.path.exists(package_to_update):
# Even if the loader was invalid, it still won't show up as a
# "new" file via filesystem notifications, so we have to
# manually load the code.
just_created_loader = False
try:
with zipfile.ZipFile(package_to_update, 'r') as rz:
# Make sure the zip file can be read
res = rz.testzip()
if res is not None:
raise zipfile.BadZipfile('zip test failed')
mode = 'a'
except (zipfile.BadZipfile, OSError):
os.unlink(package_to_update)
with zipfile.ZipFile(package_to_update, mode) as z:
if mode == 'w':
z.writestr('dependency-metadata.json', loader_metadata_enc)
z.writestr(loader_filename, code.encode('utf-8'))
__update_loaders(z)
finally:
loader_lock.release()
if not just_created_loader and not swap_event.in_process():
# Manually execute the loader code because Sublime Text does not
# detect changes to the zip archive, only if the file is new.
importer = zipimport.zipimporter(loader_package_path)
importer.load_module(loader_filename[0:-3])
# Clean things up for people who were tracking the master branch
if just_created_loader:
old_loader_sp = path.join(installed_packages_dir,
'0-package_control_loader.sublime-package')
old_loader_dir = path.join(packages_dir, '0-package_control_loader')
removed_old_loader = False
if path.exists(old_loader_sp):
removed_old_loader = True
os.remove(old_loader_sp)
if path.exists(old_loader_dir):
removed_old_loader = True
try:
shutil.rmtree(old_loader_dir)
except (OSError):
open(os.path.join(old_loader_dir, 'package-control.cleanup'),
'w').close()
if removed_old_loader:
console_write(u'''
Cleaning up remenants of old loaders
''')
pc_settings = sublime.load_settings(pc_settings_filename())
orig_installed_packages = load_list_setting(
pc_settings, 'installed_packages')
installed_packages = list(orig_installed_packages)
if '0-package_control_loader' in installed_packages:
installed_packages.remove('0-package_control_loader')
for name in ['bz2', 'ssl-linux', 'ssl-windows']:
dep_dir = path.join(packages_dir, name)
if path.exists(dep_dir):
try:
shutil.rmtree(dep_dir)
except (OSError):
open(os.path.join(dep_dir, 'package-control.cleanup'),
'w').close()
if name in installed_packages:
installed_packages.remove(name)
save_list_setting(pc_settings, pc_settings_filename(),
'installed_packages', installed_packages,
orig_installed_packages)
def _get_metadata(self, path):
requires = None
def parse_requires_data(data):
"""Create a list of dependencies from a requires.txt file.
*data*: the contents of a setuptools-produced requires.txt file.
"""
reqs = []
lines = data.splitlines()
for line in lines:
line = line.strip()
if line.startswith('['):
logger.warning(
'Unexpected line: quitting requirement scan: %r', line)
break
r = parse_requirement(line)
if not r:
logger.warning('Not recognised as a requirement: %r', line)
continue
if r.extras:
logger.warning('extra requirements in requires.txt are '
'not supported')
if not r.constraints:
reqs.append(r.name)
else:
cons = ', '.join('%s%s' % c for c in r.constraints)
reqs.append('%s (%s)' % (r.name, cons))
return reqs
def parse_requires_path(req_path):
"""Create a list of dependencies from a requires.txt file.
*req_path*: the path to a setuptools-produced requires.txt file.
"""
reqs = []
try:
with codecs.open(req_path, 'r', 'utf-8') as fp:
reqs = parse_requires_data(fp.read())
except IOError:
pass
return reqs
tl_path = tl_data = None
if path.endswith('.egg'):
if os.path.isdir(path):
p = os.path.join(path, 'EGG-INFO')
meta_path = os.path.join(p, 'PKG-INFO')
metadata = Metadata(path=meta_path, scheme='legacy')
req_path = os.path.join(p, 'requires.txt')
tl_path = os.path.join(p, 'top_level.txt')
requires = parse_requires_path(req_path)
else:
# FIXME handle the case where zipfile is not available
zipf = zipimport.zipimporter(path)
fileobj = StringIO(
zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8'))
metadata = Metadata(fileobj=fileobj, scheme='legacy')
try:
data = zipf.get_data('EGG-INFO/requires.txt')
tl_data = zipf.get_data('EGG-INFO/top_level.txt').decode(
'utf-8')
requires = parse_requires_data(data.decode('utf-8'))
except IOError:
requires = None
elif path.endswith('.egg-info'):
if os.path.isdir(path):
req_path = os.path.join(path, 'requires.txt')
requires = parse_requires_path(req_path)
path = os.path.join(path, 'PKG-INFO')
tl_path = os.path.join(path, 'top_level.txt')
metadata = Metadata(path=path, scheme='legacy')
else:
raise DistlibException('path must end with .egg-info or .egg, '
'got %r' % path)
if requires:
metadata.add_requirements(requires)
# look for top-level modules in top_level.txt, if present
if tl_data is None:
if tl_path is not None and os.path.exists(tl_path):
with open(tl_path, 'rb') as f:
tl_data = f.read().decode('utf-8')
if not tl_data:
tl_data = []
else:
tl_data = tl_data.splitlines()
self.modules = tl_data
return metadata
#!/usr/bin/python
import re
import sys
from collections import Counter
import zipimport
importer = zipimport.zipimporter('nltk.zip')
nltk = importer.load_module('nltk')
from nltk.stem.snowball import SnowballStemmer
stemmer = SnowballStemmer('english')
COUNTS_FILE_NAME = "output_full_stemmed_stopwords.txt"
CLASSES_LIST_FILE_NAME = "classes.txt"
STOPWORDS_FILE_NAME = "stopwords.txt"
ALPHA = 0.1
VOCABULARY_SIZE = 300000
def remove_till_first_quote(text):
regex = r"^(.*?)\""
text = re.sub(regex, '', text)
return text
def remove_unicode(text):
"""Replace unicode codes like \uxxxx with space"""
regex = r"(\\u....)"
text = re.sub(regex, ' ', text)
return text
def __init__(self, *args, **kwargs):
self.importer = zipimporter(*args, **kwargs)
import traceback
traceback.print_exc()
print(50 * '#')
raise
extrasLoaded.append(modname)
if hasattr(sys, 'frozen') and not hasattr(m, '__loader__') and len(
m.__path__) == 1:
zip = m.__path__[0].split('/wooExtra/')[0].split(
'\\wooExtra\\')[0]
if not (zip.endswith('.zip') or zip.endswith('.egg')):
print(
'wooExtra.%s: not a .zip or .egg, no __loader__ set (%s)'
% (modname, zip))
else:
print('wooExtra.%s: setting __loader__ and __file__' %
modname)
m.__loader__ = zipimport.zipimporter(zip)
m.__file__ = os.path.join(m.__path__[0],
os.path.basename(m.__file__))
except ImportError:
sys.stderr.write('ERROR importing wooExtra.%s:' % modname)
raise
# disable informative message if plain import into python script
if sys.argv[0].split('/')[-1].startswith('woo'):
sys.stderr.write('wooExtra modules loaded: %s.\n' %
(', '.join(extrasLoaded)))
except ImportError:
# no wooExtra packages are installed
pass
except KeyError:
try:
smpath = sys.modules['SCons.Tool'].__path__
try:
file, path, desc = imp.find_module(self.name, smpath)
module = imp.load_module(full_name, file, path, desc)
setattr(SCons.Tool, self.name, module)
if file:
file.close()
return module
except ImportError, e:
if str(e) != "No module named %s" % self.name:
raise SCons.Errors.EnvironmentError(e)
try:
import zipimport
importer = zipimport.zipimporter(
sys.modules['SCons.Tool'].__path__[0])
module = importer.load_module(full_name)
setattr(SCons.Tool, self.name, module)
return module
except ImportError, e:
m = "No tool named '%s': %s" % (self.name, e)
raise SCons.Errors.EnvironmentError(m)
except ImportError, e:
m = "No tool named '%s': %s" % (self.name, e)
raise SCons.Errors.EnvironmentError(m)
def __call__(self, env, *args, **kw):
if self.init_kw is not None:
# Merge call kws into init kws;
# but don't bash self.init_kw.
if kw is not None:
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Retrieving the code for a module within a zip archive.
"""
#end_pymotw_header
import zipimport
importer = zipimport.zipimporter('zipimport_example.zip')
module = importer.load_module('zipimport_get_code')
print 'Name :', module.__name__
print 'Loader :', module.__loader__
print 'Code :', module.code
def _module_file(modpath, path=None):
"""get a module type / file path
:type modpath: list or tuple
:param modpath:
splitted module's name (i.e name of a module or package splitted
on '.'), with leading empty strings for explicit relative import
:type path: list or None
:param path:
optional list of path where the module or package should be
searched (use sys.path if nothing or None is given)
:rtype: tuple(int, str)
:return: the module type flag and the file path for a module
"""
# egg support compat
try:
pic = sys.path_importer_cache
_path = (path is None and sys.path or path)
for __path in _path:
if not __path in pic:
try:
pic[__path] = zipimport.zipimporter(__path)
except zipimport.ZipImportError:
pic[__path] = None
checkeggs = True
except AttributeError:
checkeggs = False
# pkg_resources support (aka setuptools namespace packages)
if pkg_resources is not None and modpath[
0] in pkg_resources._namespace_packages and len(modpath) > 1:
# setuptools has added into sys.modules a module object with proper
# __path__, get back information from there
module = sys.modules[modpath.pop(0)]
path = module.__path__
imported = []
while modpath:
modname = modpath[0]
# take care to changes in find_module implementation wrt builtin modules
#
# Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23)
# >>> imp.find_module('posix')
# (None, 'posix', ('', '', 6))
#
# Python 3.3.1 (default, Apr 26 2013, 12:08:46)
# >>> imp.find_module('posix')
# (None, None, ('', '', 6))
try:
_, mp_filename, mp_desc = find_module(modname, path)
except ImportError:
if checkeggs:
return _search_zip(modpath, pic)[:2]
raise
else:
if checkeggs and mp_filename:
fullabspath = [abspath(x) for x in _path]
try:
pathindex = fullabspath.index(dirname(
abspath(mp_filename)))
emtype, emp_filename, zippath = _search_zip(modpath, pic)
if pathindex > _path.index(zippath):
# an egg takes priority
return emtype, emp_filename
except ValueError:
# XXX not in _path
pass
except ImportError:
pass
checkeggs = False
imported.append(modpath.pop(0))
mtype = mp_desc[2]
if modpath:
if mtype != PKG_DIRECTORY:
raise ImportError('No module %s in %s' %
('.'.join(modpath), '.'.join(imported)))
# XXX guess if package is using pkgutil.extend_path by looking for
# those keywords in the first four Kbytes
try:
with open(join(mp_filename, '__init__.py')) as stream:
data = stream.read(4096)
except IOError:
path = [mp_filename]
else:
if 'pkgutil' in data and 'extend_path' in data:
# extend_path is called, search sys.path for module/packages
# of this name see pkgutil.extend_path documentation
path = [
join(p, *imported) for p in sys.path
if isdir(join(p, *imported))
]
else:
path = [mp_filename]
return mtype, mp_filename
def testZipImporterMethodsInSubDirectory(self):
packdir = TESTPACK + os.sep
packdir2 = packdir + TESTPACK2 + os.sep
files = {
packdir2 + "__init__" + pyc_ext: (NOW, test_pyc),
packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)
}
self.addCleanup(os_helper.unlink, TEMP_ZIP)
with ZipFile(TEMP_ZIP, "w") as z:
for name, (mtime, data) in files.items():
zinfo = ZipInfo(name, time.localtime(mtime))
zinfo.compress_type = self.compression
zinfo.comment = b"eggs"
z.writestr(zinfo, data)
zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir)
self.assertEqual(zi.archive, TEMP_ZIP)
self.assertEqual(zi.prefix, packdir)
self.assertTrue(zi.is_package(TESTPACK2))
# PEP 302
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
mod = zi.load_module(TESTPACK2)
self.assertEqual(zi.get_filename(TESTPACK2), mod.__file__)
# PEP 451
spec = zi.find_spec(TESTPACK2)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
self.assertEqual(spec.loader.get_filename(TESTPACK2), mod.__file__)
self.assertFalse(zi.is_package(TESTPACK2 + os.sep + '__init__'))
self.assertFalse(zi.is_package(TESTPACK2 + os.sep + TESTMOD))
pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2
zi2 = zipimport.zipimporter(pkg_path)
# PEP 302
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
find_mod_dotted = zi2.find_module(TESTMOD)
self.assertIsNotNone(find_mod_dotted)
self.assertIsInstance(find_mod_dotted, zipimport.zipimporter)
self.assertFalse(zi2.is_package(TESTMOD))
load_mod = find_mod_dotted.load_module(TESTMOD)
self.assertEqual(find_mod_dotted.get_filename(TESTMOD),
load_mod.__file__)
# PEP 451
spec = zi2.find_spec(TESTMOD)
self.assertIsNotNone(spec)
self.assertIsInstance(spec.loader, zipimport.zipimporter)
self.assertFalse(spec.loader.is_package(TESTMOD))
load_mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(load_mod)
self.assertEqual(spec.loader.get_filename(TESTMOD), load_mod.__file__)
mod_path = TESTPACK2 + os.sep + TESTMOD
mod_name = module_path_to_dotted_name(mod_path)
mod = importlib.import_module(mod_name)
self.assertTrue(mod_name in sys.modules)
self.assertIsNone(zi.get_source(TESTPACK2))
self.assertIsNone(zi.get_source(mod_path))
self.assertEqual(zi.get_filename(mod_path), mod.__file__)
# To pass in the module name instead of the path, we must use the
# right importer.
loader = mod.__loader__
self.assertIsNone(loader.get_source(mod_name))
self.assertEqual(loader.get_filename(mod_name), mod.__file__)
def __init__(self, path):
try:
self.__zip = zipimport.zipimporter(path)
except zipimport.ZipImportError, e:
raise OwnerError('%s: %s' % (str(e), path))
def zipimporter_unicode(archivepath):
if sys.version_info.major == 2 and isinstance(archivepath, unicode):
archivepath = archivepath.encode(sys.getfilesystemencoding())
return zipimport.zipimporter(archivepath)
def _module_file(modpath, path=None):
"""get a module type / file path
:type modpath: list or tuple
:param modpath:
splitted module's name (i.e name of a module or package splitted
on '.'), with leading empty strings for explicit relative import
:type path: list or None
:param path:
optional list of path where the module or package should be
searched (use sys.path if nothing or None is given)
:rtype: tuple(int, str)
:return: the module type flag and the file path for a module
"""
# egg support compat
try:
pic = sys.path_importer_cache
_path = (path is None and sys.path or path)
for __path in _path:
if not __path in pic:
try:
pic[__path] = zipimport.zipimporter(__path)
except zipimport.ZipImportError:
pic[__path] = None
checkeggs = True
except AttributeError:
checkeggs = False
imported = []
while modpath:
try:
_, mp_filename, mp_desc = find_module(modpath[0], path)
except ImportError:
if checkeggs:
return _search_zip(modpath, pic)[:2]
raise
else:
if checkeggs:
fullabspath = [abspath(x) for x in _path]
try:
pathindex = fullabspath.index(dirname(abspath(mp_filename)))
emtype, emp_filename, zippath = _search_zip(modpath, pic)
if pathindex > _path.index(zippath):
# an egg takes priority
return emtype, emp_filename
except ValueError:
# XXX not in _path
pass
except ImportError:
pass
checkeggs = False
imported.append(modpath.pop(0))
mtype = mp_desc[2]
if modpath:
if mtype != PKG_DIRECTORY:
raise ImportError('No module %s in %s' % ('.'.join(modpath),
'.'.join(imported)))
path = [mp_filename]
return mtype, mp_filename
def _tool_module(self):
oldpythonpath = sys.path
sys.path = self.toolpath + sys.path
# sys.stderr.write("Tool:%s\nPATH:%s\n"%(self.name,sys.path))
# From: http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path/67692#67692
# import importlib.util
# spec = importlib.util.spec_from_file_location("module.name", "/path/to/file.py")
# foo = importlib.util.module_from_spec(spec)
# spec.loader.exec_module(foo)
# foo.MyClass()
# Py 3 code
# sys.stderr.write("toolpath:%s\n" % self.toolpath)
# sys.stderr.write("SCONS.TOOL path:%s\n" % sys.modules['SCons.Tool'].__path__)
debug = False
spec = None
found_name = self.name
add_to_scons_tools_namespace = False
for path in self.toolpath:
sepname = self.name.replace('.', os.path.sep)
file_path = os.path.join(path, "%s.py" % sepname)
file_package = os.path.join(path, sepname)
if debug:
sys.stderr.write("Trying:%s %s\n" % (file_path, file_package))
if os.path.isfile(file_path):
spec = importlib.util.spec_from_file_location(
self.name, file_path)
if debug: print("file_Path:%s FOUND" % file_path)
break
elif os.path.isdir(file_package):
file_package = os.path.join(file_package, '__init__.py')
spec = importlib.util.spec_from_file_location(
self.name, file_package)
if debug: print("PACKAGE:%s Found" % file_package)
break
else:
continue
if spec is None:
if debug: sys.stderr.write("NO SPEC :%s\n" % self.name)
spec = importlib.util.find_spec("." + self.name,
package='SCons.Tool')
if spec:
found_name = 'SCons.Tool.' + self.name
add_to_scons_tools_namespace = True
if debug:
sys.stderr.write("Spec Found? .%s :%s\n" % (self.name, spec))
if spec is None:
error_string = "No module named %s" % self.name
raise SCons.Errors.SConsEnvironmentError(error_string)
module = importlib.util.module_from_spec(spec)
if module is None:
if debug: print("MODULE IS NONE:%s" % self.name)
error_string = "No module named %s" % self.name
raise SCons.Errors.SConsEnvironmentError(error_string)
# Don't reload a tool we already loaded.
sys_modules_value = sys.modules.get(found_name, False)
found_module = None
if sys_modules_value and sys_modules_value.__file__ == spec.origin:
found_module = sys.modules[found_name]
else:
# Not sure what to do in the case that there already
# exists sys.modules[self.name] but the source file is
# different.. ?
module = spec.loader.load_module(spec.name)
sys.modules[found_name] = module
if add_to_scons_tools_namespace:
# If we found it in SCons.Tool, then add it to the module
setattr(SCons.Tool, self.name, module)
found_module = module
if found_module is not None:
sys.path = oldpythonpath
return found_module
sys.path = oldpythonpath
full_name = 'SCons.Tool.' + self.name
try:
return sys.modules[full_name]
except KeyError:
try:
smpath = sys.modules['SCons.Tool'].__path__
try:
module, file = self._load_dotted_module_py2(
self.name, full_name, smpath)
setattr(SCons.Tool, self.name, module)
if file:
file.close()
return module
except ImportError as e:
if str(e) != "No module named %s" % self.name:
raise SCons.Errors.SConsEnvironmentError(e)
try:
import zipimport
importer = zipimport.zipimporter(
sys.modules['SCons.Tool'].__path__[0])
module = importer.load_module(full_name)
setattr(SCons.Tool, self.name, module)
return module
except ImportError as e:
m = "No tool named '%s': %s" % (self.name, e)
raise SCons.Errors.SConsEnvironmentError(m)
except ImportError as e:
m = "No tool named '%s': %s" % (self.name, e)
raise SCons.Errors.SConsEnvironmentError(m)
#!/usr/bin/env python
import sys, zipimport
# from operator import itemgetter
reload(sys)
sys.setdefaultencoding('utf-8')
importer = zipimport.zipimporter('nltk.mod')
nltk = importer.load_module('nltk')
nltk.data.path += ["./nltkData/"]
stopwords_list = nltk.corpus.stopwords.words('english')
word = None
tag = None
current_word = None
curent_tag = None
current_count = 0
for line in sys.stdin:
line = line.strip() # remove whitespace
word, tag, count = line.split(',', 2) # splits tuple into 3 parts
try:
count = int(count)
except ValueError:
continue
if word not in stopwords_list:
FILE_NAME = sys.executable
DIR_NAME = os.path.dirname(FILE_NAME)
INITSCRIPT_ZIP_FILE_NAME = os.path.dirname(os.__file__)
paths = os.environ.get("LD_LIBRARY_PATH", "").split(os.pathsep)
if DIR_NAME not in paths:
paths.insert(0, DIR_NAME)
os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(paths)
os.execv(sys.executable, sys.argv)
sys.frozen = True
sys.path = sys.path[:4]
os.environ["TCL_LIBRARY"] = os.path.join(DIR_NAME, "tcl")
os.environ["TK_LIBRARY"] = os.path.join(DIR_NAME, "tk")
INITSCRIPT_ZIP_FILE_NAME = os.path.dirname(os.__file__)
m = __import__("__main__")
importer = zipimport.zipimporter(INITSCRIPT_ZIP_FILE_NAME)
name, ext = os.path.splitext(os.path.basename(os.path.normcase(FILE_NAME)))
moduleName = "%s__main__" % name
code = importer.get_code(moduleName)
exec(code, m.__dict__)
versionInfo = sys.version_info[:3]
if versionInfo >= (2, 5, 0) and versionInfo <= (2, 6, 4):
module = sys.modules.get("threading")
if module is not None:
module._shutdown()