def read(self, size):
if self._eof:
raise SubprocessStreamEOFError
if self._pos + size <= self._buf_len:
buf = bytes(self._buf[self._pos:self._pos + size])
self._pos += size
return buf
else:
size_left = size
bio = BytesIO()
while size_left:
c = self._buf[self._pos:min(self._pos +
size_left, self._buf_len)]
bio.write(c)
size_left -= len(c)
self._pos += len(c)
if size_left <= 0:
break
self._q_req.put(options.cupid.mp_buffer_size)
recv_size = self._q_rep.get()
if recv_size < 0:
self._eof = True
break
self._buf_len = recv_size
self._pos = 0
return bio.getvalue()
def testBinaryImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.so', '')
zip_f.close()
zip_io.seek(0)
self.assertRaises(SystemError, CompressImporter,
zipfile.ZipFile(zip_io, 'r'))
try:
zip_io.seek(0)
CompressImporter(zipfile.ZipFile(zip_io, 'r'),
extract=True,
_match_version=False)
self.assertTrue(os.path.exists(CompressImporter._extract_path))
finally:
shutil.rmtree(CompressImporter._extract_path)
CompressImporter._extract_path = None
temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-')
lib_path = os.path.join(temp_path, 'mylib')
os.makedirs(lib_path)
lib_dict = dict()
try:
six_path = os.path.join(
os.path.dirname(os.path.abspath(six.__file__)), 'six.py')
shutil.copy(six_path, os.path.join(lib_path, 'five.py'))
dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
dummy_bin.close()
lib_files = ['five.py', 'dummy.so']
lib_dict = dict((os.path.join(lib_path, fn),
open(os.path.join(lib_path, fn), 'r'))
for fn in lib_files)
importer.ALLOW_BINARY = False
self.assertRaises(SystemError, CompressImporter, lib_dict)
importer.ALLOW_BINARY = True
sys.meta_path.append(CompressImporter(lib_dict))
import five
self.assertEqual(list(to_binary('abc')),
list(five.binary_type(to_binary('abc'))))
finally:
[f.close() for f in six.itervalues(lib_dict)]
shutil.rmtree(temp_path)
def testRealImport(self):
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
'six.py')
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.write(six_path, arcname='mylib/five.py')
zip_f.close()
zip_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
sys.meta_path.append(CompressImporter(zip_f))
import five
self.assertEqual(list(to_binary('abc')),
list(five.binary_type(to_binary('abc'))))
def read(self, size=-1):
if Subprocess_StdIStream_eof(self._ptr):
raise SubprocessStreamEOFError('Subprocess stream exhausted')
if size < 0:
buf_io = BytesIO()
while True:
try:
chunk = self.read(READ_CHUNK)
except SubprocessStreamEOFError:
chunk = b''
buf_io.write(chunk)
if len(chunk) < READ_CHUNK:
break
return buf_io.getvalue()
else:
p = create_string_buffer(size)
Subprocess_StdIStream_read(self._ptr, p, size)
return p.raw
def testRealImport(self):
raw_meta_path = sys.meta_path
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py')
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.write(six_path, arcname='mylib/five.py')
zip_f.close()
zip_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
sys.meta_path.append(CompressImporter(zip_f))
try:
import five
self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc'))))
finally:
sys.meta_path = raw_meta_path
def testImport(self):
raw_meta_path = sys.meta_path
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
sys.meta_path.append(CompressImporter(zip_f, tar_f))
try:
from testb.b import b
self.assertEqual(b, 2)
finally:
sys.meta_path = raw_meta_path
def testImport(self):
raw_meta_path = sys.meta_path
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'), fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
sys.meta_path.append(CompressImporter(zip_f, tar_f))
try:
from testb.b import b
self.assertEqual(b, 2)
finally:
sys.meta_path = raw_meta_path
def testImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
dict_io_init = dict()
dict_io_init['/opt/test_pyodps_dev/testc/__init__.py'] = StringIO()
dict_io_init['/opt/test_pyodps_dev/testc/c.py'] = StringIO(
to_binary('from a import a; c = a + 2'))
dict_io_file = dict()
dict_io_file['/opt/test_pyodps_dev/testd/d.py'] = StringIO(
to_binary('from a import a; d = a + 3'))
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
importer.ALLOW_BINARY = False
imp = CompressImporter(zip_f, tar_f, dict_io_init, dict_io_file)
self.assertEqual(len(imp._files), 4)
sys.meta_path.append(imp)
from testb.b import b
self.assertEqual(b, 2)
from testc.c import c
self.assertEqual(c, 3)
self.assertRaises(ImportError, __import__, 'c', fromlist=[])
from d import d
self.assertEqual(d, 4)
def testBinaryImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.so', '')
zip_f.close()
zip_io.seek(0)
self.assertRaises(SystemError, CompressImporter,
zipfile.ZipFile(zip_io, 'r'))
try:
zip_io.seek(0)
CompressImporter(zipfile.ZipFile(zip_io, 'r'),
extract=True,
_match_version=False)
self.assertTrue(os.path.exists(CompressImporter._extract_path))
finally:
shutil.rmtree(CompressImporter._extract_path)
CompressImporter._extract_path = None
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
'six.py')
temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-')
lib_path = os.path.join(temp_path, 'mylib')
lib_path2 = os.path.join(temp_path, 'mylib2')
os.makedirs(lib_path)
os.makedirs(lib_path2)
lib_dict = dict()
try:
with open(os.path.join(lib_path, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(lib_path, 'fake_six.py'))
dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
dummy_bin.close()
sub_lib_path = os.path.join(lib_path, 'sub_path')
os.makedirs(sub_lib_path)
with open(os.path.join(sub_lib_path, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py'))
lib_files = [
'__init__.py', 'fake_six.py', 'dummy.so',
os.path.join('sub_path', '__init__.py'),
os.path.join('sub_path', 'fake_six.py')
]
lib_dict = dict((os.path.join(lib_path, fn),
open(os.path.join(lib_path, fn), 'r'))
for fn in lib_files)
importer.ALLOW_BINARY = False
self.assertRaises(SystemError, CompressImporter, lib_dict)
importer.ALLOW_BINARY = True
importer_obj = CompressImporter(lib_dict)
sys.meta_path.append(importer_obj)
from mylib import fake_six
self.assertEqual(list(to_binary('abc')),
list(fake_six.binary_type(to_binary('abc'))))
self.assertRaises(ImportError, __import__, 'sub_path', fromlist=[])
with open(os.path.join(lib_path2, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(lib_path2, 'fake_six.py'))
dummy_bin = open(os.path.join(lib_path2, 'dummy.so'), 'w')
dummy_bin.close()
sub_lib_path = os.path.join(lib_path2, 'sub_path2')
os.makedirs(sub_lib_path)
with open(os.path.join(sub_lib_path, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py'))
lib_files = [
'__init__.py', 'fake_six.py', 'dummy.so',
os.path.join('sub_path2', '__init__.py'),
os.path.join('sub_path2', 'fake_six.py')
]
importer.ALLOW_BINARY = True
importer_obj = CompressImporter(lib_files)
sys.meta_path.append(importer_obj)
from mylib2 import fake_six
self.assertEqual(list(to_binary('abc')),
list(fake_six.binary_type(to_binary('abc'))))
self.assertRaises(ImportError,
__import__,
'sub_path2',
fromlist=[])
finally:
[f.close() for f in six.itervalues(lib_dict)]
shutil.rmtree(temp_path)