def testImport(self):
raw_meta_path = sys.meta_path
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
sys.meta_path.append(CompressImporter(zip_f, tar_f))
try:
from testb.b import b
self.assertEqual(b, 2)
finally:
sys.meta_path = raw_meta_path
def testBinaryImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.so', '')
zip_f.close()
self.assertRaises(SystemError, CompressImporter, zip_f)
temp_path = tempfile.mkdtemp(prefix='tmp_pyodps')
lib_path = os.path.join(temp_path, 'mylib')
os.makedirs(lib_path)
lib_dict = dict()
try:
six_path = os.path.join(
os.path.dirname(os.path.abspath(six.__file__)), 'six.py')
shutil.copy(six_path, os.path.join(lib_path, 'five.py'))
dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
dummy_bin.close()
lib_files = ['five.py', 'dummy.so']
lib_dict = dict((os.path.join(lib_path, fn),
open(os.path.join(lib_path, fn), 'r'))
for fn in lib_files)
importer.ALLOW_BINARY = False
self.assertRaises(SystemError, CompressImporter, lib_dict)
importer.ALLOW_BINARY = True
sys.meta_path.append(CompressImporter(lib_dict))
import five
self.assertEqual(list(to_binary('abc')),
list(five.binary_type(to_binary('abc'))))
finally:
[f.close() for f in six.itervalues(lib_dict)]
shutil.rmtree(temp_path)
def testRealImport(self):
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
'six.py')
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.write(six_path, arcname='mylib/five.py')
zip_f.close()
zip_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
sys.meta_path.append(CompressImporter(zip_f))
import five
self.assertEqual(list(to_binary('abc')),
list(five.binary_type(to_binary('abc'))))
def testImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
dict_io_init = dict()
dict_io_init['/opt/test_pyodps_dev/testc/__init__.py'] = StringIO()
dict_io_init['/opt/test_pyodps_dev/testc/c.py'] = StringIO(
to_binary('from a import a; c = a + 2'))
dict_io_file = dict()
dict_io_file['/opt/test_pyodps_dev/testd/d.py'] = StringIO(
to_binary('from a import a; d = a + 3'))
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
importer.ALLOW_BINARY = False
imp = CompressImporter(zip_f, tar_f, dict_io_init, dict_io_file)
self.assertEqual(len(imp._files), 4)
sys.meta_path.append(imp)
from testb.b import b
self.assertEqual(b, 2)
from testc.c import c
self.assertEqual(c, 3)
self.assertRaises(ImportError, __import__, 'c', fromlist=[])
from d import d
self.assertEqual(d, 4)
def execute(self, line, cell=''):
if self._odps is None:
self._odps = enter().odps
sql = line + '\n' + cell
sql = sql.strip()
if sql:
bar = init_progress_bar()
instance = self._odps.run_sql(sql)
percent = 0
while not instance.is_terminated():
task_names = instance.get_task_names()
last_percent = percent
if len(task_names) > 0:
percent = sum(
self._get_task_percent(instance, name)
for name in task_names) / len(task_names)
else:
percent = 0
percent = min(1, max(percent, last_percent))
bar.update(percent)
time.sleep(1)
instance.wait_for_success()
bar.update(1)
with instance.open_reader() as reader:
try:
import pandas as pd
try:
return pd.read_csv(StringIO(reader.raw))
except ValueError:
return reader.raw
except ImportError:
return ResultFrame(list(reader), columns=reader._columns)
def testBinaryImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.so', '')
zip_f.close()
zip_io.seek(0)
self.assertRaises(SystemError, CompressImporter,
zipfile.ZipFile(zip_io, 'r'))
try:
zip_io.seek(0)
CompressImporter(zipfile.ZipFile(zip_io, 'r'),
extract=True,
_match_version=False)
self.assertTrue(os.path.exists(CompressImporter._extract_path))
finally:
shutil.rmtree(CompressImporter._extract_path)
CompressImporter._extract_path = None
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
'six.py')
temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-')
lib_path = os.path.join(temp_path, 'mylib')
os.makedirs(lib_path)
lib_dict = dict()
try:
with open(os.path.join(lib_path, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(lib_path, 'fake_six.py'))
dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
dummy_bin.close()
sub_lib_path = os.path.join(lib_path, 'sub_path')
os.makedirs(sub_lib_path)
with open(os.path.join(sub_lib_path, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py'))
lib_files = [
'__init__.py', 'fake_six.py', 'dummy.so',
os.path.join('sub_path', '__init__.py'),
os.path.join('sub_path', 'fake_six.py')
]
lib_dict = dict((os.path.join(lib_path, fn),
open(os.path.join(lib_path, fn), 'r'))
for fn in lib_files)
importer.ALLOW_BINARY = False
self.assertRaises(SystemError, CompressImporter, lib_dict)
importer.ALLOW_BINARY = True
importer_obj = CompressImporter(lib_dict)
sys.meta_path.append(importer_obj)
from mylib import fake_six
self.assertEqual(list(to_binary('abc')),
list(fake_six.binary_type(to_binary('abc'))))
self.assertRaises(ImportError, __import__, 'sub_path', fromlist=[])
finally:
[f.close() for f in six.itervalues(lib_dict)]
shutil.rmtree(temp_path)
def execute(self, line, cell=''):
self._set_odps()
content = line + '\n' + cell
content = content.strip()
sql = None
hints = dict()
splits = content.split(';')
for s in splits:
stripped = s.strip()
if stripped.lower().startswith('set '):
hint = stripped.split(' ', 1)[1]
k, v = hint.split('=', 1)
k, v = k.strip(), v.strip()
hints[k] = v
elif len(stripped) == 0:
continue
else:
if sql is None:
sql = s
else:
sql = '%s;%s' % (sql, s)
# replace user defined parameters
sql = replace_sql_parameters(sql, self.shell.user_ns)
if sql:
bar = init_progress_bar()
instance = self._odps.run_sql(sql, hints=hints)
if options.verbose:
stdout = options.verbose_log or self._to_stdout
stdout('Instance ID: ' + instance.id)
stdout(' Log view: ' + instance.get_logview_address())
percent = 0
while not instance.is_terminated():
task_names = instance.get_task_names()
last_percent = percent
if len(task_names) > 0:
percent = sum(self._get_task_percent(instance, name)
for name in task_names) / len(task_names)
else:
percent = 0
percent = min(1, max(percent, last_percent))
bar.update(percent)
time.sleep(1)
instance.wait_for_success()
bar.update(1)
try:
with instance.open_reader() as reader:
try:
import pandas as pd
from pandas.parser import CParserError
try:
return pd.read_csv(StringIO(reader.raw))
except (ValueError, CParserError):
return reader.raw
except ImportError:
try:
return ResultFrame(list(reader), columns=reader._columns)
except TypeError:
return reader.raw
finally:
bar.close()
def setUp(self):
super(Test, self).setUp()
# install CloudUnpickler
cloudpickle.CloudUnpickler(StringIO('abcdefg'))
