def testImport(self): raw_meta_path = sys.meta_path zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.py', 'a = 1') zip_f.close() tar_io = StringIO() tar_f = tarfile.TarFile(fileobj=tar_io, mode='w') tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'), fileobj=StringIO()) info = tarfile.TarInfo(name='testb/b.py') c = to_binary('from a import a; b = a + 1') s = StringIO(c) info.size = len(c) tar_f.addfile(info, fileobj=s) tar_f.close() zip_io.seek(0) tar_io.seek(0) zip_f = zipfile.ZipFile(zip_io) tar_f = tarfile.TarFile(fileobj=tar_io) sys.meta_path.append(CompressImporter(zip_f, tar_f)) try: from testb.b import b self.assertEqual(b, 2) finally: sys.meta_path = raw_meta_path
def testBinaryImport(self): zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.so', '') zip_f.close() self.assertRaises(SystemError, CompressImporter, zip_f) temp_path = tempfile.mkdtemp(prefix='tmp_pyodps') lib_path = os.path.join(temp_path, 'mylib') os.makedirs(lib_path) lib_dict = dict() try: six_path = os.path.join( os.path.dirname(os.path.abspath(six.__file__)), 'six.py') shutil.copy(six_path, os.path.join(lib_path, 'five.py')) dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w') dummy_bin.close() lib_files = ['five.py', 'dummy.so'] lib_dict = dict((os.path.join(lib_path, fn), open(os.path.join(lib_path, fn), 'r')) for fn in lib_files) importer.ALLOW_BINARY = False self.assertRaises(SystemError, CompressImporter, lib_dict) importer.ALLOW_BINARY = True sys.meta_path.append(CompressImporter(lib_dict)) import five self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc')))) finally: [f.close() for f in six.itervalues(lib_dict)] shutil.rmtree(temp_path)
def testRealImport(self): six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py') zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.write(six_path, arcname='mylib/five.py') zip_f.close() zip_io.seek(0) zip_f = zipfile.ZipFile(zip_io) sys.meta_path.append(CompressImporter(zip_f)) import five self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc'))))
def testImport(self): zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.py', 'a = 1') zip_f.close() tar_io = StringIO() tar_f = tarfile.TarFile(fileobj=tar_io, mode='w') tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'), fileobj=StringIO()) info = tarfile.TarInfo(name='testb/b.py') c = to_binary('from a import a; b = a + 1') s = StringIO(c) info.size = len(c) tar_f.addfile(info, fileobj=s) tar_f.close() dict_io_init = dict() dict_io_init['/opt/test_pyodps_dev/testc/__init__.py'] = StringIO() dict_io_init['/opt/test_pyodps_dev/testc/c.py'] = StringIO( to_binary('from a import a; c = a + 2')) dict_io_file = dict() dict_io_file['/opt/test_pyodps_dev/testd/d.py'] = StringIO( to_binary('from a import a; d = a + 3')) zip_io.seek(0) tar_io.seek(0) zip_f = zipfile.ZipFile(zip_io) tar_f = tarfile.TarFile(fileobj=tar_io) importer.ALLOW_BINARY = False imp = CompressImporter(zip_f, tar_f, dict_io_init, dict_io_file) self.assertEqual(len(imp._files), 4) sys.meta_path.append(imp) from testb.b import b self.assertEqual(b, 2) from testc.c import c self.assertEqual(c, 3) self.assertRaises(ImportError, __import__, 'c', fromlist=[]) from d import d self.assertEqual(d, 4)
def execute(self, line, cell=''): if self._odps is None: self._odps = enter().odps sql = line + '\n' + cell sql = sql.strip() if sql: bar = init_progress_bar() instance = self._odps.run_sql(sql) percent = 0 while not instance.is_terminated(): task_names = instance.get_task_names() last_percent = percent if len(task_names) > 0: percent = sum( self._get_task_percent(instance, name) for name in task_names) / len(task_names) else: percent = 0 percent = min(1, max(percent, last_percent)) bar.update(percent) time.sleep(1) instance.wait_for_success() bar.update(1) with instance.open_reader() as reader: try: import pandas as pd try: return pd.read_csv(StringIO(reader.raw)) except ValueError: return reader.raw except ImportError: return ResultFrame(list(reader), columns=reader._columns)
def testBinaryImport(self): zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.so', '') zip_f.close() zip_io.seek(0) self.assertRaises(SystemError, CompressImporter, zipfile.ZipFile(zip_io, 'r')) try: zip_io.seek(0) CompressImporter(zipfile.ZipFile(zip_io, 'r'), extract=True, _match_version=False) self.assertTrue(os.path.exists(CompressImporter._extract_path)) finally: shutil.rmtree(CompressImporter._extract_path) CompressImporter._extract_path = None six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py') temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-') lib_path = os.path.join(temp_path, 'mylib') os.makedirs(lib_path) lib_dict = dict() try: with open(os.path.join(lib_path, '__init__.py'), 'w'): pass shutil.copy(six_path, os.path.join(lib_path, 'fake_six.py')) dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w') dummy_bin.close() sub_lib_path = os.path.join(lib_path, 'sub_path') os.makedirs(sub_lib_path) with open(os.path.join(sub_lib_path, '__init__.py'), 'w'): pass shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py')) lib_files = [ '__init__.py', 'fake_six.py', 'dummy.so', os.path.join('sub_path', '__init__.py'), os.path.join('sub_path', 'fake_six.py') ] lib_dict = dict((os.path.join(lib_path, fn), open(os.path.join(lib_path, fn), 'r')) for fn in lib_files) importer.ALLOW_BINARY = False self.assertRaises(SystemError, CompressImporter, lib_dict) importer.ALLOW_BINARY = True importer_obj = CompressImporter(lib_dict) sys.meta_path.append(importer_obj) from mylib import fake_six self.assertEqual(list(to_binary('abc')), list(fake_six.binary_type(to_binary('abc')))) self.assertRaises(ImportError, __import__, 'sub_path', fromlist=[]) finally: [f.close() for f in six.itervalues(lib_dict)] shutil.rmtree(temp_path)
def execute(self, line, cell=''): self._set_odps() content = line + '\n' + cell content = content.strip() sql = None hints = dict() splits = content.split(';') for s in splits: stripped = s.strip() if stripped.lower().startswith('set '): hint = stripped.split(' ', 1)[1] k, v = hint.split('=', 1) k, v = k.strip(), v.strip() hints[k] = v elif len(stripped) == 0: continue else: if sql is None: sql = s else: sql = '%s;%s' % (sql, s) # replace user defined parameters sql = replace_sql_parameters(sql, self.shell.user_ns) if sql: bar = init_progress_bar() instance = self._odps.run_sql(sql, hints=hints) if options.verbose: stdout = options.verbose_log or self._to_stdout stdout('Instance ID: ' + instance.id) stdout(' Log view: ' + instance.get_logview_address()) percent = 0 while not instance.is_terminated(): task_names = instance.get_task_names() last_percent = percent if len(task_names) > 0: percent = sum(self._get_task_percent(instance, name) for name in task_names) / len(task_names) else: percent = 0 percent = min(1, max(percent, last_percent)) bar.update(percent) time.sleep(1) instance.wait_for_success() bar.update(1) try: with instance.open_reader() as reader: try: import pandas as pd from pandas.parser import CParserError try: return pd.read_csv(StringIO(reader.raw)) except (ValueError, CParserError): return reader.raw except ImportError: try: return ResultFrame(list(reader), columns=reader._columns) except TypeError: return reader.raw finally: bar.close()
def setUp(self): super(Test, self).setUp() # install CloudUnpickler cloudpickle.CloudUnpickler(StringIO('abcdefg'))