def sync_call(self, method, parameter, timeout=-1): method = to_binary(method) parameter = to_binary(parameter) result_ptr = _call_with_raise(ChannelSlaveClient_SyncCall, self._ptr, method, parameter, len(parameter), timeout) p = _read_std_string_ptr(result_ptr) return p.raw
def testBinaryImport(self): zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.so', '') zip_f.close() self.assertRaises(SystemError, CompressImporter, zip_f) temp_path = tempfile.mkdtemp(prefix='tmp_pyodps') lib_path = os.path.join(temp_path, 'mylib') os.makedirs(lib_path) lib_dict = dict() try: six_path = os.path.join( os.path.dirname(os.path.abspath(six.__file__)), 'six.py') shutil.copy(six_path, os.path.join(lib_path, 'five.py')) dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w') dummy_bin.close() lib_files = ['five.py', 'dummy.so'] lib_dict = dict((os.path.join(lib_path, fn), open(os.path.join(lib_path, fn), 'r')) for fn in lib_files) importer.ALLOW_BINARY = False self.assertRaises(SystemError, CompressImporter, lib_dict) importer.ALLOW_BINARY = True sys.meta_path.append(CompressImporter(lib_dict)) import five self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc')))) finally: [f.close() for f in six.itervalues(lib_dict)] shutil.rmtree(temp_path)
def test3to2NestedFunc(self): executable = self.config.get('test', 'py3_executable') if not executable: return func = _gen_nested_fun() py3_serial = to_binary(self._invoke_other_python_pickle(executable, _gen_nested_fun)) self.assertEqual(run_pickled(py3_serial, 20), func(20))
def testImport(self): raw_meta_path = sys.meta_path zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.py', 'a = 1') zip_f.close() tar_io = StringIO() tar_f = tarfile.TarFile(fileobj=tar_io, mode='w') tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'), fileobj=StringIO()) info = tarfile.TarInfo(name='testb/b.py') c = to_binary('from a import a; b = a + 1') s = StringIO(c) info.size = len(c) tar_f.addfile(info, fileobj=s) tar_f.close() zip_io.seek(0) tar_io.seek(0) zip_f = zipfile.ZipFile(zip_io) tar_f = tarfile.TarFile(fileobj=tar_io) sys.meta_path.append(CompressImporter(zip_f, tar_f)) try: from testb.b import b self.assertEqual(b, 2) finally: sys.meta_path = raw_meta_path
def testRunScript(self): import pandas as pd from io import BytesIO from odps.utils import to_binary client = self.odps.create_mars_cluster(1, 4, 8, name=str(uuid.uuid4())) try: mars_source_table_name = tn('mars_script_datasource') mars_des_table_name = tn('mars_script_datastore') self._create_table(mars_source_table_name) self.odps.delete_table(mars_des_table_name, if_exists=True) data = self._gen_data() self.odps.write_table(mars_source_table_name, data) code = BytesIO( to_binary( script.format(mars_source_table_name, self.odps.endpoint, mars_des_table_name, self.odps.endpoint))) self.odps.run_script_in_mars(code, runtime_endpoint=self.odps.endpoint) result = self.odps.get_table( mars_des_table_name).to_df().to_pandas() expected = self.odps.get_table( mars_source_table_name).to_df().to_pandas() pd.testing.assert_frame_equal(result, expected) finally: client.stop_server()
def testRealImport(self): six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py') zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.write(six_path, arcname='mylib/five.py') zip_f.close() zip_io.seek(0) zip_f = zipfile.ZipFile(zip_io) sys.meta_path.append(CompressImporter(zip_f)) import five self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc'))))
def test3to2NestedFunc(self): executable = self.config.get('test', 'py3_executable') if not executable: return func = _gen_nested_fun() py3_serial = to_binary( self._invoke_other_python_pickle(executable, _gen_nested_fun)) self.assertEqual(run_pickled(py3_serial, 20), func(20))
def test3to2BuildUnpack(self): executable = self.config.get('test', 'py3_executable') if not executable: return func = _gen_build_unpack_func() py3_serial = to_binary( self._invoke_other_python_pickle(executable, _gen_build_unpack_func)) self.assertEqual(run_pickled(py3_serial, 20), func(20))
def test3to2FormatString(self): executable = self.config.get('test', 'py3_executable') if not executable: return func = _gen_format_string_func() py3_serial = to_binary( self._invoke_other_python_pickle(executable, _gen_format_string_func)) self.assertEqual(run_pickled(py3_serial, 20), func(20))
def testFromImport(self): executable = self.config.get('test', 'py3_executable') if not executable: return func = _gen_from_import_func() py3_serial = to_binary( self._invoke_other_python_pickle(executable, _gen_from_import_func)) self.assertEqual(run_pickled(py3_serial, 20), func(20))
def testImport(self): zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.py', 'a = 1') zip_f.close() tar_io = StringIO() tar_f = tarfile.TarFile(fileobj=tar_io, mode='w') tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'), fileobj=StringIO()) info = tarfile.TarInfo(name='testb/b.py') c = to_binary('from a import a; b = a + 1') s = StringIO(c) info.size = len(c) tar_f.addfile(info, fileobj=s) tar_f.close() dict_io_init = dict() dict_io_init['/opt/test_pyodps_dev/testc/__init__.py'] = StringIO() dict_io_init['/opt/test_pyodps_dev/testc/c.py'] = StringIO( to_binary('from a import a; c = a + 2')) dict_io_file = dict() dict_io_file['/opt/test_pyodps_dev/testd/d.py'] = StringIO( to_binary('from a import a; d = a + 3')) zip_io.seek(0) tar_io.seek(0) zip_f = zipfile.ZipFile(zip_io) tar_f = tarfile.TarFile(fileobj=tar_io) importer.ALLOW_BINARY = False imp = CompressImporter(zip_f, tar_f, dict_io_init, dict_io_file) self.assertEqual(len(imp._files), 4) sys.meta_path.append(imp) from testb.b import b self.assertEqual(b, 2) from testc.c import c self.assertEqual(c, 3) self.assertRaises(ImportError, __import__, 'c', fromlist=[]) from d import d self.assertEqual(d, 4)
def testRealImport(self): raw_meta_path = sys.meta_path six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py') zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.write(six_path, arcname='mylib/five.py') zip_f.close() zip_io.seek(0) zip_f = zipfile.ZipFile(zip_io) sys.meta_path.append(CompressImporter(zip_f)) try: import five self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc')))) finally: sys.meta_path = raw_meta_path
def test3to27NestedClassObj(self): try: executable = self.config.get('test', 'py3_executable') if not executable: return except: return cls = _gen_class_builder_func()() py3_serial = to_binary(self._invoke_other_python_pickle(executable, _gen_class_builder_func)) self.assertEqual(run_pickled(py3_serial, 5, wrapper=lambda cls, a, kw: cls()().b(*a, **kw), impl='CP3'), cls().b(5))
def test26to27NestedYieldObj(self): try: executable = self.config.get('test', 'py26_executable') if not executable: return except: return func = _gen_nested_yield_obj() py26_serial = to_binary(self._invoke_other_python_pickle(executable, _gen_nested_yield_obj)) self.assertEqual(run_pickled(py26_serial, 20, wrapper=lambda fun, a, kw: sum(fun()(*a, **kw)), impl='CP26'), sum(func()(20)))
def testCEncodeAndDecode(self): try: from odps.tunnel.pb.encoder_c import Encoder from odps.tunnel.pb.decoder_c import Decoder encoder = Encoder() encoder.append_tag(0, WIRETYPE_VARINT) encoder.append_tag(1, WIRETYPE_VARINT) encoder.append_sint64(-2**40) encoder.append_tag(2, WIRETYPE_LENGTH_DELIMITED) encoder.append_string(to_binary("hello")) encoder.append_tag(3, WIRETYPE_VARINT) encoder.append_bool(True) encoder.append_tag(4, WIRETYPE_FIXED64) encoder.append_float(3.14) encoder.append_double(0.31415926) encoder.append_tag(5, WIRETYPE_VARINT) encoder.append_uint32(2**30) encoder.append_tag(6, WIRETYPE_VARINT) encoder.append_uint64(2**40) buffer_size = len(encoder) tube = io.BytesIO(encoder.tostring()) decoder = Decoder(tube) self.assertEquals((0, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals((1, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(-2**40, decoder.read_sint64()) self.assertEquals((2, WIRETYPE_LENGTH_DELIMITED), decoder.read_field_number_and_wire_type()) self.assertEquals(to_str("hello"), to_str(decoder.read_string())) self.assertEquals((3, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(True, decoder.read_bool()) self.assertEquals((4, WIRETYPE_FIXED64), decoder.read_field_number_and_wire_type()) self.assertAlmostEqual(3.14, decoder.read_float(), delta=0.001) self.assertEquals(0.31415926, decoder.read_double()) self.assertEquals((5, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(2**30, decoder.read_uint32()) self.assertEquals((6, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(2**40, decoder.read_uint64()) self.assertEquals(buffer_size, decoder.position()) except ImportError: warnings.warn('No Encoder or Decoder built by cython found')
def test26to27NestedYieldObj(self): try: executable = self.config.get('test', 'py26_executable') if not executable: return except: return func = _gen_nested_yield_obj() py26_serial = to_binary( self._invoke_other_python_pickle(executable, _gen_nested_yield_obj)) self.assertEqual( run_pickled(py26_serial, 20, wrapper=lambda fun, a, kw: sum(fun()(*a, **kw))), sum(func()(20)))
def test3to27NestedClassObj(self): try: executable = self.config.get('test', 'py3_executable') if not executable: return except: return cls = _gen_class_builder_func()() py3_serial = to_binary( self._invoke_other_python_pickle(executable, _gen_class_builder_func)) self.assertEqual( run_pickled(py3_serial, 5, wrapper=lambda cls, a, kw: cls()().b(*a, **kw)), cls().b(5))
def testCEncodeAndDecode(self): try: from odps.tunnel.pb.encoder_c import Encoder from odps.tunnel.pb.decoder_c import Decoder encoder = Encoder() encoder.append_tag(0, WIRETYPE_VARINT) encoder.append_tag(1, WIRETYPE_VARINT) encoder.append_sint64(-2 ** 40) encoder.append_tag(2, WIRETYPE_LENGTH_DELIMITED) encoder.append_string(to_binary("hello")) encoder.append_tag(3, WIRETYPE_VARINT) encoder.append_bool(True) encoder.append_tag(4, WIRETYPE_FIXED64) encoder.append_float(3.14) encoder.append_double(0.31415926) encoder.append_tag(5, WIRETYPE_VARINT) encoder.append_uint32(2 ** 30) encoder.append_tag(6, WIRETYPE_VARINT) encoder.append_uint64(2 ** 40) buffer_size = len(encoder) tube = io.BytesIO(encoder.tostring()) decoder = Decoder(tube) self.assertEquals((0, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals((1, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(-2 ** 40, decoder.read_sint64()) self.assertEquals((2, WIRETYPE_LENGTH_DELIMITED), decoder.read_field_number_and_wire_type()) self.assertEquals(to_str("hello"), to_str(decoder.read_string())) self.assertEquals((3, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(True, decoder.read_bool()) self.assertEquals((4, WIRETYPE_FIXED64), decoder.read_field_number_and_wire_type()) self.assertAlmostEqual(3.14, decoder.read_float(), delta=0.001) self.assertEquals(0.31415926, decoder.read_double()) self.assertEquals((5, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(2 ** 30, decoder.read_uint32()) self.assertEquals((6, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type()) self.assertEquals(2 ** 40, decoder.read_uint64()) self.assertEquals(buffer_size, decoder.position()) except ImportError: warnings.warn("No Encoder or Decoder built by cython found")
def testBinaryImport(self): zip_io = StringIO() zip_f = zipfile.ZipFile(zip_io, 'w') zip_f.writestr('testa/a.so', '') zip_f.close() zip_io.seek(0) self.assertRaises(SystemError, CompressImporter, zipfile.ZipFile(zip_io, 'r')) try: zip_io.seek(0) CompressImporter(zipfile.ZipFile(zip_io, 'r'), extract=True, _match_version=False) self.assertTrue(os.path.exists(CompressImporter._extract_path)) finally: shutil.rmtree(CompressImporter._extract_path) CompressImporter._extract_path = None six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py') temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-') lib_path = os.path.join(temp_path, 'mylib') os.makedirs(lib_path) lib_dict = dict() try: with open(os.path.join(lib_path, '__init__.py'), 'w'): pass shutil.copy(six_path, os.path.join(lib_path, 'fake_six.py')) dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w') dummy_bin.close() sub_lib_path = os.path.join(lib_path, 'sub_path') os.makedirs(sub_lib_path) with open(os.path.join(sub_lib_path, '__init__.py'), 'w'): pass shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py')) lib_files = [ '__init__.py', 'fake_six.py', 'dummy.so', os.path.join('sub_path', '__init__.py'), os.path.join('sub_path', 'fake_six.py') ] lib_dict = dict((os.path.join(lib_path, fn), open(os.path.join(lib_path, fn), 'r')) for fn in lib_files) importer.ALLOW_BINARY = False self.assertRaises(SystemError, CompressImporter, lib_dict) importer.ALLOW_BINARY = True importer_obj = CompressImporter(lib_dict) sys.meta_path.append(importer_obj) from mylib import fake_six self.assertEqual(list(to_binary('abc')), list(fake_six.binary_type(to_binary('abc')))) self.assertRaises(ImportError, __import__, 'sub_path', fromlist=[]) finally: [f.close() for f in six.itervalues(lib_dict)] shutil.rmtree(temp_path)