def sync_call(self, method, parameter, timeout=-1):
method = to_binary(method)
parameter = to_binary(parameter)
result_ptr = _call_with_raise(ChannelSlaveClient_SyncCall, self._ptr, method,
parameter, len(parameter), timeout)
p = _read_std_string_ptr(result_ptr)
return p.raw
def testBinaryImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.so', '')
zip_f.close()
self.assertRaises(SystemError, CompressImporter, zip_f)
temp_path = tempfile.mkdtemp(prefix='tmp_pyodps')
lib_path = os.path.join(temp_path, 'mylib')
os.makedirs(lib_path)
lib_dict = dict()
try:
six_path = os.path.join(
os.path.dirname(os.path.abspath(six.__file__)), 'six.py')
shutil.copy(six_path, os.path.join(lib_path, 'five.py'))
dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
dummy_bin.close()
lib_files = ['five.py', 'dummy.so']
lib_dict = dict((os.path.join(lib_path, fn),
open(os.path.join(lib_path, fn), 'r'))
for fn in lib_files)
importer.ALLOW_BINARY = False
self.assertRaises(SystemError, CompressImporter, lib_dict)
importer.ALLOW_BINARY = True
sys.meta_path.append(CompressImporter(lib_dict))
import five
self.assertEqual(list(to_binary('abc')),
list(five.binary_type(to_binary('abc'))))
finally:
[f.close() for f in six.itervalues(lib_dict)]
shutil.rmtree(temp_path)
def test3to2NestedFunc(self):
executable = self.config.get('test', 'py3_executable')
if not executable:
return
func = _gen_nested_fun()
py3_serial = to_binary(self._invoke_other_python_pickle(executable, _gen_nested_fun))
self.assertEqual(run_pickled(py3_serial, 20), func(20))
def testImport(self):
raw_meta_path = sys.meta_path
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'), fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
sys.meta_path.append(CompressImporter(zip_f, tar_f))
try:
from testb.b import b
self.assertEqual(b, 2)
finally:
sys.meta_path = raw_meta_path
def testRunScript(self):
import pandas as pd
from io import BytesIO
from odps.utils import to_binary
client = self.odps.create_mars_cluster(1, 4, 8, name=str(uuid.uuid4()))
try:
mars_source_table_name = tn('mars_script_datasource')
mars_des_table_name = tn('mars_script_datastore')
self._create_table(mars_source_table_name)
self.odps.delete_table(mars_des_table_name, if_exists=True)
data = self._gen_data()
self.odps.write_table(mars_source_table_name, data)
code = BytesIO(
to_binary(
script.format(mars_source_table_name, self.odps.endpoint,
mars_des_table_name, self.odps.endpoint)))
self.odps.run_script_in_mars(code,
runtime_endpoint=self.odps.endpoint)
result = self.odps.get_table(
mars_des_table_name).to_df().to_pandas()
expected = self.odps.get_table(
mars_source_table_name).to_df().to_pandas()
pd.testing.assert_frame_equal(result, expected)
finally:
client.stop_server()
def testRealImport(self):
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
'six.py')
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.write(six_path, arcname='mylib/five.py')
zip_f.close()
zip_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
sys.meta_path.append(CompressImporter(zip_f))
import five
self.assertEqual(list(to_binary('abc')),
list(five.binary_type(to_binary('abc'))))
def testImport(self):
raw_meta_path = sys.meta_path
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
sys.meta_path.append(CompressImporter(zip_f, tar_f))
try:
from testb.b import b
self.assertEqual(b, 2)
finally:
sys.meta_path = raw_meta_path
def test3to2NestedFunc(self):
executable = self.config.get('test', 'py3_executable')
if not executable:
return
func = _gen_nested_fun()
py3_serial = to_binary(
self._invoke_other_python_pickle(executable, _gen_nested_fun))
self.assertEqual(run_pickled(py3_serial, 20), func(20))
def test3to2BuildUnpack(self):
executable = self.config.get('test', 'py3_executable')
if not executable:
return
func = _gen_build_unpack_func()
py3_serial = to_binary(
self._invoke_other_python_pickle(executable,
_gen_build_unpack_func))
self.assertEqual(run_pickled(py3_serial, 20), func(20))
def test3to2FormatString(self):
executable = self.config.get('test', 'py3_executable')
if not executable:
return
func = _gen_format_string_func()
py3_serial = to_binary(
self._invoke_other_python_pickle(executable,
_gen_format_string_func))
self.assertEqual(run_pickled(py3_serial, 20), func(20))
def testFromImport(self):
executable = self.config.get('test', 'py3_executable')
if not executable:
return
func = _gen_from_import_func()
py3_serial = to_binary(
self._invoke_other_python_pickle(executable,
_gen_from_import_func))
self.assertEqual(run_pickled(py3_serial, 20), func(20))
def testImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.py', 'a = 1')
zip_f.close()
tar_io = StringIO()
tar_f = tarfile.TarFile(fileobj=tar_io, mode='w')
tar_f.addfile(tarfile.TarInfo(name='testb/__init__.py'),
fileobj=StringIO())
info = tarfile.TarInfo(name='testb/b.py')
c = to_binary('from a import a; b = a + 1')
s = StringIO(c)
info.size = len(c)
tar_f.addfile(info, fileobj=s)
tar_f.close()
dict_io_init = dict()
dict_io_init['/opt/test_pyodps_dev/testc/__init__.py'] = StringIO()
dict_io_init['/opt/test_pyodps_dev/testc/c.py'] = StringIO(
to_binary('from a import a; c = a + 2'))
dict_io_file = dict()
dict_io_file['/opt/test_pyodps_dev/testd/d.py'] = StringIO(
to_binary('from a import a; d = a + 3'))
zip_io.seek(0)
tar_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
tar_f = tarfile.TarFile(fileobj=tar_io)
importer.ALLOW_BINARY = False
imp = CompressImporter(zip_f, tar_f, dict_io_init, dict_io_file)
self.assertEqual(len(imp._files), 4)
sys.meta_path.append(imp)
from testb.b import b
self.assertEqual(b, 2)
from testc.c import c
self.assertEqual(c, 3)
self.assertRaises(ImportError, __import__, 'c', fromlist=[])
from d import d
self.assertEqual(d, 4)
def testRealImport(self):
raw_meta_path = sys.meta_path
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)), 'six.py')
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.write(six_path, arcname='mylib/five.py')
zip_f.close()
zip_io.seek(0)
zip_f = zipfile.ZipFile(zip_io)
sys.meta_path.append(CompressImporter(zip_f))
try:
import five
self.assertEqual(list(to_binary('abc')), list(five.binary_type(to_binary('abc'))))
finally:
sys.meta_path = raw_meta_path
def test3to27NestedClassObj(self):
try:
executable = self.config.get('test', 'py3_executable')
if not executable:
return
except:
return
cls = _gen_class_builder_func()()
py3_serial = to_binary(self._invoke_other_python_pickle(executable, _gen_class_builder_func))
self.assertEqual(run_pickled(py3_serial, 5, wrapper=lambda cls, a, kw: cls()().b(*a, **kw), impl='CP3'),
cls().b(5))
def test26to27NestedYieldObj(self):
try:
executable = self.config.get('test', 'py26_executable')
if not executable:
return
except:
return
func = _gen_nested_yield_obj()
py26_serial = to_binary(self._invoke_other_python_pickle(executable, _gen_nested_yield_obj))
self.assertEqual(run_pickled(py26_serial, 20, wrapper=lambda fun, a, kw: sum(fun()(*a, **kw)), impl='CP26'),
sum(func()(20)))
def testCEncodeAndDecode(self):
try:
from odps.tunnel.pb.encoder_c import Encoder
from odps.tunnel.pb.decoder_c import Decoder
encoder = Encoder()
encoder.append_tag(0, WIRETYPE_VARINT)
encoder.append_tag(1, WIRETYPE_VARINT)
encoder.append_sint64(-2**40)
encoder.append_tag(2, WIRETYPE_LENGTH_DELIMITED)
encoder.append_string(to_binary("hello"))
encoder.append_tag(3, WIRETYPE_VARINT)
encoder.append_bool(True)
encoder.append_tag(4, WIRETYPE_FIXED64)
encoder.append_float(3.14)
encoder.append_double(0.31415926)
encoder.append_tag(5, WIRETYPE_VARINT)
encoder.append_uint32(2**30)
encoder.append_tag(6, WIRETYPE_VARINT)
encoder.append_uint64(2**40)
buffer_size = len(encoder)
tube = io.BytesIO(encoder.tostring())
decoder = Decoder(tube)
self.assertEquals((0, WIRETYPE_VARINT),
decoder.read_field_number_and_wire_type())
self.assertEquals((1, WIRETYPE_VARINT),
decoder.read_field_number_and_wire_type())
self.assertEquals(-2**40, decoder.read_sint64())
self.assertEquals((2, WIRETYPE_LENGTH_DELIMITED),
decoder.read_field_number_and_wire_type())
self.assertEquals(to_str("hello"), to_str(decoder.read_string()))
self.assertEquals((3, WIRETYPE_VARINT),
decoder.read_field_number_and_wire_type())
self.assertEquals(True, decoder.read_bool())
self.assertEquals((4, WIRETYPE_FIXED64),
decoder.read_field_number_and_wire_type())
self.assertAlmostEqual(3.14, decoder.read_float(), delta=0.001)
self.assertEquals(0.31415926, decoder.read_double())
self.assertEquals((5, WIRETYPE_VARINT),
decoder.read_field_number_and_wire_type())
self.assertEquals(2**30, decoder.read_uint32())
self.assertEquals((6, WIRETYPE_VARINT),
decoder.read_field_number_and_wire_type())
self.assertEquals(2**40, decoder.read_uint64())
self.assertEquals(buffer_size, decoder.position())
except ImportError:
warnings.warn('No Encoder or Decoder built by cython found')
def test26to27NestedYieldObj(self):
try:
executable = self.config.get('test', 'py26_executable')
if not executable:
return
except:
return
func = _gen_nested_yield_obj()
py26_serial = to_binary(
self._invoke_other_python_pickle(executable,
_gen_nested_yield_obj))
self.assertEqual(
run_pickled(py26_serial,
20,
wrapper=lambda fun, a, kw: sum(fun()(*a, **kw))),
sum(func()(20)))
def test3to27NestedClassObj(self):
try:
executable = self.config.get('test', 'py3_executable')
if not executable:
return
except:
return
cls = _gen_class_builder_func()()
py3_serial = to_binary(
self._invoke_other_python_pickle(executable,
_gen_class_builder_func))
self.assertEqual(
run_pickled(py3_serial,
5,
wrapper=lambda cls, a, kw: cls()().b(*a, **kw)),
cls().b(5))
def testCEncodeAndDecode(self):
try:
from odps.tunnel.pb.encoder_c import Encoder
from odps.tunnel.pb.decoder_c import Decoder
encoder = Encoder()
encoder.append_tag(0, WIRETYPE_VARINT)
encoder.append_tag(1, WIRETYPE_VARINT)
encoder.append_sint64(-2 ** 40)
encoder.append_tag(2, WIRETYPE_LENGTH_DELIMITED)
encoder.append_string(to_binary("hello"))
encoder.append_tag(3, WIRETYPE_VARINT)
encoder.append_bool(True)
encoder.append_tag(4, WIRETYPE_FIXED64)
encoder.append_float(3.14)
encoder.append_double(0.31415926)
encoder.append_tag(5, WIRETYPE_VARINT)
encoder.append_uint32(2 ** 30)
encoder.append_tag(6, WIRETYPE_VARINT)
encoder.append_uint64(2 ** 40)
buffer_size = len(encoder)
tube = io.BytesIO(encoder.tostring())
decoder = Decoder(tube)
self.assertEquals((0, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type())
self.assertEquals((1, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type())
self.assertEquals(-2 ** 40, decoder.read_sint64())
self.assertEquals((2, WIRETYPE_LENGTH_DELIMITED), decoder.read_field_number_and_wire_type())
self.assertEquals(to_str("hello"), to_str(decoder.read_string()))
self.assertEquals((3, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type())
self.assertEquals(True, decoder.read_bool())
self.assertEquals((4, WIRETYPE_FIXED64), decoder.read_field_number_and_wire_type())
self.assertAlmostEqual(3.14, decoder.read_float(), delta=0.001)
self.assertEquals(0.31415926, decoder.read_double())
self.assertEquals((5, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type())
self.assertEquals(2 ** 30, decoder.read_uint32())
self.assertEquals((6, WIRETYPE_VARINT), decoder.read_field_number_and_wire_type())
self.assertEquals(2 ** 40, decoder.read_uint64())
self.assertEquals(buffer_size, decoder.position())
except ImportError:
warnings.warn("No Encoder or Decoder built by cython found")
def testBinaryImport(self):
zip_io = StringIO()
zip_f = zipfile.ZipFile(zip_io, 'w')
zip_f.writestr('testa/a.so', '')
zip_f.close()
zip_io.seek(0)
self.assertRaises(SystemError, CompressImporter,
zipfile.ZipFile(zip_io, 'r'))
try:
zip_io.seek(0)
CompressImporter(zipfile.ZipFile(zip_io, 'r'),
extract=True,
_match_version=False)
self.assertTrue(os.path.exists(CompressImporter._extract_path))
finally:
shutil.rmtree(CompressImporter._extract_path)
CompressImporter._extract_path = None
six_path = os.path.join(os.path.dirname(os.path.abspath(six.__file__)),
'six.py')
temp_path = tempfile.mkdtemp(prefix='tmp-pyodps-')
lib_path = os.path.join(temp_path, 'mylib')
os.makedirs(lib_path)
lib_dict = dict()
try:
with open(os.path.join(lib_path, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(lib_path, 'fake_six.py'))
dummy_bin = open(os.path.join(lib_path, 'dummy.so'), 'w')
dummy_bin.close()
sub_lib_path = os.path.join(lib_path, 'sub_path')
os.makedirs(sub_lib_path)
with open(os.path.join(sub_lib_path, '__init__.py'), 'w'):
pass
shutil.copy(six_path, os.path.join(sub_lib_path, 'fake_six.py'))
lib_files = [
'__init__.py', 'fake_six.py', 'dummy.so',
os.path.join('sub_path', '__init__.py'),
os.path.join('sub_path', 'fake_six.py')
]
lib_dict = dict((os.path.join(lib_path, fn),
open(os.path.join(lib_path, fn), 'r'))
for fn in lib_files)
importer.ALLOW_BINARY = False
self.assertRaises(SystemError, CompressImporter, lib_dict)
importer.ALLOW_BINARY = True
importer_obj = CompressImporter(lib_dict)
sys.meta_path.append(importer_obj)
from mylib import fake_six
self.assertEqual(list(to_binary('abc')),
list(fake_six.binary_type(to_binary('abc'))))
self.assertRaises(ImportError, __import__, 'sub_path', fromlist=[])
finally:
[f.close() for f in six.itervalues(lib_dict)]
shutil.rmtree(temp_path)