def to_string(cls, filename, offset, length):
stream = StringIO()
if hadoop_version_info().has_variable_isplit_encoding():
serialize_text(filename, stream)
else:
serialize_old_style_filename(filename, stream)
serialize_long(offset, stream)
serialize_long(length, stream)
return stream.getvalue()
def to_string(cls, filename, offset, length):
stream = StringIO()
if hadoop_version_info().has_variable_isplit_encoding():
serialize_text(filename, stream)
else:
serialize_old_style_filename(filename, stream)
serialize_long(offset, stream)
serialize_long(length, stream)
return stream.getvalue()
def test_string(self):
N = 10
stream = self.stream
test_file = __file__.replace("pyc", "py")
with open(test_file) as f:
s = unicode(f.read(), 'utf-8')
t = s
for _ in range(N):
srl.serialize_text(t, stream)
stream.seek(0)
t = s
for _ in range(N):
s1 = srl.deserialize_text(stream)
self.assertEqual(t, s1)
def test_string(self):
N = 10
stream = self.stream
test_file = __file__.replace("pyc", "py")
with open(test_file) as f:
s = unicode(f.read(), "utf-8")
t = s
for _ in range(N):
srl.serialize_text(t, stream)
stream.seek(0)
t = s
for _ in range(N):
s1 = srl.deserialize_text(stream)
self.assertEqual(t, s1)
def test_mixture(self):
stream = self.stream
vals = [1, 0.33, 0.3290, 1902, 'sshjdhsj', 0.3, -33, 'ueiwriuqrei']
for v in vals:
if isinstance(v, int):
srl.serialize_int(v, stream)
elif isinstance(v, float):
srl.serialize_float(v, stream)
elif isinstance(v, str):
srl.serialize_text(v, stream)
stream.seek(0)
for v in vals:
if isinstance(v, int):
x = srl.deserialize_int(stream)
self.assertEqual(v, x)
elif isinstance(v, float):
x = srl.deserialize_float(stream)
self.assertTrue(abs(v - x) / abs(v + x) < 1e-6)
elif isinstance(v, str):
x = srl.deserialize_text(stream)
self.assertEqual(v, x)
def test_mixture(self):
stream = self.stream
vals = [1, 0.33, 0.3290, 1902, "sshjdhsj", 0.3, -33, "ueiwriuqrei"]
for v in vals:
if isinstance(v, int):
srl.serialize_int(v, stream)
elif isinstance(v, float):
srl.serialize_float(v, stream)
elif isinstance(v, str):
srl.serialize_text(v, stream)
stream.seek(0)
for v in vals:
if isinstance(v, int):
x = srl.deserialize_int(stream)
self.assertEqual(v, x)
elif isinstance(v, float):
x = srl.deserialize_float(stream)
self.assertTrue(abs(v - x) / abs(v + x) < 1e-6)
elif isinstance(v, str):
x = srl.deserialize_text(stream)
self.assertEqual(v, x)
def test_simulate_java_output_1(self):
try:
byte_stream = _get_java_output_stream(self.wd)
out_stream = StringIO()
# write integers
srl.serialize_vint(42, out_stream)
srl.serialize_vint(4242, out_stream)
srl.serialize_vint(424242, out_stream)
srl.serialize_vint(42424242, out_stream)
srl.serialize_vint(-42, out_stream)
# write longs
srl.serialize_vint(42, out_stream)
srl.serialize_vint(424242, out_stream)
srl.serialize_vint(4242424242, out_stream)
# strings
wu.writeString(out_stream, u"hello world")
# second has accented characters
wu.writeString(out_stream, u"oggi è giovedì")
#
srl.serialize_text(u"à Text object", out_stream)
self.assertEqual(byte_stream.getvalue(), out_stream.getvalue())
finally:
pass
def test_simulate_java_output_1(self):
try:
byte_stream = _get_java_output_stream(self.wd)
out_stream = StringIO()
# write integers
srl.serialize_vint(42, out_stream)
srl.serialize_vint(4242, out_stream)
srl.serialize_vint(424242, out_stream)
srl.serialize_vint(42424242, out_stream)
srl.serialize_vint(-42, out_stream)
# write longs
srl.serialize_vint(42, out_stream)
srl.serialize_vint(424242, out_stream)
srl.serialize_vint(4242424242, out_stream)
# strings
wu.writeString(out_stream, u"hello world")
# second has accented characters
wu.writeString(out_stream, u"oggi è giovedì")
#
srl.serialize_text(u"à Text object", out_stream)
self.assertEqual(byte_stream.getvalue(), out_stream.getvalue())
finally:
pass
