def wu_helper(self, data):
out_stream = StringIO()
for x in data:
if isinstance(x, int):
srl.serialize_vint(x, out_stream)
elif isinstance(x, basestring):
wu.writeString(out_stream, x)
return out_stream.getvalue()
def __init__(self, data):
stream = StringIO(data)
if hadoop_version_info().has_variable_isplit_encoding():
self.filename = deserialize_text(stream)
else:
self.filename = deserialize_old_style_filename(stream)
self.offset = deserialize_long(stream)
self.length = deserialize_long(stream)
def to_string(cls, filename, offset, length):
stream = StringIO()
if hadoop_version_info().has_variable_isplit_encoding():
serialize_text(filename, stream)
else:
serialize_old_style_filename(filename, stream)
serialize_long(offset, stream)
serialize_long(length, stream)
return stream.getvalue()
def _get_java_output_stream(wd):
this_directory = os.path.abspath(os.path.dirname(__file__))
src = os.path.join(this_directory, "%s.java" % _HADOOP_SERIALIZE_CLASS)
shutil.copy(src, wd)
classpath = '.:%s:%s' % (pydoop.hadoop_classpath(), wd)
filename_root = os.path.join(wd, _HADOOP_SERIALIZE_CLASS)
_compile_java_part(filename_root + ".class", classpath)
output = subprocess.check_output(
[JAVA, '-cp', classpath, _HADOOP_SERIALIZE_CLASS],
cwd=wd,
stderr=open('/dev/null', 'w'))
stream = StringIO(output)
return stream
def test_simulate_java_output_1(self):
try:
byte_stream = _get_java_output_stream(self.wd)
out_stream = StringIO()
# write integers
srl.serialize_vint(42, out_stream)
srl.serialize_vint(4242, out_stream)
srl.serialize_vint(424242, out_stream)
srl.serialize_vint(42424242, out_stream)
srl.serialize_vint(-42, out_stream)
# write longs
srl.serialize_vint(42, out_stream)
srl.serialize_vint(424242, out_stream)
srl.serialize_vint(4242424242, out_stream)
# strings
wu.writeString(out_stream, u"hello world")
# second has accented characters
wu.writeString(out_stream, u"oggi è giovedì")
#
srl.serialize_text(u"à Text object", out_stream)
self.assertEqual(byte_stream.getvalue(), out_stream.getvalue())
finally:
pass
def get_java_output_stream(jclass, classpath, args, wd):
output = subprocess.check_output([JAVA, '-cp', classpath, jclass] + args,
cwd=wd,
stderr=open('/dev/null', 'w'))
return StringIO(output)
def serialize(self, record):
f = StringIO()
encoder = BinaryEncoder(f)
self.datum_writer.write(record, encoder)
return f.getvalue()
def deserialize(self, rec_bytes):
return self.reader.read(BinaryDecoder(StringIO(rec_bytes)))
def setUp(self):
self.stream = StringIO()
self.wd = tempfile.mkdtemp(prefix="pydoop_")