def testBytesType(self):
ex = self.gateway.jvm.py4j.examples.UTFExample()
int_list = [0, 1, 10, 127, 128, 255]
ba1 = bytearray(int_list)
# Same for Python2, bytes for Python 3
ba2 = bytearray2(int_list)
a1 = ex.getBytesValue(ba1)
a2 = ex.getBytesValue(ba2)
for i1, i2 in zip(a1, int_list):
self.assertEqual(i1, i2)
for i1, i2 in zip(a2, int_list):
self.assertEqual(i1, i2)
def testBytesType(self):
ex = self.gateway.jvm.py4j.examples.UTFExample()
int_list = [0, 1, 10, 127, 128, 255]
ba1 = bytearray(int_list)
# Same for Python2, bytes for Python 3
ba2 = bytearray2(int_list)
a1 = ex.getBytesValue(ba1)
a2 = ex.getBytesValue(ba2)
for i1, i2 in zip(a1, int_list):
self.assertEqual(i1, i2)
for i1, i2 in zip(a2, int_list):
self.assertEqual(i1, i2)
def write_chunk(jvm, chunk_number, filename, storage, file_data,
conf):
"""
Writes a single chunk in HDFS. Chunks are provided by the interface and
are blocks of data (binary)
"""
storage_url = storage.url if storage.url[-1] != '/' \
else storage.url[:-1]
parsed = req_compat.urlparse(storage_url)
if parsed.scheme == 'file':
str_uri = '{proto}://{path}'.format(
proto=parsed.scheme, path=parsed.path)
else:
str_uri = '{proto}://{host}:{port}'.format(
proto=parsed.scheme, host=parsed.hostname,
port=parsed.port)
uri = jvm.java.net.URI(str_uri)
hdfs = jvm.org.apache.hadoop.fs.FileSystem.get(uri, conf)
log.info('================== %s', uri)
tmp_path = get_tmp_path(jvm, hdfs, parsed, filename)
chunk_filename = "{tmp}/{file}.part{part:09d}".format(
tmp=tmp_path.toString(), file=filename, part=chunk_number)
chunk_path = jvm.org.apache.hadoop.fs.Path(chunk_filename)
output_stream = hdfs.create(chunk_path)
block = bytearray2(file_data)
output_stream.write(block, 0, len(block))
output_stream.close()
# Checks if all file's parts are present
full_path = tmp_path
list_iter = hdfs.listFiles(full_path, False)
counter = 0
while list_iter.hasNext():
counter += 1
list_iter.next()
return file_data, hdfs, str_uri, tmp_path, counter
def decode_bytearray(encoded):
return bytearray2([ord(c) for c in base64.decodestring(encoded)])
def decode_bytearray(encoded):
new_bytes = strtobyte(encoded)
return bytearray2([bytetoint(b) for b in standard_b64decode(new_bytes)])
def decode_bytearray(encoded):
new_bytes = strtobyte(encoded)
return bytearray2([bytetoint(b) for b in standard_b64decode(new_bytes)])
def decode_bytearray(encoded):
new_bytes = strtobyte(encoded)
return bytearray2([bytetoint(b) for b in decodeb64(new_bytes)])
def decode_bytearray(encoded):
return bytearray2(((ord(c) >> 8) for c in encoded))
def decode_bytearray(encoded):
return bytearray2(((ord(c) >> 8) for c in encoded))
def decode_bytearray(encoded):
return bytearray2([ord(c) for c in base64.decodestring(encoded)])
def decode_bytearray(encoded):
new_bytes = strtobyte(encoded)
return bytearray2([bytetoint(b) for b in decodeb64(new_bytes)])
