def test_view_snappy_compressed_avro():
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
c = make_logged_in_client()
cluster.fs.setuser(cluster.superuser)
if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
test_schema = schema.parse("""
{
"name": "test",
"type": "record",
"fields": [
{ "name": "name", "type": "string" },
{ "name": "integer", "type": "int" }
]
}
""")
# Cannot use StringIO with datafile writer!
f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
writers_schema=test_schema,
codec='snappy')
dummy_datum = {
'name': 'Test',
'integer': 10,
}
data_file_writer.append(dummy_datum)
data_file_writer.close()
f.close()
# Check to see if snappy is the codec
f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "r")
assert_true('snappy' in f.read())
f.close()
# Snappy compressed succeed
response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
assert_equal('avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
finally:
for done in finish:
done()
try:
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
except:
pass # Don't let cleanup errors mask earlier failures
def test_view_snappy_compressed_avro():
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
c = make_logged_in_client()
cluster.fs.setuser(cluster.superuser)
if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
test_schema = schema.parse("""
{
"name": "test",
"type": "record",
"fields": [
{ "name": "name", "type": "string" },
{ "name": "integer", "type": "int" }
]
}
""")
# Cannot use StringIO with datafile writer!
f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
writers_schema=test_schema,
codec='snappy')
dummy_datum = {
'name': 'Test',
'integer': 10,
}
data_file_writer.append(dummy_datum)
data_file_writer.close()
f.close()
# Check to see if snappy is the codec
f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "r")
assert_true('snappy' in f.read())
f.close()
# Snappy compressed succeed
response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
assert_equal('avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
finally:
for done in finish:
done()
try:
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
except:
pass # Don't let cleanup errors mask earlier failures
def test_view_snappy_compressed():
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
c = make_logged_in_client()
cluster.fs.setuser(cluster.superuser)
if cluster.fs.isdir('/tmp/test-snappy-filebrowser'):
cluster.fs.rmtree('/tmp/test-snappy-filebrowser')
cluster.fs.mkdir('/tmp/test-snappy-avro-filebrowser/')
f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.snappy', "w")
f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
f.close()
f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.stillsnappy', "w")
f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
f.close()
f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.notsnappy', "w")
f.write('foobar')
f.close()
# Snappy compressed fail
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.notsnappy?compression=snappy')
assert_true('Failed to decompress' in response.context['message'], response)
# Snappy compressed succeed
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.snappy')
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
# Snappy compressed succeed
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy')
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
# Largest snappy compressed file
finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy?compression=snappy')
assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
finally:
for done in finish:
done()
try:
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
except:
pass # Don't let cleanup errors mask earlier failures
def test_view_snappy_compressed():
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
c = make_logged_in_client()
cluster.fs.setuser(cluster.superuser)
if cluster.fs.isdir('/tmp/test-snappy-filebrowser'):
cluster.fs.rmtree('/tmp/test-snappy-filebrowser')
cluster.fs.mkdir('/tmp/test-snappy-avro-filebrowser/')
f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.snappy', "w")
f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
f.close()
f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.stillsnappy', "w")
f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
f.close()
f = cluster.fs.open('/tmp/test-snappy-filebrowser/test-view.notsnappy', "w")
f.write('foobar')
f.close()
# Snappy compressed fail
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.notsnappy?compression=snappy')
assert_true('Failed to decompress' in response.context['message'], response)
# Snappy compressed succeed
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.snappy')
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
# Snappy compressed succeed
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy')
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
# Largest snappy compressed file
finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
response = c.get('/filebrowser/view/tmp/test-snappy-filebrowser/test-view.stillsnappy?compression=snappy')
assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
finally:
for done in finish:
done()
try:
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
except:
pass # Don't let cleanup errors mask earlier failures
def test_view_parquet_snappy(self):
if not snappy_installed():
raise SkipTest
prefix = self.cluster.fs_prefix + '/test_view_parquet_snappy'
self.cluster.fs.mkdir(prefix)
with open('apps/filebrowser/src/filebrowser/test_data/parquet-snappy.parquet') as f:
hdfs = self.cluster.fs.open(prefix + '/test-parquet-snappy.parquet', "w")
hdfs.write(f.read())
# autodetect
response = self.c.get('/filebrowser/view=%s/test-parquet-snappy.parquet' % prefix)
assert_true('SR3_ndw_otlt_cmf_xref_INA' in response.context['view']['contents'], response.context['view']['contents'])
def test_view_snappy_compressed_avro(self):
if not snappy_installed():
raise SkipTest
import snappy
finish = []
try:
prefix = self.cluster.fs_prefix + '/test-snappy-avro-filebrowser'
self.cluster.fs.mkdir(prefix)
test_schema = schema.parse("""
{
"name": "test",
"type": "record",
"fields": [
{ "name": "name", "type": "string" },
{ "name": "integer", "type": "int" }
]
}
""")
# Cannot use StringIO with datafile writer!
f = self.cluster.fs.open(prefix +'/test-view.compressed.avro', "w")
data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
writers_schema=test_schema,
codec='snappy')
dummy_datum = {
'name': 'Test',
'integer': 10,
}
data_file_writer.append(dummy_datum)
data_file_writer.close()
f.close()
# Check to see if snappy is the codec
f = self.cluster.fs.open(prefix + '/test-view.compressed.avro', "r")
assert_true('snappy' in f.read())
f.close()
# Snappy compressed succeed
response = self.c.get('/filebrowser/view=%s/test-view.compressed.avro' % prefix)
assert_equal('avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
finally:
for done in finish:
done()
def test_view_snappy_compressed_avro(self):
if not snappy_installed():
raise SkipTest
import snappy
finish = []
try:
prefix = self.cluster.fs_prefix + '/test-snappy-avro-filebrowser'
self.cluster.fs.mkdir(prefix)
test_schema = schema.parse("""
{
"name": "test",
"type": "record",
"fields": [
{ "name": "name", "type": "string" },
{ "name": "integer", "type": "int" }
]
}
""")
# Cannot use StringIO with datafile writer!
f = self.cluster.fs.open(prefix +'/test-view.compressed.avro', "w")
data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
writers_schema=test_schema,
codec='snappy')
dummy_datum = {
'name': 'Test',
'integer': 10,
}
data_file_writer.append(dummy_datum)
data_file_writer.close()
f.close()
# Check to see if snappy is the codec
f = self.cluster.fs.open(prefix + '/test-view.compressed.avro', "r")
assert_true('snappy' in f.read())
f.close()
# Snappy compressed succeed
response = self.c.get('/filebrowser/view=%s/test-view.compressed.avro' % prefix)
assert_equal('avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
finally:
for done in finish:
done()
def test_view_snappy_compressed(self):
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
prefix = self.cluster.fs_prefix + '/test_view_snappy_compressed'
self.self.cluster.fs.mkdir(prefix)
f = cluster.fs.open(prefix + '/test-view.snappy', "w")
f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
f.close()
f = cluster.fs.open(prefix + '/test-view.stillsnappy', "w")
f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
f.close()
f = cluster.fs.open(prefix + '/test-view.notsnappy', "w")
f.write('foobar')
f.close()
# Snappy compressed fail
response = c.get('/filebrowser/view=%s/test-view.notsnappy?compression=snappy' % prefix)
assert_true('Failed to decompress' in response.context['message'], response)
# Snappy compressed succeed
response = c.get('/filebrowser/view=%s/test-view.snappy' % prefix)
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
# Snappy compressed succeed
response = c.get('/filebrowser/view=%s/test-view.stillsnappy' % prefix)
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
# Largest snappy compressed file
finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
response = c.get('/filebrowser/view=%s/test-view.stillsnappy?compression=snappy' % prefix)
assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
finally:
for done in finish:
done()
def test_view_snappy_compressed(self):
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
prefix = self.cluster.fs_prefix + '/test_view_snappy_compressed'
self.self.cluster.fs.mkdir(prefix)
f = cluster.fs.open(prefix + '/test-view.snappy', "w")
f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
f.close()
f = cluster.fs.open(prefix + '/test-view.stillsnappy', "w")
f.write(snappy.compress('The broadcasters of your area in voluntary cooperation with the FCC and other authorities.'))
f.close()
f = cluster.fs.open(prefix + '/test-view.notsnappy', "w")
f.write('foobar')
f.close()
# Snappy compressed fail
response = c.get('/filebrowser/view=%s/test-view.notsnappy?compression=snappy' % prefix)
assert_true('Failed to decompress' in response.context['message'], response)
# Snappy compressed succeed
response = c.get('/filebrowser/view=%s/test-view.snappy' % prefix)
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
# Snappy compressed succeed
response = c.get('/filebrowser/view=%s/test-view.stillsnappy' % prefix)
assert_equal('snappy', response.context['view']['compression'])
assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
# Largest snappy compressed file
finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
response = c.get('/filebrowser/view=%s/test-view.stillsnappy?compression=snappy' % prefix)
assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
finally:
for done in finish:
done()
def test_view_snappy_compressed_avro():
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
c = make_logged_in_client()
cluster.fs.setuser(cluster.superuser)
if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
test_schema = schema.parse("""
{
"name": "test",
"type": "record",
"fields": [
{ "name": "name", "type": "string" },
{ "name": "integer", "type": "int" }
]
}
""")
# Cannot use StringIO with datafile writer!
f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro',
"w")
data_file_writer = datafile.DataFileWriter(f,
io.DatumWriter(),
writers_schema=test_schema,
codec='deflate')
dummy_datum = {
'name': 'Test',
'integer': 10,
}
data_file_writer.append(dummy_datum)
data_file_writer.close()
fh = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro',
'r')
f = cluster.fs.open(
'/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
f.write(snappy.compress(fh.read()))
f.close()
fh.close()
# Snappy compressed fail
response = c.get(
'/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro'
)
assert_true('Failed to decompress' in response.context['message'],
response)
# Snappy compressed succeed
response = c.get(
'/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro'
)
assert_equal('snappy_avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum,
response)
response = c.get(
'/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=snappy_avro'
)
assert_equal('snappy_avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum,
response)
# Avro should also decompress snappy
response = c.get(
'/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=avro'
)
assert_equal('snappy_avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum,
response)
# Largest snappy compressed file
finish.append(MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1))
response = c.get(
'/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro'
)
assert_true(
'File size is greater than allowed max snappy decompression size of 1'
in response.context['message'], response)
finally:
for done in finish:
done()
try:
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
except:
pass # Don't let cleanup errors mask earlier failures
def test_view_snappy_compressed_avro():
if not snappy_installed():
raise SkipTest
import snappy
cluster = pseudo_hdfs4.shared_cluster()
finish = []
try:
c = make_logged_in_client()
cluster.fs.setuser(cluster.superuser)
if cluster.fs.isdir("/test-snappy-avro-filebrowser"):
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
cluster.fs.mkdir('/test-snappy-avro-filebrowser/')
test_schema = schema.parse("""
{
"name": "test",
"type": "record",
"fields": [
{ "name": "name", "type": "string" },
{ "name": "integer", "type": "int" }
]
}
""")
# Cannot use StringIO with datafile writer!
f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', "w")
data_file_writer = datafile.DataFileWriter(f, io.DatumWriter(),
writers_schema=test_schema,
codec='deflate')
dummy_datum = {
'name': 'Test',
'integer': 10,
}
data_file_writer.append(dummy_datum)
data_file_writer.close()
fh = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.avro', 'r')
f = cluster.fs.open('/test-snappy-avro-filebrowser/test-view.compressed.avro', "w")
f.write(snappy.compress(fh.read()))
f.close()
fh.close()
# Snappy compressed fail
response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
assert_true('Failed to decompress' in response.context['message'], response)
# Snappy compressed succeed
response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro')
assert_equal('snappy_avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=snappy_avro')
assert_equal('snappy_avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
# Avro should also decompress snappy
response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.compressed.avro?compression=avro')
assert_equal('snappy_avro', response.context['view']['compression'])
assert_equal(eval(response.context['view']['contents']), dummy_datum, response)
# Largest snappy compressed file
finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
response = c.get('/filebrowser/view/test-snappy-avro-filebrowser/test-view.avro?compression=snappy_avro')
assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
finally:
for done in finish:
done()
try:
cluster.fs.rmtree('/test-snappy-avro-filebrowser/')
except:
pass # Don't let cleanup errors mask earlier failures