class TestPersist(unittest.TestCase):
def setUp(self):
test_dir = self.get_test_dir()
self.schema = TelemetrySchema(self.get_schema_spec())
self.storage = StorageLayout(self.schema, test_dir, 10000)
assert not os.path.exists(test_dir)
os.makedirs(test_dir)
def tearDown(self):
shutil.rmtree(self.get_test_dir())
def get_test_dir(self):
return "/tmp/test_telemetry_persist"
def get_schema_spec(self):
return {
"version": 1,
"dimensions": [
{
"field_name": "reason",
"allowed_values": ["r1","r2"]
},
{
"field_name": "appName",
"allowed_values": ["a1"]
},
{
"field_name": "appUpdateChannel",
"allowed_values": ["c1", "c2", "c3"]
},
{
"field_name": "appVersion",
"allowed_values": "*"
},
{
"field_name": "appBuildID",
"allowed_values": "*"
},
{
"field_name": "submission_date",
"allowed_values": {
"min": "20130101",
"max": "20131231"
}
}
]
}
def test_write_filename(self):
test_file = os.path.join(self.get_test_dir(), "test.log")
self.storage.write_filename("foo", '{"bar":"baz"}', test_file)
test_file_md5, test_file_size = fileutil.md5file(test_file)
self.assertEqual(test_file_md5, "0ea91df239ea79ed2ebab34b46d455fc")
test_file = os.path.join(self.get_test_dir(), "test2.log")
# Now test writing an object
self.storage.write_filename("foo", {"bar":"baz"}, test_file)
test_file_md5, test_file_size = fileutil.md5file(test_file)
self.assertEqual(test_file_md5, "0ea91df239ea79ed2ebab34b46d455fc")
def test_write(self):
dims = ["r1", "a1", "c1", "v1", "b1", "20130102"]
test_dir = self.get_test_dir()
test_file = self.schema.get_filename(test_dir, dims)
self.assertEquals(test_file, test_dir + "/r1/a1/c1/v1/b1.20130102.v1.log")
self.storage.write("foo", '{"bar":"baz"}', dims)
md5, size = fileutil.md5file(test_file)
self.assertEqual(md5, "0ea91df239ea79ed2ebab34b46d455fc")
def test_clean_newlines(self):
self.assertEqual(self.storage.clean_newlines("ab\n\ncd\r\n"), "ab cd ")
def test_rotate(self):
test_file = os.path.join(self.get_test_dir(), "test.log")
key = "01234567890123456789012345678901234567890123456789"
value = '{"some filler stuff here":"fffffffffffffffffff"}'
# each iteration should be 100 bytes.
for i in range(99):
result = self.storage.write_filename(key, value, test_file)
self.assertEquals(result, test_file)
# The 100th iteration should cause the file to rotate
rolled = self.storage.write_filename(key, value, test_file)
# rolled should be <test_dir>/test.log.<pid>.<timestamp><suffix>
self.assertNotEqual(rolled, test_file)
self.assertTrue(rolled.startswith(test_file))
self.assertTrue(rolled.endswith(StorageLayout.PENDING_COMPRESSION_SUFFIX))
"allowed_values": ["c1", "c2", "c3"]
},
{
"field_name": "appVersion",
"allowed_values": "*"
},
{
"field_name": "appBuildID",
"allowed_values": "*"
},
{
"field_name": "submission_date",
"allowed_values": {
"min": "20130101",
"max": "20131231"
}
}
]
}
try:
schema = TelemetrySchema(schema_spec)
storage = StorageLayout(schema, test_dir, 10000)
test_file_1 = os.path.join(test_dir, "test.log")
storage.write_filename("foo", '{"bar": "baz"}', test_file_1)
test_file_1_md5, test_file_1_size = fileutil.md5file(test_file_1)
assert test_file_1_md5 == "206dd2d33a04802c31d2c74f10cc472b"
assert storage.clean_newlines("ab\n\ncd\r\n") == "ab cd "
finally:
shutil.rmtree(test_dir)
class TestPersist(unittest.TestCase):
def setUp(self):
test_dir = self.get_test_dir()
self.schema = TelemetrySchema(self.get_schema_spec())
self.storage = StorageLayout(self.schema, test_dir, 10000)
assert not os.path.exists(test_dir)
os.makedirs(test_dir)
def tearDown(self):
shutil.rmtree(self.get_test_dir())
def get_test_dir(self):
return "/tmp/test_telemetry_persist"
def get_schema_spec(self):
return {
"version":
1,
"dimensions": [{
"field_name": "reason",
"allowed_values": ["r1", "r2"]
}, {
"field_name": "appName",
"allowed_values": ["a1"]
}, {
"field_name": "appUpdateChannel",
"allowed_values": ["c1", "c2", "c3"]
}, {
"field_name": "appVersion",
"allowed_values": "*"
}, {
"field_name": "appBuildID",
"allowed_values": "*"
}, {
"field_name": "submission_date",
"allowed_values": {
"min": "20130101",
"max": "20131231"
}
}]
}
def test_write_filename(self):
test_file = os.path.join(self.get_test_dir(), "test.log")
self.storage.write_filename("foo", '{"bar":"baz"}', test_file)
test_file_md5, test_file_size = fileutil.md5file(test_file)
self.assertEqual(test_file_md5, "0ea91df239ea79ed2ebab34b46d455fc")
test_file = os.path.join(self.get_test_dir(), "test2.log")
# Now test writing an object
self.storage.write_filename("foo", {"bar": "baz"}, test_file)
test_file_md5, test_file_size = fileutil.md5file(test_file)
self.assertEqual(test_file_md5, "0ea91df239ea79ed2ebab34b46d455fc")
def test_write(self):
dims = ["r1", "a1", "c1", "v1", "b1", "20130102"]
test_dir = self.get_test_dir()
test_file = self.schema.get_filename(test_dir, dims)
self.assertEquals(test_file,
test_dir + "/r1/a1/c1/v1/b1.20130102.v1.log")
self.storage.write("foo", '{"bar":"baz"}', dims)
md5, size = fileutil.md5file(test_file)
self.assertEqual(md5, "0ea91df239ea79ed2ebab34b46d455fc")
def test_clean_newlines(self):
self.assertEqual(self.storage.clean_newlines("ab\n\ncd\r\n"),
"ab cd ")
def test_rotate(self):
test_file = os.path.join(self.get_test_dir(), "test.log")
key = "01234567890123456789012345678901234567890123456789"
value = '{"some filler stuff here":"fffffffffffffffffff"}'
# each iteration should be 100 bytes.
for i in range(99):
result = self.storage.write_filename(key, value, test_file)
self.assertEquals(result, test_file)
# The 100th iteration should cause the file to rotate
rolled = self.storage.write_filename(key, value, test_file)
# rolled should be <test_dir>/test.log.<pid>.<timestamp><suffix>
self.assertNotEqual(rolled, test_file)
self.assertTrue(rolled.startswith(test_file))
self.assertTrue(
rolled.endswith(StorageLayout.PENDING_COMPRESSION_SUFFIX))
