def test_write_raise_value_error_if_item_in_deviceapps_list_is_not_dict(
self):
devapps = [1, 2, 3]
with self.assertRaisesRegexp(
ValueError,
"Item in list of 'deviceapps' must be a dictionary type"):
pb.deviceapps_xwrite_pb(devapps, TEST_FILE)
def test_invalid_lon(self):
messages = [{"lon": "invalid"}]
with self.assertRaises(Exception) as err:
pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertEqual("Could not serialize the data.", str(err.exception))
def test_ivalid_apps(self):
messages = [{"apps": "invalid", "lat": 2.314141, "lon": 5.66523423}]
with self.assertRaises(Exception) as err:
pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertEqual("Could not serialize the data.", str(err.exception))
def test_read(self):
pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
count = 0
for i, d in enumerate(pb.deviceapps_xread_pb(TEST_FILE)):
self.assertEqual(d, self.deviceapps[i])
count += 1
self.assertEqual(count, len(self.deviceapps))
def test_invalid_device_type(self):
messages = [{"device": {"type": None}}]
with self.assertRaises(Exception) as err:
pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertEqual("Could not serialize the data.", str(err.exception))
def test_message_device_not_a_dict(self):
with self.assertRaises(TypeError):
pb.deviceapps_xwrite_pb(
{
"device": ["type"],
"lat": 42,
"lon": -42,
"apps": [1, 2]
}, TEST_FILE)
def test_message_device_type_not_a_string(self):
with self.assertRaises(TypeError):
pb.deviceapps_xwrite_pb(
{
"device": {
"type": 111,
"id": "e7e1a50c0ec2747ca56cd9e1558c0d7d"
}
}, TEST_FILE)
def test_write_raise_value_error_if_lat_is_not_integer_or_float(self):
devapps = [{
"lat": {
"type": 123,
"id": "e7e1a50c0ec2747ca56cd9e1558c0d7d"
}
}]
with self.assertRaisesRegexp(
ValueError, "'lat' must be a float or an integer type"):
pb.deviceapps_xwrite_pb(devapps, TEST_FILE)
def test_write_raise_value_error_if_type_is_not_string(self):
devapps = [{
"device": {
"type": 123,
"id": "e7e1a50c0ec2747ca56cd9e1558c0d7d"
}
}]
with self.assertRaisesRegexp(ValueError,
"'type' must be a string type"):
pb.deviceapps_xwrite_pb(devapps, TEST_FILE)
def test_message_apps_app_not_a_int(self):
with self.assertRaises(TypeError):
pb.deviceapps_xwrite_pb(
{
"device": {
"type": "gaid",
"id": "e7e1a50c0ec2747ca56cd9e1558c0d7d"
},
"lat": 42,
"lon": -42,
"apps": ["1", "2"]
}, TEST_FILE)
def test_write(self):
header_size = 8
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE, 'r') as fi:
for deviceapp in self.deviceapps:
magic, device_type, message_length = unpack(
'<Ihh', fi.read(header_size))
self.assertEqual(magic, MAGIC)
self.assertEqual(device_type, DEVICE_APPS_TYPE)
message = fi.read(message_length)
da = dapps_pb2.DeviceApps()
da.ParseFromString(message)
da_orig = dapps_pb2.DeviceApps()
da_orig.device.id = deviceapp['device']['id']
da_orig.device.type = deviceapp['device']['type']
da_orig.apps.extend(deviceapp['apps'])
if 'lat' in deviceapp:
da_orig.lat = deviceapp['lat']
if 'lon' in deviceapp:
da_orig.lon = deviceapp['lon']
self.assertEqual(da, da_orig)
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
# Test Data
header_size = 8
with gzip.open(TEST_FILE, 'rb') as f:
for deviceapp in self.deviceapps:
# Test Header
header = f.read(header_size)
magic, dev_apps_type, length = struct.unpack('<IHH', header)
self.assertEqual(magic, MAGIC)
self.assertEqual(dev_apps_type, DEVICE_APPS_TYPE)
# Test Body
raw_data = f.read(length)
data = deviceapps_pb2.DeviceApps()
data.ParseFromString(raw_data)
self.assertEqual(data.device.id,
deviceapp.get('device', {}).get('id', ''))
self.assertEqual(data.device.type,
deviceapp.get('device', {}).get('type', ''))
self.assertEqual(data.lat, deviceapp.get('lat', 0))
self.assertEqual(data.lon, deviceapp.get('lon', 0))
self.assertEqual(data.apps, deviceapp.get('apps', []))
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE) as fd:
item_index = 0
while True:
header = self.read_header(fd)
if not header:
break
magic, type_, next_item_len = header
self.assertEqual(magic, MAGIC)
self.assertEqual(type_, DEVICE_APPS_TYPE)
self.assertTrue(next_item_len >= 0)
reference_item = self.deviceapps[item_index]
device_apps = deviceapps_pb2.DeviceApps()
data = fd.read(next_item_len)
device_apps.ParseFromString(data)
self.assertEqual(device_apps.device.id,
reference_item['device']['id'])
self.assertEqual(device_apps.device.type,
reference_item['device']['type'])
self.assertEqual(device_apps.apps, reference_item['apps'])
self.assertEqual(device_apps.lat, reference_item.get('lat', 0))
self.assertEqual(device_apps.lon, reference_item.get('lon', 0))
item_index += 1
self.assertEqual(item_index, len(self.deviceapps))
def test_write_ok(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE) as f:
for deviceapp in self.deviceapps:
# Test header
magic, device_apps_type, length = unpack(
'<IHH', f.read(HEADER_SIZE))
self.assertEqual(magic, MAGIC)
self.assertEqual(device_apps_type, DEVICE_APPS_TYPE)
# Test body
unpacked = deviceapps_pb2.DeviceApps()
unpacked.ParseFromString(f.read(length))
self.assertEqual(
unpacked.device.id,
deviceapp.get('device', {}).get('id', '').encode())
self.assertEqual(
unpacked.device.type,
deviceapp.get('device', {}).get('type', '').encode())
self.assertEqual(unpacked.lat, deviceapp.get('lat', 0))
self.assertEqual(unpacked.lon, deviceapp.get('lon', 0))
self.assertEqual(unpacked.apps, deviceapp.get('apps', []))
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE) as fd:
data = fd.read()
self.assertEquals(bytes_written, len(data))
offset = 0
for el in self.deviceapps:
magic, dev_apps_type, length = unpack('<IHH',
data[offset:offset + 8])
self.assertEquals(magic, MAGIC)
self.assertEquals(dev_apps_type, DEVICE_APPS_TYPE)
msg_data = data[offset + 8:offset + 8 + length]
da = dapps.DeviceApps()
da.ParseFromString(msg_data)
da_orig = dapps.DeviceApps()
da_orig.device.id = el['device']['id']
da_orig.device.type = el['device']['type']
da_orig.apps.extend(el['apps'])
if 'lat' in el:
da_orig.lat = el['lat']
if 'lon' in el:
da_orig.lon = el['lon']
self.assertEquals(da, da_orig)
offset += 8 + length
def test_empty_list(self):
messages = []
bytes_written = pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertEqual(0, bytes_written)
self.assertTrue(os.path.isfile(TEST_FILE))
def test_list_with_invalid_messages(self):
messages = [1, 2, 3]
bytes_written = pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertEqual(0, bytes_written)
self.assertTrue(os.path.isfile(TEST_FILE))
def test_valid_coordinates(self):
messages = [
{
"lat": 0
},
{
"lon": 0
},
{
"lat": 0,
"lon": 0
},
{
"lat": 42,
"lon": 42
},
{
"lat": -42.123123123,
"lon": 42.123213
},
]
bytes_written = pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertTrue(bytes_written > 0)
self.assertTrue(os.path.isfile(TEST_FILE))
unpacked = list(pb.deviceapps_xread_pb(TEST_FILE))
self.assertEqual(len(messages), len(unpacked))
for m1, m2 in zip(messages, unpacked):
self.assertMessagesEqual(m1, m2)
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE) as f:
for deviceapp in self.deviceapps:
# Test header
header = f.read(8) # uint32 + 2 * uint16 (arch depend)
magic, dev_apps_type, length = struct.unpack('<IHH', header)
self.assertEqual(magic, MAGIC)
self.assertEqual(dev_apps_type, DEVICE_APPS_TYPE)
# Test unpacked data
packed = f.read(length)
unpacked = deviceapps_pb2.DeviceApps()
unpacked.ParseFromString(packed)
self.assertEqual(unpacked.device.type,
deviceapp['device']['type'])
self.assertEqual(unpacked.device.id, deviceapp['device']['id'])
self.assertEqual(unpacked.HasField('lat'), 'lat' in deviceapp)
self.assertEqual(unpacked.HasField('lon'), 'lon' in deviceapp)
if unpacked.HasField('lat'):
self.assertEqual(unpacked.lat, deviceapp['lat'])
if unpacked.HasField('lon'):
self.assertEqual(unpacked.lon, deviceapp['lon'])
self.assertEqual(unpacked.apps, deviceapp['apps'])
def test_write_bad_device_dict(self):
bytes_written = pb.deviceapps_xwrite_pb([
{
"device": 42,
"apps": [1]
},
], TEST_FILE)
self.assertEqual(0, bytes_written)
def test_header(self):
pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
with gzip.open(TEST_FILE, "rb") as f:
content = f.read()
self.assertTrue(isinstance(content, bytes))
# First message
magic, device_apps_type, length = struct.unpack("IHH", content[:8])
self.assertEqual((MAGIC, DEVICE_APPS_TYPE), (magic, device_apps_type))
# Second message
offset = 8 + length
magic, device_apps_type, length = struct.unpack(
"IHH", content[offset:(offset + 8)])
self.assertEqual((MAGIC, DEVICE_APPS_TYPE), (magic, device_apps_type))
self.assertEqual(len(content), offset + 8 + length)
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE) as gz_file:
for _ in range(len(self.deviceapps)):
magic, dev_type, length = unpack("<Ihh", gz_file.read(HEADER_SIZE))
self.assertEqual(magic, MAGIC)
self.assertEqual(dev_type, DEVICE_APPS_TYPE)
_ = gz_file.read(length)
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
print(bytes_written)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE, mode='rb') as f:
record_magic = int.from_bytes(f.read(4), byteorder="little")
self.assertEqual(MAGIC, record_magic)
record_device_apps_type = int.from_bytes(f.read(2),
byteorder="little")
self.assertEqual(DEVICE_APPS_TYPE, record_device_apps_type)
msg_len = int.from_bytes(f.read(2), byteorder="little")
self.assertTrue(msg_len > 0)
def test_empty_device(self):
messages = [{"lat": 2.314141, "lon": 5.66523423}]
bytes_written = pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertTrue(bytes_written > 0)
self.assertTrue(os.path.isfile(TEST_FILE))
unpacked = list(pb.deviceapps_xread_pb(TEST_FILE))
self.assertEqual(len(unpacked), 1)
for m1, m2 in zip(messages, unpacked):
self.assertMessagesEqual(m1, m2)
def test_empty_messages(self):
messages = [{}, {}, {}]
bytes_written = pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertTrue(bytes_written > 0)
self.assertTrue(os.path.isfile(TEST_FILE))
unpacked = list(pb.deviceapps_xread_pb(TEST_FILE))
self.assertEqual(len(unpacked), 3)
for m1, m2 in zip(messages, unpacked):
self.assertMessagesEqual(m1, m2)
def test_invalid_device_keys(self):
messages = [{"device": {"foo": 42, "bar": 777}}]
expected = [{"device": {}}]
bytes_written = pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertTrue(bytes_written > 0)
self.assertTrue(os.path.isfile(TEST_FILE))
unpacked = list(pb.deviceapps_xread_pb(TEST_FILE))
self.assertEqual(len(unpacked), 1)
for m1, m2 in zip(expected, unpacked):
self.assertMessagesEqual(m1, m2)
def test_write_wrong_apps_types(self):
with self.assertRaises(ValueError):
bytes_written = pb.deviceapps_xwrite_pb([
{
"device": {
"type": "gaid",
"id": "e7e1a50c0ec2747ca56cd9e1558c0d7d"
},
"apps": ["wrong_type"]
},
], TEST_FILE)
print("bytes_written: {}".format(bytes_written))
with self.assertRaises(ValueError):
bytes_written = pb.deviceapps_xwrite_pb([
{
"device": {
"type": "gaid",
"id": "e7e1a50c0ec2747ca56cd9e1558c0d7d"
},
"apps": [1, 2, "wrong_type"]
},
], TEST_FILE)
print("bytes_written: {}".format(bytes_written))
with self.assertRaises(ValueError):
bytes_written = pb.deviceapps_xwrite_pb([
{
"device": {
"type": "gaid",
"id": "e7e1a50c0ec2747ca56cd9e1558c0d7d"
},
"apps": [1.0]
},
], TEST_FILE)
print("bytes_written: {}".format(bytes_written))
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
dapps = deviceapps_pb2.DeviceApps()
with gzip.open(TEST_FILE, 'rb') as fp:
for app in self.deviceapps:
magic, apps_type, length = struct.unpack('Ihh', fp.read(8))
self.assertEqual(MAGIC, magic)
self.assertEqual(DEVICE_APPS_TYPE, apps_type)
dapps.ParseFromString(fp.read(length))
if hasattr(app['device'], 'type'): self.assertEqual(app['device']['type'], dapps.device.type)
if hasattr(app['device'], 'id'): self.assertEqual(app['device']['id'], dapps.device.id)
if hasattr(app, 'lat'): self.assertEqual(app['lat'], dapps.lat)
if hasattr(app, 'lon'): self.assertEqual(app['lon'], dapps.lon)
for i in range(len(app['apps'])): self.assertEqual(app['apps'][i], dapps.apps[i])
def test_write(self):
bytes_written = pb.deviceapps_xwrite_pb(self.deviceapps, TEST_FILE)
self.assertTrue(bytes_written > 0)
with gzip.open(TEST_FILE, 'rb') as f:
content = f.read()
self.assertEqual(len(content), bytes_written)
offset = 0
for i, _ in enumerate(self.deviceapps):
r_magic, = struct.unpack('I', content[offset:(offset + 4)])
self.assertEqual(r_magic, MAGIC)
offset = offset + 4
r_device_apps_type, = struct.unpack('H',
content[offset:offset + 2])
self.assertEqual(r_device_apps_type, DEVICE_APPS_TYPE)
offset = offset + 2
r_len, = struct.unpack('H', content[offset:offset + 2])
offset = offset + r_len + 2
self.assertEqual(offset, bytes_written)
def test_valid_apps(self):
messages = [
{
"apps": [0]
},
{
"apps": [None, "aaa", []]
},
{
"apps": ["1", 2, 3.0, "4", None, 5]
},
{
"apps": [-(2**64), -1, 1, 2**32 - 1, 2**32, 2**64]
},
]
expected = [
{
"apps": [0]
},
{
"apps": []
},
{
"apps": [2, 5]
},
{
"apps": [1, 2**32 - 1]
},
]
bytes_written = pb.deviceapps_xwrite_pb(messages, TEST_FILE)
self.assertTrue(bytes_written > 0)
self.assertTrue(os.path.isfile(TEST_FILE))
unpacked = list(pb.deviceapps_xread_pb(TEST_FILE))
self.assertEqual(len(unpacked), len(messages))
for m1, m2 in zip(expected, unpacked):
self.assertMessagesEqual(m1, m2)