def test_store_batch_update(conn):
"""Test batch wise storing with updates using staging table."""
with closing(conn.cursor()) as cursor:
attribute_names = ['CCR', 'Drops']
timestamp = pytz.utc.localize(datetime.utcnow())
datasource = DataSource.from_name(cursor, "integration-test")
entitytype = EntityType.from_name(cursor, "UtranCell")
datapackage = DataPackage(
attribute_names,
[(10023 + i, timestamp, ('0.9919', '17')) for i in range(100)]
)
update_datapackage = DataPackage(
attribute_names,
[(10023 + i, timestamp, ('0.9918', '18')) for i in range(100)]
)
attributes = datapackage.deduce_attributes()
attributestore = AttributeStore(datasource, entitytype, attributes)
attributestore.create(cursor)
attributestore.store_batch(cursor, datapackage)
conn.commit()
modified_query = (
'SELECT modified FROM {0} '
'WHERE entity_id = 10023').format(
attributestore.history_table.render())
cursor.execute(modified_query)
modified_a, = cursor.fetchone()
attributestore.store_batch(cursor, update_datapackage)
conn.commit()
cursor.execute(modified_query)
modified_b, = cursor.fetchone()
assert modified_b > modified_a
cursor.execute(
"SELECT attribute_directory.materialize_curr_ptr(attributestore) "
"FROM attribute_directory.attributestore "
"WHERE id = %s", (attributestore.id,))
query = (
'SELECT timestamp, "Drops" '
'FROM {0}').format(attributestore.table.render())
cursor.execute(query)
# Row count should be the same as the stored batch size
eq_(cursor.rowcount, len(datapackage.rows))
stored_timestamp, drops = cursor.fetchone()
# Timestamp should be the same as the stored batch timestamp
eq_(stored_timestamp, timestamp)
eq_(drops, 18)
def test_compact(conn):
"""Test compacting of redundant data."""
def make_rows(timestamp):
return [
(10023 + i, timestamp, ('0.9919', '17'))
for i in range(100)
]
with closing(conn.cursor()) as cursor:
attribute_names = ['CCR', 'Drops']
datasource = DataSource.from_name(cursor, "integration-test")
entitytype = EntityType.from_name(cursor, "UtranCell")
timestamp = pytz.utc.localize(datetime.utcnow())
datapackage_a = DataPackage(
attribute_names=attribute_names,
rows=make_rows(timestamp)
)
datapackage_b = DataPackage(
attribute_names=attribute_names,
rows=make_rows(timestamp + timedelta(10))
)
attributes = datapackage_a.deduce_attributes()
attributestore = AttributeStore(datasource, entitytype, attributes)
attributestore.create(cursor)
attributestore.store_batch(cursor, datapackage_a)
conn.commit()
attributestore.store_batch(cursor, datapackage_b)
conn.commit()
count_query = (
"SELECT count(*) "
"FROM {0}").format(attributestore.history_table.render())
cursor.execute(count_query)
count, = cursor.fetchone()
# Row count should be the same as the stored batch sizes summed
eq_(count, len(datapackage_b.rows) + len(datapackage_a.rows))
attributestore.compact(cursor)
conn.commit()
cursor.execute(count_query)
count, = cursor.fetchone()
# Row count should be the same as the first stored batch size
eq_(count, len(datapackage_a.rows))
def test_store_batch_with_list_c(conn):
"""Test batch wise storing using staging table."""
attribute_names = ['height', 'refs']
timestamp = pytz.utc.localize(datetime.utcnow())
data_rows = [
(10023, timestamp, ('19.5', ['', '', '', ''])),
(10024, timestamp, ('19.3', ['', '', '', '']))
]
datapackage = DataPackage(attribute_names, data_rows)
attributes = datapackage.deduce_attributes()
with closing(conn.cursor()) as cursor:
datasource = DataSource.from_name(cursor, "integration-test")
entitytype = EntityType.from_name(cursor, "UtranCell")
attributestore = AttributeStore(datasource, entitytype, attributes)
attributestore.create(cursor)
attributestore.store_batch(cursor, datapackage)
conn.commit()
def test_store_batch_with_list_a(conn):
"""Test batch wise storing using staging table."""
attribute_names = ['height', 'refs']
timestamp = pytz.utc.localize(datetime.utcnow())
data_rows = [
(10023 + i, timestamp, ('19.5', ['r34', 'r23', 'r33']))
for i in range(100)]
datapackage = DataPackage(attribute_names, data_rows)
attributes = datapackage.deduce_attributes()
with closing(conn.cursor()) as cursor:
datasource = DataSource.from_name(cursor, "integration-test")
entitytype = EntityType.from_name(cursor, "UtranCell")
attributestore = AttributeStore(datasource, entitytype, attributes)
attributestore.create(cursor)
attributestore.store_batch(cursor, datapackage)
conn.commit()
cursor.execute(
"SELECT attribute_directory.materialize_curr_ptr(attributestore) "
"FROM attribute_directory.attributestore "
"WHERE id = %s", (attributestore.id,))
query = (
"SELECT timestamp, height "
"FROM {0}").format(attributestore.table.render())
cursor.execute(query)
# Row count should be the same as the stored batch size
eq_(cursor.rowcount, len(datapackage.rows))
stored_timestamp, height = cursor.fetchone()
# Timestamp should be the same as the stored batch timestamp
eq_(stored_timestamp, timestamp)
eq_(height, 19.5)
