def test_idx_pairs(self):
"""Testing method / function idx_pairs."""
# Initialize key variables
checksum = data.hashstring(str(random()))
polling_interval = 1
keypairs = []
idx_pairs = []
for _ in range(0, 10):
time.sleep(0.05)
key = data.hashstring(str(random()))
value = data.hashstring(str(random()))
keypairs.append((key, value))
# Create a new Agent entry
agent_id = data.hashstring(str(random()))
agent_target = data.hashstring(str(random()))
agent_program = data.hashstring(str(random()))
agent.insert_row(agent_id, agent_target, agent_program)
idx_agent = agent.exists(agent_id, agent_target)
# Insert values in tables
pair.insert_rows(keypairs)
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Test
for key, value in keypairs:
idx_pairs.append(pair.pair_exists(key, value))
glue.insert_rows(idx_datapoint, idx_pairs)
result = glue.idx_pairs(idx_datapoint)
self.assertEqual(len(result), len(idx_pairs))
for idx_pair in idx_pairs:
self.assertTrue(idx_pair in result)
def test_insert_rows(self):
"""Testing method / function insert_rows."""
# Initialize key variables
polling_interval = 1
checksum = data.hashstring(str(random()))
key = data.hashstring(str(random()))
value = data.hashstring(str(random()))
# Create a new Agent entry
agent_id = data.hashstring(str(random()))
agent_target = data.hashstring(str(random()))
agent_program = data.hashstring(str(random()))
agent.insert_row(agent_id, agent_target, agent_program)
idx_agent = agent.exists(agent_id, agent_target)
# Insert values in tables
pair.insert_rows((key, value))
idx_pair = pair.pair_exists(key, value)
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Create entry and check
result = glue.glue_exists(idx_datapoint, idx_pair)
self.assertFalse(result)
glue.insert_rows(idx_datapoint, idx_pair)
result = glue.glue_exists(idx_datapoint, idx_pair)
self.assertTrue(bool(result))
self.assertTrue(isinstance(result, int))
def process_db_records(pattoo_db_records):
"""Insert all data values for an agent into database.
Args:
pattoo_db_records: List of dicts read from cache files.
Returns:
None
Method:
1) Get all the idx_datapoint and idx_pair values that exist in the
PattooDBrecord data from the database. All the records MUST be
from the same source.
2) Add these idx values to tracking memory variables for speedy lookup
3) Ignore non numeric data values sent
4) Add data to the database. If new checksum values are found in the
PattooDBrecord data, then create the new index values to the
database, update the tracking memory variables before hand.
"""
# Initialize key variables
_data = {}
# Return if there is nothint to process
if bool(pattoo_db_records) is False:
return
# Get DataPoint.idx_datapoint and idx_pair values from db. This is used to
# speed up the process by reducing the need for future database access.
agent_id = pattoo_db_records[0].pattoo_agent_id
checksum_table = misc.agent_checksums(agent_id)
# Process data
for pdbr in pattoo_db_records:
# We only want to insert non-string, non-None values
if pdbr.pattoo_data_type in [DATA_NONE, DATA_STRING]:
continue
# Try to make the value a float for insertion into the database
try:
float_value = float(pdbr.pattoo_value)
except:
continue
# Get the idx_datapoint value for the PattooDBrecord
if pdbr.pattoo_checksum in checksum_table:
# Get last_timestamp for existing idx_datapoint entry
idx_datapoint = checksum_table[pdbr.pattoo_checksum].idx_datapoint
else:
# Entry not in database. Update the database and get the
# required idx_datapoint
idx_datapoint = datapoint.idx_datapoint(pdbr)
if bool(idx_datapoint) is True:
# Update the lookup table
checksum_table[pdbr.pattoo_checksum] = ChecksumLookup(
idx_datapoint=idx_datapoint,
polling_interval=int(pdbr.pattoo_agent_polling_interval),
last_timestamp=1)
# Update the Glue table
idx_pairs = get.pairs(pdbr)
glue.insert_rows(idx_datapoint, idx_pairs)
else:
continue
# Append item to items
if pdbr.pattoo_timestamp > checksum_table[
pdbr.pattoo_checksum].last_timestamp:
'''
Add the Data table results to a dict in case we have duplicate
posting over the API. We need to key off a unique time dependent
value per datapoint to prevent different datapoints at the same
point in time overwriting the value. This is specifically for
removing duplicates for the _SAME_ datapoint at the same point in
time as could possibly occur with the restart of an agent causing a
double posting or network issues. We therefore use a tuple of
idx_datapoint and timestamp.
'''
_data[(pdbr.pattoo_timestamp, idx_datapoint)] = IDXTimestampValue(
idx_datapoint=idx_datapoint,
polling_interval=int(pdbr.pattoo_agent_polling_interval),
timestamp=pdbr.pattoo_timestamp,
value=float_value)
# Update the data table
if bool(_data) is True:
data.insert_rows(list(_data.values()))
# Log message
log_message = ('''\
Finished cache data processing for agent_id: {}'''.format(agent_id))
log.log2debug(20113, log_message)
def test_checksums(self):
"""Testing method / function checksums."""
# Initialize key variables
expected = {}
polling_interval = 1
pattoo_agent_polled_target = 'panda_bear'
pattoo_agent_program = 'koala_bear'
pattoo_agent_hostname = 'grizzly_bear'
# Insert an entry in the agent table
agent_id = data.hashstring(str(random()))
idx_agent = agent.idx_agent(agent_id, pattoo_agent_polled_target,
pattoo_agent_program)
# Populate database with key-value pairs
for data_index in range(0, 10):
time.sleep(0.1)
# Add the checksum to the database
checksum = data.hashstring(str(random()))
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval,
idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Define what we expect from the test function
expected[checksum] = ChecksumLookup(
idx_datapoint=idx_datapoint,
polling_interval=polling_interval,
last_timestamp=1)
# Add key-pairs to the database
record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key='key',
pattoo_agent_polling_interval=polling_interval,
pattoo_agent_id=agent_id,
pattoo_timestamp=int(time.time() * 1000),
pattoo_data_type=DATA_FLOAT,
pattoo_value=(data_index * 10),
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program=pattoo_agent_program,
pattoo_agent_hostname=pattoo_agent_hostname,
pattoo_metadata=[])
pairs = get.key_value_pairs(record)
pair.insert_rows(pairs)
idx_pairs = pair.idx_pairs(pairs)
# Create glue entry
glue.insert_rows(idx_datapoint, idx_pairs)
#######################################################################
# This is added to verify that we only get a subset of results
#######################################################################
# Insert an entry in the agent table
fake_agent_id = data.hashstring(str(random()))
idx_agent = agent.idx_agent(fake_agent_id, pattoo_agent_polled_target,
pattoo_agent_program)
# Populate database with key-value pairs
for data_index in range(0, 17):
time.sleep(0.1)
# Add the checksum to the database
checksum = data.hashstring(str(random()))
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval,
idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Add key-pairs to the database
record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key='key',
pattoo_agent_polling_interval=polling_interval,
pattoo_agent_id=fake_agent_id,
pattoo_timestamp=int(time.time() * 1000),
pattoo_data_type=DATA_FLOAT,
pattoo_value=(data_index * 10),
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program=pattoo_agent_program,
pattoo_agent_hostname=pattoo_agent_hostname,
pattoo_metadata=[])
pairs = get.key_value_pairs(record)
pair.insert_rows(pairs)
idx_pairs = pair.idx_pairs(pairs)
# Create glue entry
glue.insert_rows(idx_datapoint, idx_pairs)
# Test
result = misc.agent_checksums(agent_id)
self.assertTrue(bool(result))
self.assertTrue(isinstance(result, dict))
self.assertEqual(len(result), len(expected))
for key, value in result.items():
self.assertEqual(value.idx_datapoint, expected[key].idx_datapoint)
self.assertEqual(value.polling_interval,
expected[key].polling_interval)
self.assertEqual(value.last_timestamp,
expected[key].last_timestamp)