def test_key_value_pairs(self):
"""Testing method / function key_value_pairs."""
# Create a PattooDBrecord
record = PattooDBrecord(
pattoo_checksum='1',
pattoo_key='3',
pattoo_agent_polling_interval=1,
pattoo_agent_id=4,
pattoo_timestamp=5,
pattoo_data_type=DATA_FLOAT,
pattoo_value=6,
pattoo_agent_polled_target='x',
pattoo_agent_program='y',
pattoo_agent_hostname='z',
pattoo_metadata=[('key1', 'value'), ('key2', 'value')]
)
# Test
expected = [
('pattoo_key', 'y_3'), ('y_key1', 'value'), ('y_key2', 'value')
]
result = get.key_value_pairs(record)
self.assertEqual(sorted(result), expected)
# Test with a list
result = get.key_value_pairs([record])
self.assertEqual(result, expected)
def test_process_db_records(self):
"""Testing method / function process_db_records."""
# Initialize key variables
items = make_records()
timestamps = items['timestamps']
records = items['records']
expected = items['expected']
checksum = items['checksum']
# Entry should not exist
result = datapoint.checksum_exists(checksum)
self.assertFalse(result)
# Create key-pair values in the database
kvps = get.key_value_pairs(records)
pair.insert_rows(kvps)
# Insert
ingest_data.process_db_records(records)
# Get data from database
idx_datapoint = datapoint.checksum_exists(checksum)
_dp = datapoint.DataPoint(idx_datapoint)
ts_start = min(timestamps)
ts_stop = max(timestamps)
results = _dp.data(ts_start, ts_stop)
# Test
for index, result in enumerate(results):
self.assertEqual(result['value'], expected[index]['value'])
self.assertEqual(result['timestamp'], expected[index]['timestamp'])
def singleprocess_pairs(self):
"""Update rows in the Pair database table if necessary.
Args:
None
Returns:
None
"""
# Process data
pairs = []
for item in self._arguments:
row = item[0]
per_process_key_value_pairs = get.key_value_pairs(row)
pairs.append(per_process_key_value_pairs)
# Insert
pair.insert_rows(pairs)
def _process_kvps_exception(pattoo_db_records):
"""Get all the key-value pairs found.
Traps any exceptions and return them for processing. Very helpful in
troubleshooting multiprocessing
Args:
pattoo_db_records: List of dicts read from cache files.
Returns:
None
"""
# Initialize key variables
result = []
'''
Sleep for a short random time. We have seen where on very fast systems
SQLAlchemy will hang the creation of multiprocessing subprocesses. The
typical behaviour is the creation of one fewer
pattoo.db._add_engine_pidguard() log messages than agents to process.
These messages correspond to the creation of a subprocess which immediately
invalidates a parent process's DB connection that will cause errors
if used, which provided the clue to the source of the problem.
Though SQLAlchemy isn't used by key_value_pairs. It's added as a
future precaution in case it does.
'''
time.sleep((random.random() / 10) + 0.1)
# Execute
try:
result = get.key_value_pairs(pattoo_db_records)
except Exception as error:
_exception = sys.exc_info()
log.log2exception(20133, _exception)
return ExceptionWrapper(error)
except:
_exception = sys.exc_info()
log.log2exception_die(20111, _exception)
# Return
return result
def test_checksums(self):
"""Testing method / function checksums."""
# Initialize key variables
expected = {}
polling_interval = 1
pattoo_agent_polled_target = 'panda_bear'
pattoo_agent_program = 'koala_bear'
pattoo_agent_hostname = 'grizzly_bear'
# Insert an entry in the agent table
agent_id = data.hashstring(str(random()))
idx_agent = agent.idx_agent(agent_id, pattoo_agent_polled_target,
pattoo_agent_program)
# Populate database with key-value pairs
for data_index in range(0, 10):
time.sleep(0.1)
# Add the checksum to the database
checksum = data.hashstring(str(random()))
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval,
idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Define what we expect from the test function
expected[checksum] = ChecksumLookup(
idx_datapoint=idx_datapoint,
polling_interval=polling_interval,
last_timestamp=1)
# Add key-pairs to the database
record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key='key',
pattoo_agent_polling_interval=polling_interval,
pattoo_agent_id=agent_id,
pattoo_timestamp=int(time.time() * 1000),
pattoo_data_type=DATA_FLOAT,
pattoo_value=(data_index * 10),
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program=pattoo_agent_program,
pattoo_agent_hostname=pattoo_agent_hostname,
pattoo_metadata=[])
pairs = get.key_value_pairs(record)
pair.insert_rows(pairs)
idx_pairs = pair.idx_pairs(pairs)
# Create glue entry
glue.insert_rows(idx_datapoint, idx_pairs)
#######################################################################
# This is added to verify that we only get a subset of results
#######################################################################
# Insert an entry in the agent table
fake_agent_id = data.hashstring(str(random()))
idx_agent = agent.idx_agent(fake_agent_id, pattoo_agent_polled_target,
pattoo_agent_program)
# Populate database with key-value pairs
for data_index in range(0, 17):
time.sleep(0.1)
# Add the checksum to the database
checksum = data.hashstring(str(random()))
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval,
idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Add key-pairs to the database
record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key='key',
pattoo_agent_polling_interval=polling_interval,
pattoo_agent_id=fake_agent_id,
pattoo_timestamp=int(time.time() * 1000),
pattoo_data_type=DATA_FLOAT,
pattoo_value=(data_index * 10),
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program=pattoo_agent_program,
pattoo_agent_hostname=pattoo_agent_hostname,
pattoo_metadata=[])
pairs = get.key_value_pairs(record)
pair.insert_rows(pairs)
idx_pairs = pair.idx_pairs(pairs)
# Create glue entry
glue.insert_rows(idx_datapoint, idx_pairs)
# Test
result = misc.agent_checksums(agent_id)
self.assertTrue(bool(result))
self.assertTrue(isinstance(result, dict))
self.assertEqual(len(result), len(expected))
for key, value in result.items():
self.assertEqual(value.idx_datapoint, expected[key].idx_datapoint)
self.assertEqual(value.polling_interval,
expected[key].polling_interval)
self.assertEqual(value.last_timestamp,
expected[key].last_timestamp)
