def multiprocess_pairs(self):
"""Update rows in the Pair database table if necessary.
Do all multiprocessing outside of the class for consistent results
without unexpected hanging waiting for pool.join() to happen.
Args:
None
Returns:
None
"""
# Initialize key variables
pattoo_db_records_lists_tuple = self._arguments
pool_size = self._pool_size
# Create a pool of sub process resources
with get_context('spawn').Pool(processes=pool_size) as pool:
# Create sub processes from the pool
per_process_key_value_pairs = pool.starmap(
_process_kvps_exception, pattoo_db_records_lists_tuple)
# Wait for all the processes to end and get results
pool.join()
# Test for exceptions
for result in per_process_key_value_pairs:
if isinstance(result, ExceptionWrapper):
result.re_raise()
# Update the database with key value pairs
pair.insert_rows(per_process_key_value_pairs)
def test_process_db_records(self):
"""Testing method / function process_db_records."""
# Initialize key variables
items = make_records()
timestamps = items['timestamps']
records = items['records']
expected = items['expected']
checksum = items['checksum']
# Entry should not exist
result = datapoint.checksum_exists(checksum)
self.assertFalse(result)
# Create key-pair values in the database
kvps = get.key_value_pairs(records)
pair.insert_rows(kvps)
# Insert
ingest_data.process_db_records(records)
# Get data from database
idx_datapoint = datapoint.checksum_exists(checksum)
_dp = datapoint.DataPoint(idx_datapoint)
ts_start = min(timestamps)
ts_stop = max(timestamps)
results = _dp.data(ts_start, ts_stop)
# Test
for index, result in enumerate(results):
self.assertEqual(result['value'], expected[index]['value'])
self.assertEqual(result['timestamp'], expected[index]['timestamp'])
def test_insert_rows(self):
"""Testing method / function insert_rows."""
# Initialize key variables
polling_interval = 1
checksum = data.hashstring(str(random()))
key = data.hashstring(str(random()))
value = data.hashstring(str(random()))
# Create a new Agent entry
agent_id = data.hashstring(str(random()))
agent_target = data.hashstring(str(random()))
agent_program = data.hashstring(str(random()))
agent.insert_row(agent_id, agent_target, agent_program)
idx_agent = agent.exists(agent_id, agent_target)
# Insert values in tables
pair.insert_rows((key, value))
idx_pair = pair.pair_exists(key, value)
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Create entry and check
result = glue.glue_exists(idx_datapoint, idx_pair)
self.assertFalse(result)
glue.insert_rows(idx_datapoint, idx_pair)
result = glue.glue_exists(idx_datapoint, idx_pair)
self.assertTrue(bool(result))
self.assertTrue(isinstance(result, int))
def test_idx_pairs(self):
"""Testing method / function idx_pairs."""
# Initialize key variables
checksum = data.hashstring(str(random()))
polling_interval = 1
keypairs = []
idx_pairs = []
for _ in range(0, 10):
time.sleep(0.05)
key = data.hashstring(str(random()))
value = data.hashstring(str(random()))
keypairs.append((key, value))
# Create a new Agent entry
agent_id = data.hashstring(str(random()))
agent_target = data.hashstring(str(random()))
agent_program = data.hashstring(str(random()))
agent.insert_row(agent_id, agent_target, agent_program)
idx_agent = agent.exists(agent_id, agent_target)
# Insert values in tables
pair.insert_rows(keypairs)
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Test
for key, value in keypairs:
idx_pairs.append(pair.pair_exists(key, value))
glue.insert_rows(idx_datapoint, idx_pairs)
result = glue.idx_pairs(idx_datapoint)
self.assertEqual(len(result), len(idx_pairs))
for idx_pair in idx_pairs:
self.assertTrue(idx_pair in result)
def test_insert_rows(self):
"""Testing method / function insert_rows."""
# Initialize key variables
key = data.hashstring(str(random()))
value = data.hashstring(str(random()))
result = pair.pair_exists(key, value)
self.assertFalse(result)
# Create entry and check
pair.insert_rows((key, value))
result = pair.pair_exists(key, value)
self.assertTrue(bool(result))
self.assertTrue(isinstance(result, int))
def singleprocess_pairs(self):
"""Update rows in the Pair database table if necessary.
Args:
None
Returns:
None
"""
# Process data
pairs = []
for item in self._arguments:
row = item[0]
per_process_key_value_pairs = get.key_value_pairs(row)
pairs.append(per_process_key_value_pairs)
# Insert
pair.insert_rows(pairs)
def test_idx_pairs(self):
"""Testing method / function idx_pairs."""
# Initialize key variables
keypairs = []
idx_pairs = []
for _ in range(0, 10):
time.sleep(0.05)
key = data.hashstring(str(random()))
value = data.hashstring(str(random()))
keypairs.append((key, value))
# Insert values in tables
pair.insert_rows(keypairs)
# Test
for key, value in keypairs:
idx_pairs.append(pair.pair_exists(key, value))
result = pair.idx_pairs(keypairs)
self.assertEqual(len(result), len(idx_pairs))
for idx_pair in idx_pairs:
self.assertTrue(idx_pair in result)
def pairs(pattoo_db_record):
"""Create db Pair table entries.
Args:
pattoo_db_record: PattooDBrecord object
Returns:
result: List of Pair.idx_pair database index values
"""
# Initialize key variables
result = []
# Get key-values
_pairs = key_value_pairs(pattoo_db_record)
# Get list of pairs in the database
pair.insert_rows(_pairs)
result = pair.idx_pairs(_pairs)
# Return
return result
def test_checksums(self):
"""Testing method / function checksums."""
# Initialize key variables
expected = {}
polling_interval = 1
pattoo_agent_polled_target = 'panda_bear'
pattoo_agent_program = 'koala_bear'
pattoo_agent_hostname = 'grizzly_bear'
# Insert an entry in the agent table
agent_id = data.hashstring(str(random()))
idx_agent = agent.idx_agent(agent_id, pattoo_agent_polled_target,
pattoo_agent_program)
# Populate database with key-value pairs
for data_index in range(0, 10):
time.sleep(0.1)
# Add the checksum to the database
checksum = data.hashstring(str(random()))
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval,
idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Define what we expect from the test function
expected[checksum] = ChecksumLookup(
idx_datapoint=idx_datapoint,
polling_interval=polling_interval,
last_timestamp=1)
# Add key-pairs to the database
record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key='key',
pattoo_agent_polling_interval=polling_interval,
pattoo_agent_id=agent_id,
pattoo_timestamp=int(time.time() * 1000),
pattoo_data_type=DATA_FLOAT,
pattoo_value=(data_index * 10),
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program=pattoo_agent_program,
pattoo_agent_hostname=pattoo_agent_hostname,
pattoo_metadata=[])
pairs = get.key_value_pairs(record)
pair.insert_rows(pairs)
idx_pairs = pair.idx_pairs(pairs)
# Create glue entry
glue.insert_rows(idx_datapoint, idx_pairs)
#######################################################################
# This is added to verify that we only get a subset of results
#######################################################################
# Insert an entry in the agent table
fake_agent_id = data.hashstring(str(random()))
idx_agent = agent.idx_agent(fake_agent_id, pattoo_agent_polled_target,
pattoo_agent_program)
# Populate database with key-value pairs
for data_index in range(0, 17):
time.sleep(0.1)
# Add the checksum to the database
checksum = data.hashstring(str(random()))
datapoint.insert_row(checksum, DATA_FLOAT, polling_interval,
idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Add key-pairs to the database
record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key='key',
pattoo_agent_polling_interval=polling_interval,
pattoo_agent_id=fake_agent_id,
pattoo_timestamp=int(time.time() * 1000),
pattoo_data_type=DATA_FLOAT,
pattoo_value=(data_index * 10),
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program=pattoo_agent_program,
pattoo_agent_hostname=pattoo_agent_hostname,
pattoo_metadata=[])
pairs = get.key_value_pairs(record)
pair.insert_rows(pairs)
idx_pairs = pair.idx_pairs(pairs)
# Create glue entry
glue.insert_rows(idx_datapoint, idx_pairs)
# Test
result = misc.agent_checksums(agent_id)
self.assertTrue(bool(result))
self.assertTrue(isinstance(result, dict))
self.assertEqual(len(result), len(expected))
for key, value in result.items():
self.assertEqual(value.idx_datapoint, expected[key].idx_datapoint)
self.assertEqual(value.polling_interval,
expected[key].polling_interval)
self.assertEqual(value.last_timestamp,
expected[key].last_timestamp)