def multiprocess_pairs(self): """Update rows in the Pair database table if necessary. Do all multiprocessing outside of the class for consistent results without unexpected hanging waiting for pool.join() to happen. Args: None Returns: None """ # Initialize key variables pattoo_db_records_lists_tuple = self._arguments pool_size = self._pool_size # Create a pool of sub process resources with get_context('spawn').Pool(processes=pool_size) as pool: # Create sub processes from the pool per_process_key_value_pairs = pool.starmap( _process_kvps_exception, pattoo_db_records_lists_tuple) # Wait for all the processes to end and get results pool.join() # Test for exceptions for result in per_process_key_value_pairs: if isinstance(result, ExceptionWrapper): result.re_raise() # Update the database with key value pairs pair.insert_rows(per_process_key_value_pairs)
def test_process_db_records(self): """Testing method / function process_db_records.""" # Initialize key variables items = make_records() timestamps = items['timestamps'] records = items['records'] expected = items['expected'] checksum = items['checksum'] # Entry should not exist result = datapoint.checksum_exists(checksum) self.assertFalse(result) # Create key-pair values in the database kvps = get.key_value_pairs(records) pair.insert_rows(kvps) # Insert ingest_data.process_db_records(records) # Get data from database idx_datapoint = datapoint.checksum_exists(checksum) _dp = datapoint.DataPoint(idx_datapoint) ts_start = min(timestamps) ts_stop = max(timestamps) results = _dp.data(ts_start, ts_stop) # Test for index, result in enumerate(results): self.assertEqual(result['value'], expected[index]['value']) self.assertEqual(result['timestamp'], expected[index]['timestamp'])
def test_insert_rows(self): """Testing method / function insert_rows.""" # Initialize key variables polling_interval = 1 checksum = data.hashstring(str(random())) key = data.hashstring(str(random())) value = data.hashstring(str(random())) # Create a new Agent entry agent_id = data.hashstring(str(random())) agent_target = data.hashstring(str(random())) agent_program = data.hashstring(str(random())) agent.insert_row(agent_id, agent_target, agent_program) idx_agent = agent.exists(agent_id, agent_target) # Insert values in tables pair.insert_rows((key, value)) idx_pair = pair.pair_exists(key, value) datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent) idx_datapoint = datapoint.checksum_exists(checksum) # Create entry and check result = glue.glue_exists(idx_datapoint, idx_pair) self.assertFalse(result) glue.insert_rows(idx_datapoint, idx_pair) result = glue.glue_exists(idx_datapoint, idx_pair) self.assertTrue(bool(result)) self.assertTrue(isinstance(result, int))
def test_idx_pairs(self): """Testing method / function idx_pairs.""" # Initialize key variables checksum = data.hashstring(str(random())) polling_interval = 1 keypairs = [] idx_pairs = [] for _ in range(0, 10): time.sleep(0.05) key = data.hashstring(str(random())) value = data.hashstring(str(random())) keypairs.append((key, value)) # Create a new Agent entry agent_id = data.hashstring(str(random())) agent_target = data.hashstring(str(random())) agent_program = data.hashstring(str(random())) agent.insert_row(agent_id, agent_target, agent_program) idx_agent = agent.exists(agent_id, agent_target) # Insert values in tables pair.insert_rows(keypairs) datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent) idx_datapoint = datapoint.checksum_exists(checksum) # Test for key, value in keypairs: idx_pairs.append(pair.pair_exists(key, value)) glue.insert_rows(idx_datapoint, idx_pairs) result = glue.idx_pairs(idx_datapoint) self.assertEqual(len(result), len(idx_pairs)) for idx_pair in idx_pairs: self.assertTrue(idx_pair in result)
def test_insert_rows(self): """Testing method / function insert_rows.""" # Initialize key variables key = data.hashstring(str(random())) value = data.hashstring(str(random())) result = pair.pair_exists(key, value) self.assertFalse(result) # Create entry and check pair.insert_rows((key, value)) result = pair.pair_exists(key, value) self.assertTrue(bool(result)) self.assertTrue(isinstance(result, int))
def singleprocess_pairs(self): """Update rows in the Pair database table if necessary. Args: None Returns: None """ # Process data pairs = [] for item in self._arguments: row = item[0] per_process_key_value_pairs = get.key_value_pairs(row) pairs.append(per_process_key_value_pairs) # Insert pair.insert_rows(pairs)
def test_idx_pairs(self): """Testing method / function idx_pairs.""" # Initialize key variables keypairs = [] idx_pairs = [] for _ in range(0, 10): time.sleep(0.05) key = data.hashstring(str(random())) value = data.hashstring(str(random())) keypairs.append((key, value)) # Insert values in tables pair.insert_rows(keypairs) # Test for key, value in keypairs: idx_pairs.append(pair.pair_exists(key, value)) result = pair.idx_pairs(keypairs) self.assertEqual(len(result), len(idx_pairs)) for idx_pair in idx_pairs: self.assertTrue(idx_pair in result)
def pairs(pattoo_db_record): """Create db Pair table entries. Args: pattoo_db_record: PattooDBrecord object Returns: result: List of Pair.idx_pair database index values """ # Initialize key variables result = [] # Get key-values _pairs = key_value_pairs(pattoo_db_record) # Get list of pairs in the database pair.insert_rows(_pairs) result = pair.idx_pairs(_pairs) # Return return result
def test_checksums(self): """Testing method / function checksums.""" # Initialize key variables expected = {} polling_interval = 1 pattoo_agent_polled_target = 'panda_bear' pattoo_agent_program = 'koala_bear' pattoo_agent_hostname = 'grizzly_bear' # Insert an entry in the agent table agent_id = data.hashstring(str(random())) idx_agent = agent.idx_agent(agent_id, pattoo_agent_polled_target, pattoo_agent_program) # Populate database with key-value pairs for data_index in range(0, 10): time.sleep(0.1) # Add the checksum to the database checksum = data.hashstring(str(random())) datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent) idx_datapoint = datapoint.checksum_exists(checksum) # Define what we expect from the test function expected[checksum] = ChecksumLookup( idx_datapoint=idx_datapoint, polling_interval=polling_interval, last_timestamp=1) # Add key-pairs to the database record = PattooDBrecord( pattoo_checksum=checksum, pattoo_key='key', pattoo_agent_polling_interval=polling_interval, pattoo_agent_id=agent_id, pattoo_timestamp=int(time.time() * 1000), pattoo_data_type=DATA_FLOAT, pattoo_value=(data_index * 10), pattoo_agent_polled_target=pattoo_agent_polled_target, pattoo_agent_program=pattoo_agent_program, pattoo_agent_hostname=pattoo_agent_hostname, pattoo_metadata=[]) pairs = get.key_value_pairs(record) pair.insert_rows(pairs) idx_pairs = pair.idx_pairs(pairs) # Create glue entry glue.insert_rows(idx_datapoint, idx_pairs) ####################################################################### # This is added to verify that we only get a subset of results ####################################################################### # Insert an entry in the agent table fake_agent_id = data.hashstring(str(random())) idx_agent = agent.idx_agent(fake_agent_id, pattoo_agent_polled_target, pattoo_agent_program) # Populate database with key-value pairs for data_index in range(0, 17): time.sleep(0.1) # Add the checksum to the database checksum = data.hashstring(str(random())) datapoint.insert_row(checksum, DATA_FLOAT, polling_interval, idx_agent) idx_datapoint = datapoint.checksum_exists(checksum) # Add key-pairs to the database record = PattooDBrecord( pattoo_checksum=checksum, pattoo_key='key', pattoo_agent_polling_interval=polling_interval, pattoo_agent_id=fake_agent_id, pattoo_timestamp=int(time.time() * 1000), pattoo_data_type=DATA_FLOAT, pattoo_value=(data_index * 10), pattoo_agent_polled_target=pattoo_agent_polled_target, pattoo_agent_program=pattoo_agent_program, pattoo_agent_hostname=pattoo_agent_hostname, pattoo_metadata=[]) pairs = get.key_value_pairs(record) pair.insert_rows(pairs) idx_pairs = pair.idx_pairs(pairs) # Create glue entry glue.insert_rows(idx_datapoint, idx_pairs) # Test result = misc.agent_checksums(agent_id) self.assertTrue(bool(result)) self.assertTrue(isinstance(result, dict)) self.assertEqual(len(result), len(expected)) for key, value in result.items(): self.assertEqual(value.idx_datapoint, expected[key].idx_datapoint) self.assertEqual(value.polling_interval, expected[key].polling_interval) self.assertEqual(value.last_timestamp, expected[key].last_timestamp)