def test_idx_datapoint(self):
"""Testing method / function idx_datapoint."""
# Initialize key variables
checksum = data.hashstring(str(random()))
pattoo_db_record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_metadata=[('key', 'value')],
pattoo_data_type=32,
pattoo_key='polar_bear',
pattoo_value=0.0,
pattoo_timestamp=1575789070108,
pattoo_agent_polled_target='panda_bear',
pattoo_agent_program='koala_bear',
pattoo_agent_hostname='grizzly_bear',
pattoo_agent_id='red_stripe_beer',
pattoo_agent_polling_interval=10000)
# Checksum should not exist
self.assertFalse(datapoint.checksum_exists(checksum))
# Test creation
result = datapoint.idx_datapoint(pattoo_db_record)
expected = datapoint.checksum_exists(checksum)
self.assertEqual(result, expected)
# Test after creation
result = datapoint.idx_datapoint(pattoo_db_record)
expected = datapoint.checksum_exists(checksum)
self.assertEqual(result, expected)
def test_data(self):
"""Testing method / function data."""
# Initialize key variables
_data = []
expected = []
checksum = data.hashstring(str(random()))
pattoo_key = data.hashstring(str(random()))
agent_id = data.hashstring(str(random()))
polling_interval = 300 * 1000
data_type = DATA_FLOAT
_pattoo_value = 27
_timestamp = int(time.time() * 1000)
ts_start = _timestamp
for count in range(0, 10):
timestamp = _timestamp + (polling_interval * count)
ts_stop = timestamp
pattoo_value = _pattoo_value * count
insert = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=polling_interval,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=pattoo_value * count,
pattoo_agent_polled_target='pattoo_agent_polled_target',
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[])
# Create checksum entry in the DB, then update the data table
idx_datapoint = datapoint.idx_datapoint(insert)
_data.append(
IDXTimestampValue(idx_datapoint=idx_datapoint,
polling_interval=polling_interval,
timestamp=timestamp,
value=pattoo_value))
# Append to expected results
expected.append({
'timestamp':
times.normalized_timestamp(polling_interval, timestamp),
'value':
pattoo_value
})
# Insert rows of new data
lib_data.insert_rows(_data)
# Test
obj = DataPoint(idx_datapoint)
result = obj.data(ts_start, ts_stop)
self.assertEqual(result, expected)
def test_route_graphql(self):
"""Testing method / function add_url_rule (graphql)."""
# Initialize key variables
_data = []
pattoo_checksum = data.hashstring(str(random()))
pattoo_key = data.hashstring(str(random()))
agent_id = data.hashstring(str(random()))
pattoo_agent_polled_target = data.hashstring(str(random()))
_pi = 300 * 1000
data_type = DATA_FLOAT
timestamp = int(time.time()) * 1000
pattoo_value = round(uniform(1, 100), 5)
insert = PattooDBrecord(
pattoo_checksum=pattoo_checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=_pi,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=pattoo_value,
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[])
# Create checksum entry in the DB, then update the data table
idx_datapoint = datapoint.idx_datapoint(insert)
_data.append(
IDXTimestampValue(idx_datapoint=idx_datapoint,
polling_interval=_pi,
timestamp=timestamp,
value=pattoo_value))
# Insert rows of new data
lib_data.insert_rows(_data)
# Test
query = ('''\
{
allDatapoints(idxDatapoint: "IDX") {
edges {
node {
checksum
}
}
}
}
'''.replace('IDX', str(idx_datapoint)))
# Test
graphql_result = _get(query)
result = graphql_result['data']['allDatapoints']['edges'][0]['node']
self.assertEqual(result['checksum'], pattoo_checksum)
def test_insert_rows(self):
"""Testing method / function insert_rows."""
# Initialize key variables
checksum = lib_data.hashstring(str(random()))
agent_id = lib_data.hashstring(str(random()))
data_type = DATA_FLOAT
polling_interval = 10
pattoo_value = 27
pattoo_key = lib_data.hashstring(str(random()))
timestamp = int(time.time() * 1000)
insert = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=polling_interval,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=pattoo_value,
pattoo_agent_polled_target='pattoo_agent_polled_target',
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[]
)
# Create checksum entry in the DB, then update the data table
idx_datapoint = datapoint.idx_datapoint(insert)
_data = [IDXTimestampValue(
idx_datapoint=idx_datapoint,
polling_interval=polling_interval,
timestamp=timestamp,
value=pattoo_value)]
data.insert_rows(_data)
# Verify that the data is there
with db.db_query(20015) as session:
rows = session.query(
Data.value).filter(and_(
Data.idx_datapoint == idx_datapoint,
Data.timestamp == timestamp))
for row in rows:
self.assertEqual(row.value, pattoo_value)
def test_route_data(self):
"""Testing method / function route_data."""
# Initialize key variables
secondsago = 3600
ts_start = uri.chart_timestamp_args(secondsago)
# Initialize key variables
_data = []
checksum = data.hashstring(str(random()))
pattoo_key = data.hashstring(str(random()))
agent_id = data.hashstring(str(random()))
_pi = 300 * 1000
data_type = DATA_FLOAT
now = int(time.time()) * 1000
count = 0
for timestamp in range(ts_start, now, _pi):
insert = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=_pi,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=count,
pattoo_agent_polled_target='pattoo_agent_polled_target',
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[])
count += 1
# Create checksum entry in the DB, then update the data table
idx_datapoint = datapoint.idx_datapoint(insert)
_data.append(
IDXTimestampValue(idx_datapoint=idx_datapoint,
polling_interval=_pi,
timestamp=timestamp,
value=count))
# Insert rows of new data
lib_data.insert_rows(_data)
# Test
obj = DataPoint(idx_datapoint)
ts_stop = obj.last_timestamp()
expected = obj.data(ts_start, ts_stop)
# Create URL
config = WebConfig()
url = ('{}/{}'.format(config.web_api_server_url(graphql=False),
idx_datapoint))
# Check response
with requests.get(url) as response:
result = response.json()
count = 0
for item in result:
ts_norm = times.normalized_timestamp(_pi, ts_start)
if item['timestamp'] < ts_norm:
self.assertIsNone(item['value'])
else:
self.assertEqual(item, expected[count])
count += 1
def process_db_records(pattoo_db_records):
"""Insert all data values for an agent into database.
Args:
pattoo_db_records: List of dicts read from cache files.
Returns:
None
Method:
1) Get all the idx_datapoint and idx_pair values that exist in the
PattooDBrecord data from the database. All the records MUST be
from the same source.
2) Add these idx values to tracking memory variables for speedy lookup
3) Ignore non numeric data values sent
4) Add data to the database. If new checksum values are found in the
PattooDBrecord data, then create the new index values to the
database, update the tracking memory variables before hand.
"""
# Initialize key variables
_data = {}
# Return if there is nothint to process
if bool(pattoo_db_records) is False:
return
# Get DataPoint.idx_datapoint and idx_pair values from db. This is used to
# speed up the process by reducing the need for future database access.
agent_id = pattoo_db_records[0].pattoo_agent_id
checksum_table = misc.agent_checksums(agent_id)
# Process data
for pdbr in pattoo_db_records:
# We only want to insert non-string, non-None values
if pdbr.pattoo_data_type in [DATA_NONE, DATA_STRING]:
continue
# Try to make the value a float for insertion into the database
try:
float_value = float(pdbr.pattoo_value)
except:
continue
# Get the idx_datapoint value for the PattooDBrecord
if pdbr.pattoo_checksum in checksum_table:
# Get last_timestamp for existing idx_datapoint entry
idx_datapoint = checksum_table[pdbr.pattoo_checksum].idx_datapoint
else:
# Entry not in database. Update the database and get the
# required idx_datapoint
idx_datapoint = datapoint.idx_datapoint(pdbr)
if bool(idx_datapoint) is True:
# Update the lookup table
checksum_table[pdbr.pattoo_checksum] = ChecksumLookup(
idx_datapoint=idx_datapoint,
polling_interval=int(pdbr.pattoo_agent_polling_interval),
last_timestamp=1)
# Update the Glue table
idx_pairs = get.pairs(pdbr)
glue.insert_rows(idx_datapoint, idx_pairs)
else:
continue
# Append item to items
if pdbr.pattoo_timestamp > checksum_table[
pdbr.pattoo_checksum].last_timestamp:
'''
Add the Data table results to a dict in case we have duplicate
posting over the API. We need to key off a unique time dependent
value per datapoint to prevent different datapoints at the same
point in time overwriting the value. This is specifically for
removing duplicates for the _SAME_ datapoint at the same point in
time as could possibly occur with the restart of an agent causing a
double posting or network issues. We therefore use a tuple of
idx_datapoint and timestamp.
'''
_data[(pdbr.pattoo_timestamp, idx_datapoint)] = IDXTimestampValue(
idx_datapoint=idx_datapoint,
polling_interval=int(pdbr.pattoo_agent_polling_interval),
timestamp=pdbr.pattoo_timestamp,
value=float_value)
# Update the data table
if bool(_data) is True:
data.insert_rows(list(_data.values()))
# Log message
log_message = ('''\
Finished cache data processing for agent_id: {}'''.format(agent_id))
log.log2debug(20113, log_message)
def test_route_graphql(self):
"""Testing method / function add_url_rule (graphql)."""
# Initialize key variables
_data = []
pattoo_checksum = data.hashstring(str(random()))
pattoo_key = data.hashstring(str(random()))
agent_id = data.hashstring(str(random()))
pattoo_agent_polled_target = data.hashstring(str(random()))
_pi = 300 * 1000
data_type = DATA_FLOAT
timestamp = int(time.time()) * 1000
pattoo_value = round(uniform(1, 100), 5)
insert = PattooDBrecord(
pattoo_checksum=pattoo_checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=_pi,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=pattoo_value,
pattoo_agent_polled_target=pattoo_agent_polled_target,
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[])
# Create checksum entry in the DB, then update the data table
idx_datapoint = datapoint.idx_datapoint(insert)
_data.append(
IDXTimestampValue(idx_datapoint=idx_datapoint,
polling_interval=_pi,
timestamp=timestamp,
value=pattoo_value))
# Insert rows of new data
lib_data.insert_rows(_data)
# Creating required test admin
test_admin = {
"username": "******",
"first_name": "Pattoo Test",
"last_name": "Pattoo Test",
"password": "******",
"role": 0,
"password_expired": 0,
"enabled": 1
}
user.insert_row(DbRowUser(**test_admin))
# Get accesss token to make test queries
acesss_query = ('''\
mutation{
authenticate(Input: {username: "******", password: "******"}) {
accessToken
refreshToken
}
}
''')
# Replacing username and password in access_query
acesss_query = acesss_query.replace("USERNAME", test_admin['username'])
acesss_query = acesss_query.replace("PASSWORD", test_admin['password'])
access_request = _query(acesss_query, query_type="Mutation")
acesss_token = access_request['data']['authenticate']['accessToken']
# Test
query = ('''\
{
allDatapoints(idxDatapoint: "IDX", token: "TOKEN") {
edges {
node {
checksum
}
}
}
}
''')
# Replacing IDX and TOKEN in query
query = query.replace("IDX", str(idx_datapoint))
query = query.replace("TOKEN", str(acesss_token))
# Test
graphql_result = _query(query)
result = graphql_result['data']['allDatapoints']['edges'][0]['node']
self.assertEqual(result['checksum'], pattoo_checksum)
