def test_chart_timestamp_args(self):
"""Testing function chart_timestamp_args."""
# Create a new Agent entry
_pi = 1000
agent_id = data.hashstring(str(random()))
agent_target = data.hashstring(str(random()))
agent_program = data.hashstring(str(random()))
agent.insert_row(agent_id, agent_target, agent_program)
idx_agent = agent.exists(agent_id, agent_target)
# Create entry and check
checksum = data.hashstring(str(random()))
result = datapoint.checksum_exists(checksum)
self.assertFalse(result)
datapoint.insert_row(checksum, DATA_FLOAT, _pi, idx_agent)
idx_datapoint = datapoint.checksum_exists(checksum)
# Test
values = [False, None]
for value in values:
now = normalized_timestamp(_pi, int(time.time() * 1000))
result = uri.chart_timestamp_args(idx_datapoint, value)
self.assertEqual(result + 604800000, now)
values = [-1, -6011, 1, 6011]
for value in values:
now = normalized_timestamp(_pi, int(time.time() * 1000))
result = uri.chart_timestamp_args(idx_datapoint, value)
self.assertEqual(result + (abs(value) * 1000), now)
values = ['foo', [None]]
for value in values:
now = normalized_timestamp(_pi, int(time.time() * 1000))
result = uri.chart_timestamp_args(idx_datapoint, value)
self.assertEqual(result + 604800000, now)
def data(self, ts_start, ts_stop):
"""Create list of dicts of counter values retrieved from database.
Args:
ts_start: Start time for query
ts_stop: Stop time for query
Returns:
result: List of key-value pair dicts
"""
# Initialize key variables
data_type = self.data_type()
_pi = self.polling_interval()
places = 10
result = []
# Return nothing if the DataPoint does not exist
if self.exists() is False:
return result
# Normalize timestamp to match the start of the nones array. If not,
# we could get the starting timestamp of the result to have a "None"
# value
ts_start = times.normalized_timestamp(_pi, timestamp=ts_start)
# Make sure we have entries for entire time range
timestamps = times.timestamps(ts_start, ts_stop, _pi)
nones = {_key: None for _key in timestamps}
# Get data from database
with db.db_query(20092) as session:
rows = session.query(Data.timestamp, Data.value).filter(
and_(Data.timestamp <= ts_stop, Data.timestamp >= ts_start,
Data.idx_datapoint == self._idx_datapoint)).order_by(
Data.timestamp).all()
# Put values into a dict for ease of processing
for row in rows:
# Find the first timestamp in the sorted list that is greater than
# that found in the database
timestamp = times.normalized_timestamp(_pi, row.timestamp)
rounded_value = round(float(row.value), places)
nones[timestamp] = rounded_value
if data_type in [DATA_INT, DATA_FLOAT]:
# Process non-counter values
result = _response(nones)
elif data_type in [DATA_COUNT64, DATA_COUNT] and len(rows) > 1:
# Process counter values by calculating the difference between
# successive values
result = _counters(nones, _pi, places)
return result
def chart_timestamp_args(idx_datapoint, secondsago=None):
"""Create URI arguments for charts.
Args:
idx_datapoint: DataPoint index value
secondsago: Number of seconds in the past to calculate start and
stop times for charts
Returns:
result: Starting time
"""
# Calculate stop. This takes into account the ingester cycle and subtracts
# a few extra seconds to prevent zero values at the end.
datapoint = DataPoint(idx_datapoint)
polling_interval = datapoint.polling_interval()
now = normalized_timestamp(polling_interval, int(time.time() * 1000))
# Calculate start
if bool(secondsago) is True and isinstance(secondsago, int) is True:
result = now - (abs(secondsago) * 1000)
else:
# result = ts_stop - (604800 * 1000)
result = now - (3600 * 24 * 7) * 1000
# Return
return result
def test_data(self):
"""Testing method / function data."""
# Initialize key variables
_data = []
expected = []
checksum = data.hashstring(str(random()))
pattoo_key = data.hashstring(str(random()))
agent_id = data.hashstring(str(random()))
polling_interval = 300 * 1000
data_type = DATA_FLOAT
_pattoo_value = 27
_timestamp = int(time.time() * 1000)
ts_start = _timestamp
for count in range(0, 10):
timestamp = _timestamp + (polling_interval * count)
ts_stop = timestamp
pattoo_value = _pattoo_value * count
insert = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=polling_interval,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=pattoo_value * count,
pattoo_agent_polled_target='pattoo_agent_polled_target',
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[])
# Create checksum entry in the DB, then update the data table
idx_datapoint = datapoint.idx_datapoint(insert)
_data.append(
IDXTimestampValue(idx_datapoint=idx_datapoint,
polling_interval=polling_interval,
timestamp=timestamp,
value=pattoo_value))
# Append to expected results
expected.append({
'timestamp':
times.normalized_timestamp(polling_interval, timestamp),
'value':
pattoo_value
})
# Insert rows of new data
lib_data.insert_rows(_data)
# Test
obj = DataPoint(idx_datapoint)
result = obj.data(ts_start, ts_stop)
self.assertEqual(result, expected)
def test_process_cache(self):
"""Testing method / function process_cache."""
# Initialize key variables
polling_interval = 20
_pi = polling_interval * 1000
_ = create_cache()
# Read data from directory
cache = Cache()
all_records = cache.records()
# Test
self.assertEqual(len(all_records), 1)
self.assertEqual(len(all_records[0]), 1)
pdbr = all_records[0][0]
# Datapoint should not exist before ingest
checksum = pdbr.pattoo_checksum
timestamp = pdbr.pattoo_timestamp
value = pdbr.pattoo_value
self.assertFalse(datapoint.checksum_exists(checksum))
# Ingest using process_cache
result = files_test.process_cache(fileage=0)
self.assertTrue(result)
# Test (checksum should exist)
idx_datapoint = datapoint.checksum_exists(checksum)
self.assertTrue(bool(idx_datapoint))
# Test (Single data entry should exist)
obj = datapoint.DataPoint(idx_datapoint)
result = obj.data(timestamp, timestamp)
self.assertEqual(len(result), 1)
key_pair = result[0]
self.assertEqual(key_pair['timestamp'],
times.normalized_timestamp(_pi, timestamp))
self.assertEqual(key_pair['value'], value)
def make_records():
"""Testing method / function process_db_records."""
# Initialize key variables
checksum = lib_data.hashstring(str(random()))
agent_id = lib_data.hashstring(str(random()))
data_type = DATA_FLOAT
_pi = 10 * 1000
pattoo_key = lib_data.hashstring(str(random()))
_timestamp = times.normalized_timestamp(_pi, int(time.time() * 1000))
expected = []
timestamps = []
records = []
result = {}
# Create a list of PattooDBrecord objects
for pattoo_value in range(0, 5):
timestamp = _timestamp + (pattoo_value * _pi)
expected.append({'timestamp': timestamp, 'value': pattoo_value})
timestamps.append(timestamp)
record = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=_pi,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=pattoo_value,
pattoo_agent_polled_target='pattoo_agent_polled_target',
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[])
records.append(record)
result['records'] = records
result['expected'] = expected
result['timestamps'] = timestamps
result['checksum'] = checksum
return result
def test_normalized_timestamp(self):
"""Testing function normalized_timestamp."""
# Initialize key variables
polling_interval = 300
timestamp = 31
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 0)
timestamp = 301
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 300)
timestamp = 900
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 900)
timestamp = 1000
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 900)
# Initialize key variables
polling_interval = 30
timestamp = 31
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 30)
timestamp = 301
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 300)
timestamp = 900
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 900)
timestamp = 1000
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 990)
# Test with boolean None values
timestamp = 900001
for polling_interval in [None, [], False, {}]:
result = times.normalized_timestamp(
polling_interval, timestamp=timestamp)
self.assertEqual(result, 900000)
# Test with non integer values
timestamp = 300
for polling_interval in ['1', [1]]:
with self.assertRaises(SystemExit):
times.normalized_timestamp(
polling_interval, timestamp=timestamp)
def test_route_data(self):
"""Testing method / function route_data."""
# Initialize key variables
secondsago = 3600
ts_start = uri.chart_timestamp_args(secondsago)
# Initialize key variables
_data = []
checksum = data.hashstring(str(random()))
pattoo_key = data.hashstring(str(random()))
agent_id = data.hashstring(str(random()))
_pi = 300 * 1000
data_type = DATA_FLOAT
now = int(time.time()) * 1000
count = 0
for timestamp in range(ts_start, now, _pi):
insert = PattooDBrecord(
pattoo_checksum=checksum,
pattoo_key=pattoo_key,
pattoo_agent_id=agent_id,
pattoo_agent_polling_interval=_pi,
pattoo_timestamp=timestamp,
pattoo_data_type=data_type,
pattoo_value=count,
pattoo_agent_polled_target='pattoo_agent_polled_target',
pattoo_agent_program='pattoo_agent_program',
pattoo_agent_hostname='pattoo_agent_hostname',
pattoo_metadata=[])
count += 1
# Create checksum entry in the DB, then update the data table
idx_datapoint = datapoint.idx_datapoint(insert)
_data.append(
IDXTimestampValue(idx_datapoint=idx_datapoint,
polling_interval=_pi,
timestamp=timestamp,
value=count))
# Insert rows of new data
lib_data.insert_rows(_data)
# Test
obj = DataPoint(idx_datapoint)
ts_stop = obj.last_timestamp()
expected = obj.data(ts_start, ts_stop)
# Create URL
config = WebConfig()
url = ('{}/{}'.format(config.web_api_server_url(graphql=False),
idx_datapoint))
# Check response
with requests.get(url) as response:
result = response.json()
count = 0
for item in result:
ts_norm = times.normalized_timestamp(_pi, ts_start)
if item['timestamp'] < ts_norm:
self.assertIsNone(item['value'])
else:
self.assertEqual(item, expected[count])
count += 1