def _make_agent_data():
"""Create generate data to post to API server"""
# Initialize key variables
config = Config()
polling_interval = 60
pattoo_agent_program = 1
pattoo_agent_polled_target = 2
pattoo_key = '3'
pattoo_value = 4
# We want to make sure we get a different AgentID each time
filename = files.agent_id_file(pattoo_agent_program, config)
if os.path.isfile(filename) is True:
os.remove(filename)
# Setup AgentPolledData
apd = AgentPolledData(pattoo_agent_program, polling_interval)
# Initialize TargetDataPoints
ddv = TargetDataPoints(pattoo_agent_polled_target)
# Setup DataPoint
data_type = DATA_INT
variable = DataPoint(pattoo_key, pattoo_value, data_type=data_type)
# Add data to TargetDataPoints
ddv.add(variable)
# Create a result
apd.add(ddv)
# Return agent data
return apd
def test_agentdata_to_post(self):
"""Testing method or function named agentdata_to_post."""
# Setup AgentPolledData
agent_program = 'panda_bear'
polling_interval = 20
apd = AgentPolledData(agent_program, polling_interval)
# Initialize TargetDataPoints
target = 'teddy_bear'
ddv = TargetDataPoints(target)
# Setup DataPoint
value = 457
key = 'gummy_bear'
data_type = DATA_INT
variable = DataPoint(key, value, data_type=data_type)
# Add data to TargetDataPoints
ddv.add(variable)
# Test TargetGateway to AgentPolledData
apd.add(ddv)
result = converter.agentdata_to_post(apd)
self.assertEqual(result.pattoo_agent_id, apd.agent_id)
self.assertEqual(
result.pattoo_agent_polling_interval, polling_interval * 1000)
self.assertTrue(isinstance(result.pattoo_datapoints, dict))
# Test the key value pairs
item = result.pattoo_datapoints['key_value_pairs']
self.assertTrue('datapoint_pairs' in result.pattoo_datapoints)
self.assertTrue('key_value_pairs' in result.pattoo_datapoints)
self.assertTrue(isinstance(item, dict))
# Convert item to a list of tuples for ease of testing
tester = [(k, v) for k, v in sorted(item.items())]
self.assertEqual(
tester[0],
(0, ('pattoo_agent_polling_interval', '20000')))
self.assertEqual(
tester[3:8],
[
(3, ('pattoo_agent_polled_target', 'teddy_bear')),
(4, ('pattoo_agent_program', 'panda_bear')),
(5, ('pattoo_key', 'gummy_bear')),
(6, ('pattoo_data_type', 99)),
(7, ('pattoo_value', 457))]
)
# Test the pointers to the key value pairs
item = result.pattoo_datapoints['datapoint_pairs']
self.assertTrue(isinstance(item, list))
self.assertEqual(len(item), 1)
self.assertEqual(len(item[0]), 10)
def _get_target_datapoints(self, item):
"""Poll each spoke in parallel.
Args:
item: TargetPollingPoints object
Returns:
ddv: TargetDataPoints for the SNMPVariable target
"""
# Intialize data gathering
ip_target = item.target
ddv = TargetDataPoints(ip_target)
# BAC0 only works with IP addresses
ip_address = network.get_ipaddress(ip_target)
if bool(ip_address) is False:
return ddv
# Get list of type DataPoint
datapoints = []
for polltarget in item.data:
# Get polling results
value = poll_target_address(ip_address, polltarget.address,
'presentValue', self._bacnet)
name = poll_target_address(ip_address, polltarget.address,
'objectName', self._bacnet)
# Skip if invalid data is received
if value is None:
continue
# Do multiplication
if data.is_numeric(value) is True:
value = float(value) * polltarget.multiplier
data_type = DATA_FLOAT
else:
data_type = DATA_STRING
# Update datapoints
datapoint = DataPoint('analog_value_point_{}'.format(
polltarget.address),
value,
data_type=data_type)
datapoint.add(DataPointMetadata('target', ip_target))
if name is not None:
datapoint.add(DataPointMetadata('object_name', name))
datapoints.append(datapoint)
# Return
ddv.add(datapoints)
return ddv
def test_add(self):
"""Testing function append."""
# Setup AgentPolledData
agent_program = 'panda_bear'
polling_interval = 20
apd = AgentPolledData(agent_program, polling_interval)
# Initialize TargetDataPoints
target = 'teddy_bear'
ddv = TargetDataPoints(target)
self.assertEqual(ddv.target, target)
self.assertFalse(ddv.valid)
self.assertEqual(ddv.data, [])
# Setup DataPoint
value = 457
_key_ = 'gummy_bear'
data_type = DATA_INT
variable = DataPoint(_key_, value, data_type=data_type)
# Add data to TargetDataPoints
self.assertFalse(ddv.valid)
ddv.add(variable)
self.assertTrue(ddv.valid)
# Test add
self.assertFalse(apd.valid)
apd.add(None)
self.assertFalse(apd.valid)
apd.add(variable)
self.assertFalse(apd.valid)
apd.add(ddv)
self.assertTrue(apd.valid)
# Test contents
data = apd.data
self.assertTrue(isinstance(data, list))
self.assertEqual(len(data), 1)
_ddv = data[0]
self.assertTrue(isinstance(_ddv, TargetDataPoints))
self.assertEqual(_ddv.target, target)
self.assertTrue(_ddv.valid)
self.assertTrue(isinstance(_ddv.data, list))
self.assertTrue(len(_ddv.data), 1)
data = _ddv.data
_variable = _ddv.data[0]
self.assertEqual(_variable.data_type, data_type)
self.assertEqual(_variable.value, value)
self.assertEqual(_variable.key, _key_)
def test_agent():
# Define the polling interval in seconds (integer).
polling_interval = 300
# Give the agent a name
agent_name = 'sample_agent_script'
# Let's assume the script has already received this data from SITE_A
site_a_data = [['ABC', 123.456], ['DEF', 456.789]]
# Let's assume the script has already received this data from WORK_1
work_1_data = [['GHI', 654.321], ['JKL', 987.654]]
# Setup the agent's AgentPolledData object.
agent = AgentPolledData(agent_name, polling_interval)
# Let's add some metadata that you don't want to affect charting in the
# event of a change. Department names change all the time.
metadata_static = DataPointMetadata('Department Name',
'The Palisadoes Foundation',
update_checksum=False)
# Let's add some metadata that will change and trigger a new chart.
metadata_dynamic = DataPointMetadata('Financial Year', '2020')
# Create target objects for SITE_A
target = TargetDataPoints('SITE_A')
for quote in site_a_data:
key, value = quote
datapoint = DataPoint(key, value, data_type=DATA_FLOAT)
datapoint.add(metadata_static)
datapoint.add(metadata_dynamic)
target.add(datapoint)
agent.add(target)
# Create target objects for WORK_1
target = TargetDataPoints('WORK_1')
for quote in work_1_data:
key, value = quote
datapoint = DataPoint(key, value, data_type=DATA_FLOAT)
datapoint.add(metadata_static)
datapoint.add(metadata_dynamic)
target.add(datapoint)
agent.add(target)
# Return agent
return agent
def create_cache():
"""Testing method / function records."""
# Initialize key variables
config = ServerConfig()
polling_interval = 20
cache_directory = config.agent_cache_directory(PATTOO_API_AGENT_NAME)
result = {
'pattoo_agent_program': data.hashstring(str(random())),
'pattoo_agent_polled_target': socket.getfqdn(),
'pattoo_key': data.hashstring(str(random())),
'pattoo_value': round(uniform(1, 100), 5),
'pattoo_agent_hostname': socket.getfqdn()
}
# We want to make sure we get a different AgentID each time
filename = files.agent_id_file(
result['pattoo_agent_program'],
config)
if os.path.isfile(filename) is True:
os.remove(filename)
result['pattoo_agent_id'] = files.get_agent_id(
result['pattoo_agent_program'],
config)
# Setup AgentPolledData
apd = AgentPolledData(result['pattoo_agent_program'], polling_interval)
# Initialize TargetDataPoints
ddv = TargetDataPoints(result['pattoo_agent_hostname'])
# Setup DataPoint
data_type = DATA_INT
variable = DataPoint(
result['pattoo_key'], result['pattoo_value'], data_type=data_type)
# Add data to TargetDataPoints
ddv.add(variable)
# Write data to cache
apd.add(ddv)
cache_dict = converter.posting_data_points(
converter.agentdata_to_post(apd))
cache_file = '{}{}cache_test.json'.format(cache_directory, os.sep)
with open(cache_file, 'w') as _fp:
json.dump(cache_dict, _fp)
return result
def test_agentdata_to_datapoints(self):
"""Testing method or function named agentdata_to_datapoints."""
# Setup AgentPolledData
agent_program = 'panda_bear'
polling_interval = 20
apd = AgentPolledData(agent_program, polling_interval)
# Initialize TargetDataPoints
target = 'teddy_bear'
ddv = TargetDataPoints(target)
# Setup DataPoint
value = 457
key = 'gummy_bear'
data_type = DATA_INT
variable = DataPoint(key, value, data_type=data_type)
# Add data to TargetDataPoints
ddv.add(variable)
# Test TargetGateway to AgentPolledData
apd.add(ddv)
# Test contents
expected_metadata = {
'pattoo_agent_id': apd.agent_id,
'pattoo_agent_program': agent_program,
'pattoo_agent_hostname': apd.agent_hostname,
'pattoo_agent_polled_target': target,
'pattoo_agent_polling_interval': apd.agent_polling_interval
}
result = converter.agentdata_to_datapoints(apd)
self.assertEqual(len(result), 1)
item = result[0]
self.assertTrue(isinstance(item, DataPoint))
self.assertEqual(item.value, value)
self.assertEqual(item.data_type, DATA_INT)
self.assertEqual(item.key, key)
self.assertTrue(isinstance(item.metadata, dict))
self.assertEqual(len(item.metadata), len(expected_metadata))
for key, value in item.metadata.items():
self.assertTrue(isinstance(value, str))
self.assertTrue(isinstance(key, str))
self.assertEqual(value, str(expected_metadata[key]))
def _walker(snmpvariable, polltargets):
"""Poll each spoke in parallel.
Args:
snmpvariable: SNMPVariable to poll
polltargets: List of PollingPoint objects to poll
Returns:
ddv: TargetDataPoints for the SNMPVariable target
"""
# Intialize data gathering
ddv = TargetDataPoints(snmpvariable.ip_target)
query = Query(snmpvariable)
results = query.everything()
datapoints = _create_datapoints(results)
ddv.add(datapoints)
return ddv
def poll(agent_program, polling_interval):
"""Get all agent data.
Performance data on linux server on which this application is installed.
Args:
agentdata: AgentPolledData object for all data gathered by the agent
polling_interval: Polling interval in seconds
Returns:
None
"""
# Initialize AgentPolledData
agent_hostname = socket.getfqdn()
agentdata = AgentPolledData(agent_program, polling_interval)
# Intialize data gathering
ddv = TargetDataPoints(agent_hostname)
#########################################################################
# Get timeseries values
#########################################################################
performance = Performance()
# Update agent with system data
ddv.add(performance.stats_system())
# Update agent with disk data
ddv.add(performance.stats_disk_swap())
ddv.add(performance.stats_disk_partitions())
ddv.add(performance.stats_disk_io())
# Update agent with network data
ddv.add(performance.stats_network())
# Add results to the AgentPolledData object for posting
agentdata.add(ddv)
return agentdata
def _walker(snmpvariable, polltargets):
"""Poll each spoke in parallel.
Args:
snmpvariable: SNMPVariable to poll
polltargets: List of PollingPoint objects to poll
Returns:
ddv: TargetDataPoints for the SNMPVariable target
"""
# Intialize data gathering
ddv = TargetDataPoints(snmpvariable.ip_target)
# Get list of type DataPoint
datapoints = []
for polltarget in polltargets:
# Get OID polling results
query = snmp.SNMP(snmpvariable)
query_datapoints = query.walk(polltarget.address)
# Apply multiplier to the results
for _dp in query_datapoints:
# Do multiplication
if data.is_data_type_numeric(_dp.data_type) is True:
value = float(_dp.value) * polltarget.multiplier
else:
value = _dp.value
# Update datapoints
datapoint = DataPoint(polltarget.address,
value,
data_type=_dp.data_type)
datapoint.add(DataPointMetadata('oid', _dp.key))
datapoints.append(datapoint)
# Return
ddv.add(datapoints)
return ddv
def test_add(self):
"""Testing function append."""
# Initialize TargetDataPoints
target = 'teddy_bear'
ddv = TargetDataPoints(target)
self.assertEqual(ddv.target, target)
self.assertFalse(ddv.valid)
self.assertEqual(ddv.data, [])
# Setup DataPoint
value = 457
_key_ = 'gummy_bear'
data_type = DATA_INT
variable = DataPoint(_key_, value, data_type=data_type)
# Test adding invalid value
ddv.add(None)
self.assertEqual(ddv.data, [])
# Test adding variable
ddv.add(variable)
self.assertTrue(bool(ddv.data))
self.assertTrue(isinstance(ddv.data, list))
self.assertEqual(len(ddv.data), 1)
checksum = ddv.data[0].checksum
# Test adding duplicate variable (There should be no changes)
ddv.add(variable)
self.assertTrue(bool(ddv.data))
self.assertTrue(isinstance(ddv.data, list))
self.assertEqual(len(ddv.data), 1)
self.assertEqual(checksum, ddv.data[0].checksum)
# Test the values in the variable
_variable = ddv.data[0]
self.assertEqual(_variable.data_type, data_type)
self.assertEqual(_variable.value, value)
self.assertEqual(_variable.key, _key_)
def main():
"""Post data to pattoo server.
Args:
None
Returns:
None
"""
'''
NOTE:
Scripts must be run at regular intervals and the polling_interval
should be automatically provided to the main() function.
Notes about CRON:
When using cron, change this value to match the cron interval in seconds.
It is not advised to use cron for polling unless you know the interval
will not change. If using cron, remember to make the polling interval to
match the cron interval in 'seconds'.
Ideally your agents should run as daemons, not as cron jobs. See the daemon
example script which explains how to do this.
'''
# Define the polling interval in seconds (integer).
polling_interval = 300
# Give the agent a name
agent_name = 'sample_agent_script'
# Let's assume the script has already received this data from SITE_A
site_a_data = [['ABC', 123.456], ['DEF', 456.789]]
# Let's assume the script has already received this data from WORK_1
work_1_data = [['GHI', 654.321], ['JKL', 987.654]]
'''
NOTE:
The AgentPolledData object contains unique identification information
that the pattoo server will use to differentiate the information source.
This includes: the hostname of the system on which the object was created,
a unique hashed identifier stored in the cache directory of the agent
configuration.
You just need to write an agent to do one thing well, such data collection
from a type of target source. For example, you don't need to give each
agent that does the same thing a different "agent_name". Just make sure
that different types of agents have different "agent_name" values.
The PattooShared library will take care of the rest.
'''
# Setup the agent's AgentPolledData object.
agent = AgentPolledData(agent_name, polling_interval)
'''
NOTE:
Metadata is normally expected to stay constant. If it changes then the
datapoint ID changes and you'll start plotting a brand new chart on the
pattoo server. The old chart will stop at the time the metadata changed.
In some cases, you may not want changing metadata to cause a brand new
plot. For example, your metadata for computer resource charting may include
the operating system version. This is useful background information, but
shouldn't impact charting if it changes. This type of metadata should be
dynamic.
'''
# Let's add some metadata that you don't want to affect charting in the
# event of a change. Department names change all the time.
metadata_static = DataPointMetadata('Department Name',
'The Palisadoes Foundation',
update_checksum=False)
# Let's add some metadata that will change and trigger a new chart.
metadata_dynamic = DataPointMetadata('Financial Year', '2020')
# Create target objects for SITE_A
target = TargetDataPoints('SITE_A')
for quote in site_a_data:
key, value = quote
'''
NOTE:
You don't have to specify the time when the data was collected.
The DataPoint object captures that information automatically. You can
also specify it using the timestamp= argument. See the class
documentation for details.
The default data_type is DATA_INT (integer). Read the documentation
for the various other data which cover float and counter values.
'''
datapoint = DataPoint(key, value, data_type=DATA_FLOAT)
datapoint.add(metadata_static)
datapoint.add(metadata_dynamic)
target.add(datapoint)
agent.add(target)
# Create target objects for WORK_1
target = TargetDataPoints('WORK_1')
for quote in work_1_data:
key, value = quote
datapoint = DataPoint(key, value, data_type=DATA_FLOAT)
datapoint.add(metadata_static)
datapoint.add(metadata_dynamic)
target.add(datapoint)
agent.add(target)
# Post the data to pattoo
post = PostAgent(agent)
post.post()
async def _serial_poller_async(tpp):
"""Poll OPCUA agent data.
Args:
tpp: TargetDataPoints object
Returns:
target_datapoints: TargetDataPoints object
"""
# Initialize key variables
connected = False
# Test for validity
if isinstance(tpp, TargetPollingPoints) is False:
return None
if isinstance(tpp.target, OPCUAauth) is False:
return None
if tpp.valid is False:
return None
# Create URL for polling
ip_target = tpp.target.ip_target
ip_port = tpp.target.ip_port
username = tpp.target.username
password = tpp.target.password
url = 'opc.tcp://{}:{}'.format(ip_target, ip_port)
# Intialize data gathering
target_datapoints = TargetDataPoints(ip_target)
# Create a client object to connect to OPCUA server
client = Client(url=url)
client.set_user(username)
client.set_password(password)
# Connect
try:
await client.connect()
connected = True
except:
log_message = (
'Authentication for polling target {} is incorrect'.format(url))
log.log2warning(51011, log_message)
pass
if connected is True:
for point in tpp.data:
# Make sure we have the right data type
if isinstance(point, PollingPoint) is False:
log_message = ('''\
Invalid polling point {} for OPC UA URL {}'''.format(point, url))
log.log2info(51012, log_message)
continue
# Get data
address = point.address
try:
node = client.get_node(address)
value = await node.read_value()
except BadNodeIdUnknown:
log_message = ('''\
OPC UA node {} not found on server {}'''.format(address, url))
log.log2warning(51015, log_message)
continue
except:
_exception = sys.exc_info()
log_message = ('OPC UA server communication error')
log.log2exception(51014, _exception, message=log_message)
log_message = ('''\
Cannot get value from polling point {} for OPC UA URL {}\
'''.format(address, url))
log.log2info(51013, log_message)
continue
# Create datapoint
if bool(point.multiplier) is True:
if is_numeric(value) is True and (is_numeric(point.multiplier)
is True):
value = value * point.multiplier
else:
value = 0
datapoint = DataPoint(address, value)
datapoint.add(DataPointMetadata('OPCUA Server', ip_target))
target_datapoints.add(datapoint)
# Disconnect client
await client.disconnect()
return target_datapoints
def _serial_poller(drv):
"""Poll each spoke in parallel.
Args:
drv: Target to poll
input_registers: Input registers to poll
holding_registers: Holding registers to poll
Returns:
ddv: TargetDataPoints for the ip_target
"""
# Intialize data gathering
ip_target = drv.target
ddv = TargetDataPoints(ip_target)
# Get list of type DataPoint
datapoints = []
for _rv in drv.data:
# Ignore invalid data
if isinstance(_rv, RegisterVariable) is False:
continue
if _rv.valid is False:
continue
# Poll
client = ModbusTcpClient(ip_target)
if isinstance(_rv, InputRegisterVariable):
try:
response = client.read_input_registers(_rv.address,
count=_rv.count,
unit=_rv.unit)
key = 'input_register'
except ConnectionException as _err:
log_message = ('''\
Cannot connect to target {} to retrieve input register {}, count {}, \
unit {}: {}'''.format(ip_target, _rv.register, _rv.count, _rv.unit, str(_err)))
log.log2warning(65028, log_message)
continue
except:
log_message = ('''\
Cause unknown failure with target {} getting input register {}, count {}, \
unit {}'''.format(ip_target, _rv.register, _rv.count, _rv.unit))
log.log2warning(65030, log_message)
continue
elif isinstance(_rv, HoldingRegisterVariable):
try:
response = client.read_holding_registers(_rv.address)
key = 'holding_register'
except ConnectionException:
log_message = ('''\
Cannot connect to target {} to retrieve input register {}, count {}, \
unit {}'''.format(ip_target, _rv.register, _rv.count, _rv.unit))
log.log2warning(65032, log_message)
continue
except:
log_message = ('''\
Cause unknown failure with target {} getting holding register {}, count {}, \
unit {}. [{}, {}, {}]\
'''.format(ip_target, _rv.register, _rv.count, _rv.unit,
sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2]))
log.log2warning(65031, log_message)
continue
# Process data
if response.isError() is True:
_log_modbus(ip_target, _rv, response)
else:
values = response.registers
for data_index, _value in enumerate(values):
# Do multiplication
value = _value * _rv.multiplier
# Create DataPoint and append
new_key = ('{}_{}'.format(key, _rv.register + data_index))
datapoint = DataPoint(new_key, value, data_type=DATA_INT)
datapoint.add(DataPointMetadata('unit',
str(_rv.unit).zfill(3)))
datapoints.append(datapoint)
ddv.add(datapoints)
# Return
return ddv