def _make_agent_data(): """Create generate data to post to API server""" # Initialize key variables config = Config() polling_interval = 60 pattoo_agent_program = 1 pattoo_agent_polled_target = 2 pattoo_key = '3' pattoo_value = 4 # We want to make sure we get a different AgentID each time filename = files.agent_id_file(pattoo_agent_program, config) if os.path.isfile(filename) is True: os.remove(filename) # Setup AgentPolledData apd = AgentPolledData(pattoo_agent_program, polling_interval) # Initialize TargetDataPoints ddv = TargetDataPoints(pattoo_agent_polled_target) # Setup DataPoint data_type = DATA_INT variable = DataPoint(pattoo_key, pattoo_value, data_type=data_type) # Add data to TargetDataPoints ddv.add(variable) # Create a result apd.add(ddv) # Return agent data return apd
def test_agentdata_to_post(self): """Testing method or function named agentdata_to_post.""" # Setup AgentPolledData agent_program = 'panda_bear' polling_interval = 20 apd = AgentPolledData(agent_program, polling_interval) # Initialize TargetDataPoints target = 'teddy_bear' ddv = TargetDataPoints(target) # Setup DataPoint value = 457 key = 'gummy_bear' data_type = DATA_INT variable = DataPoint(key, value, data_type=data_type) # Add data to TargetDataPoints ddv.add(variable) # Test TargetGateway to AgentPolledData apd.add(ddv) result = converter.agentdata_to_post(apd) self.assertEqual(result.pattoo_agent_id, apd.agent_id) self.assertEqual( result.pattoo_agent_polling_interval, polling_interval * 1000) self.assertTrue(isinstance(result.pattoo_datapoints, dict)) # Test the key value pairs item = result.pattoo_datapoints['key_value_pairs'] self.assertTrue('datapoint_pairs' in result.pattoo_datapoints) self.assertTrue('key_value_pairs' in result.pattoo_datapoints) self.assertTrue(isinstance(item, dict)) # Convert item to a list of tuples for ease of testing tester = [(k, v) for k, v in sorted(item.items())] self.assertEqual( tester[0], (0, ('pattoo_agent_polling_interval', '20000'))) self.assertEqual( tester[3:8], [ (3, ('pattoo_agent_polled_target', 'teddy_bear')), (4, ('pattoo_agent_program', 'panda_bear')), (5, ('pattoo_key', 'gummy_bear')), (6, ('pattoo_data_type', 99)), (7, ('pattoo_value', 457))] ) # Test the pointers to the key value pairs item = result.pattoo_datapoints['datapoint_pairs'] self.assertTrue(isinstance(item, list)) self.assertEqual(len(item), 1) self.assertEqual(len(item[0]), 10)
def test___repr__(self): """Testing function __repr__.""" # Setup TargetDataPoints target = 'localhost' ddv = TargetDataPoints(target) # Test expected = ('''\ <TargetDataPoints target='localhost', valid=False, data=[]''') result = ddv.__repr__() self.assertEqual(result, expected)
def _get_target_datapoints(self, item): """Poll each spoke in parallel. Args: item: TargetPollingPoints object Returns: ddv: TargetDataPoints for the SNMPVariable target """ # Intialize data gathering ip_target = item.target ddv = TargetDataPoints(ip_target) # BAC0 only works with IP addresses ip_address = network.get_ipaddress(ip_target) if bool(ip_address) is False: return ddv # Get list of type DataPoint datapoints = [] for polltarget in item.data: # Get polling results value = poll_target_address(ip_address, polltarget.address, 'presentValue', self._bacnet) name = poll_target_address(ip_address, polltarget.address, 'objectName', self._bacnet) # Skip if invalid data is received if value is None: continue # Do multiplication if data.is_numeric(value) is True: value = float(value) * polltarget.multiplier data_type = DATA_FLOAT else: data_type = DATA_STRING # Update datapoints datapoint = DataPoint('analog_value_point_{}'.format( polltarget.address), value, data_type=data_type) datapoint.add(DataPointMetadata('target', ip_target)) if name is not None: datapoint.add(DataPointMetadata('object_name', name)) datapoints.append(datapoint) # Return ddv.add(datapoints) return ddv
def test_add(self): """Testing function append.""" # Setup AgentPolledData agent_program = 'panda_bear' polling_interval = 20 apd = AgentPolledData(agent_program, polling_interval) # Initialize TargetDataPoints target = 'teddy_bear' ddv = TargetDataPoints(target) self.assertEqual(ddv.target, target) self.assertFalse(ddv.valid) self.assertEqual(ddv.data, []) # Setup DataPoint value = 457 _key_ = 'gummy_bear' data_type = DATA_INT variable = DataPoint(_key_, value, data_type=data_type) # Add data to TargetDataPoints self.assertFalse(ddv.valid) ddv.add(variable) self.assertTrue(ddv.valid) # Test add self.assertFalse(apd.valid) apd.add(None) self.assertFalse(apd.valid) apd.add(variable) self.assertFalse(apd.valid) apd.add(ddv) self.assertTrue(apd.valid) # Test contents data = apd.data self.assertTrue(isinstance(data, list)) self.assertEqual(len(data), 1) _ddv = data[0] self.assertTrue(isinstance(_ddv, TargetDataPoints)) self.assertEqual(_ddv.target, target) self.assertTrue(_ddv.valid) self.assertTrue(isinstance(_ddv.data, list)) self.assertTrue(len(_ddv.data), 1) data = _ddv.data _variable = _ddv.data[0] self.assertEqual(_variable.data_type, data_type) self.assertEqual(_variable.value, value) self.assertEqual(_variable.key, _key_)
def create_cache(): """Testing method / function records.""" # Initialize key variables config = ServerConfig() polling_interval = 20 cache_directory = config.agent_cache_directory(PATTOO_API_AGENT_NAME) result = { 'pattoo_agent_program': data.hashstring(str(random())), 'pattoo_agent_polled_target': socket.getfqdn(), 'pattoo_key': data.hashstring(str(random())), 'pattoo_value': round(uniform(1, 100), 5), 'pattoo_agent_hostname': socket.getfqdn() } # We want to make sure we get a different AgentID each time filename = files.agent_id_file( result['pattoo_agent_program'], config) if os.path.isfile(filename) is True: os.remove(filename) result['pattoo_agent_id'] = files.get_agent_id( result['pattoo_agent_program'], config) # Setup AgentPolledData apd = AgentPolledData(result['pattoo_agent_program'], polling_interval) # Initialize TargetDataPoints ddv = TargetDataPoints(result['pattoo_agent_hostname']) # Setup DataPoint data_type = DATA_INT variable = DataPoint( result['pattoo_key'], result['pattoo_value'], data_type=data_type) # Add data to TargetDataPoints ddv.add(variable) # Write data to cache apd.add(ddv) cache_dict = converter.posting_data_points( converter.agentdata_to_post(apd)) cache_file = '{}{}cache_test.json'.format(cache_directory, os.sep) with open(cache_file, 'w') as _fp: json.dump(cache_dict, _fp) return result
def test_agentdata_to_datapoints(self): """Testing method or function named agentdata_to_datapoints.""" # Setup AgentPolledData agent_program = 'panda_bear' polling_interval = 20 apd = AgentPolledData(agent_program, polling_interval) # Initialize TargetDataPoints target = 'teddy_bear' ddv = TargetDataPoints(target) # Setup DataPoint value = 457 key = 'gummy_bear' data_type = DATA_INT variable = DataPoint(key, value, data_type=data_type) # Add data to TargetDataPoints ddv.add(variable) # Test TargetGateway to AgentPolledData apd.add(ddv) # Test contents expected_metadata = { 'pattoo_agent_id': apd.agent_id, 'pattoo_agent_program': agent_program, 'pattoo_agent_hostname': apd.agent_hostname, 'pattoo_agent_polled_target': target, 'pattoo_agent_polling_interval': apd.agent_polling_interval } result = converter.agentdata_to_datapoints(apd) self.assertEqual(len(result), 1) item = result[0] self.assertTrue(isinstance(item, DataPoint)) self.assertEqual(item.value, value) self.assertEqual(item.data_type, DATA_INT) self.assertEqual(item.key, key) self.assertTrue(isinstance(item.metadata, dict)) self.assertEqual(len(item.metadata), len(expected_metadata)) for key, value in item.metadata.items(): self.assertTrue(isinstance(value, str)) self.assertTrue(isinstance(key, str)) self.assertEqual(value, str(expected_metadata[key]))
def _walker(snmpvariable, polltargets): """Poll each spoke in parallel. Args: snmpvariable: SNMPVariable to poll polltargets: List of PollingPoint objects to poll Returns: ddv: TargetDataPoints for the SNMPVariable target """ # Intialize data gathering ddv = TargetDataPoints(snmpvariable.ip_target) query = Query(snmpvariable) results = query.everything() datapoints = _create_datapoints(results) ddv.add(datapoints) return ddv
def test___init__(self): """Testing function __init__.""" # Setup TargetDataPoints target = 'localhost' ddv = TargetDataPoints(target) # Test initial vlues self.assertEqual(ddv.target, target) self.assertFalse(ddv.valid) self.assertEqual(ddv.data, [])
def _walker(snmpvariable, polltargets): """Poll each spoke in parallel. Args: snmpvariable: SNMPVariable to poll polltargets: List of PollingPoint objects to poll Returns: ddv: TargetDataPoints for the SNMPVariable target """ # Intialize data gathering ddv = TargetDataPoints(snmpvariable.ip_target) # Get list of type DataPoint datapoints = [] for polltarget in polltargets: # Get OID polling results query = snmp.SNMP(snmpvariable) query_datapoints = query.walk(polltarget.address) # Apply multiplier to the results for _dp in query_datapoints: # Do multiplication if data.is_data_type_numeric(_dp.data_type) is True: value = float(_dp.value) * polltarget.multiplier else: value = _dp.value # Update datapoints datapoint = DataPoint(polltarget.address, value, data_type=_dp.data_type) datapoint.add(DataPointMetadata('oid', _dp.key)) datapoints.append(datapoint) # Return ddv.add(datapoints) return ddv
def test_agent(): # Define the polling interval in seconds (integer). polling_interval = 300 # Give the agent a name agent_name = 'sample_agent_script' # Let's assume the script has already received this data from SITE_A site_a_data = [['ABC', 123.456], ['DEF', 456.789]] # Let's assume the script has already received this data from WORK_1 work_1_data = [['GHI', 654.321], ['JKL', 987.654]] # Setup the agent's AgentPolledData object. agent = AgentPolledData(agent_name, polling_interval) # Let's add some metadata that you don't want to affect charting in the # event of a change. Department names change all the time. metadata_static = DataPointMetadata('Department Name', 'The Palisadoes Foundation', update_checksum=False) # Let's add some metadata that will change and trigger a new chart. metadata_dynamic = DataPointMetadata('Financial Year', '2020') # Create target objects for SITE_A target = TargetDataPoints('SITE_A') for quote in site_a_data: key, value = quote datapoint = DataPoint(key, value, data_type=DATA_FLOAT) datapoint.add(metadata_static) datapoint.add(metadata_dynamic) target.add(datapoint) agent.add(target) # Create target objects for WORK_1 target = TargetDataPoints('WORK_1') for quote in work_1_data: key, value = quote datapoint = DataPoint(key, value, data_type=DATA_FLOAT) datapoint.add(metadata_static) datapoint.add(metadata_dynamic) target.add(datapoint) agent.add(target) # Return agent return agent
def test_add(self): """Testing function append.""" # Initialize TargetDataPoints target = 'teddy_bear' ddv = TargetDataPoints(target) self.assertEqual(ddv.target, target) self.assertFalse(ddv.valid) self.assertEqual(ddv.data, []) # Setup DataPoint value = 457 _key_ = 'gummy_bear' data_type = DATA_INT variable = DataPoint(_key_, value, data_type=data_type) # Test adding invalid value ddv.add(None) self.assertEqual(ddv.data, []) # Test adding variable ddv.add(variable) self.assertTrue(bool(ddv.data)) self.assertTrue(isinstance(ddv.data, list)) self.assertEqual(len(ddv.data), 1) checksum = ddv.data[0].checksum # Test adding duplicate variable (There should be no changes) ddv.add(variable) self.assertTrue(bool(ddv.data)) self.assertTrue(isinstance(ddv.data, list)) self.assertEqual(len(ddv.data), 1) self.assertEqual(checksum, ddv.data[0].checksum) # Test the values in the variable _variable = ddv.data[0] self.assertEqual(_variable.data_type, data_type) self.assertEqual(_variable.value, value) self.assertEqual(_variable.key, _key_)
def poll(agent_program, polling_interval): """Get all agent data. Performance data on linux server on which this application is installed. Args: agentdata: AgentPolledData object for all data gathered by the agent polling_interval: Polling interval in seconds Returns: None """ # Initialize AgentPolledData agent_hostname = socket.getfqdn() agentdata = AgentPolledData(agent_program, polling_interval) # Intialize data gathering ddv = TargetDataPoints(agent_hostname) ######################################################################### # Get timeseries values ######################################################################### performance = Performance() # Update agent with system data ddv.add(performance.stats_system()) # Update agent with disk data ddv.add(performance.stats_disk_swap()) ddv.add(performance.stats_disk_partitions()) ddv.add(performance.stats_disk_io()) # Update agent with network data ddv.add(performance.stats_network()) # Add results to the AgentPolledData object for posting agentdata.add(ddv) return agentdata
def main(): """Post data to pattoo server. Args: None Returns: None """ ''' NOTE: Scripts must be run at regular intervals and the polling_interval should be automatically provided to the main() function. Notes about CRON: When using cron, change this value to match the cron interval in seconds. It is not advised to use cron for polling unless you know the interval will not change. If using cron, remember to make the polling interval to match the cron interval in 'seconds'. Ideally your agents should run as daemons, not as cron jobs. See the daemon example script which explains how to do this. ''' # Define the polling interval in seconds (integer). polling_interval = 300 # Give the agent a name agent_name = 'sample_agent_script' # Let's assume the script has already received this data from SITE_A site_a_data = [['ABC', 123.456], ['DEF', 456.789]] # Let's assume the script has already received this data from WORK_1 work_1_data = [['GHI', 654.321], ['JKL', 987.654]] ''' NOTE: The AgentPolledData object contains unique identification information that the pattoo server will use to differentiate the information source. This includes: the hostname of the system on which the object was created, a unique hashed identifier stored in the cache directory of the agent configuration. You just need to write an agent to do one thing well, such data collection from a type of target source. For example, you don't need to give each agent that does the same thing a different "agent_name". Just make sure that different types of agents have different "agent_name" values. The PattooShared library will take care of the rest. ''' # Setup the agent's AgentPolledData object. agent = AgentPolledData(agent_name, polling_interval) ''' NOTE: Metadata is normally expected to stay constant. If it changes then the datapoint ID changes and you'll start plotting a brand new chart on the pattoo server. The old chart will stop at the time the metadata changed. In some cases, you may not want changing metadata to cause a brand new plot. For example, your metadata for computer resource charting may include the operating system version. This is useful background information, but shouldn't impact charting if it changes. This type of metadata should be dynamic. ''' # Let's add some metadata that you don't want to affect charting in the # event of a change. Department names change all the time. metadata_static = DataPointMetadata('Department Name', 'The Palisadoes Foundation', update_checksum=False) # Let's add some metadata that will change and trigger a new chart. metadata_dynamic = DataPointMetadata('Financial Year', '2020') # Create target objects for SITE_A target = TargetDataPoints('SITE_A') for quote in site_a_data: key, value = quote ''' NOTE: You don't have to specify the time when the data was collected. The DataPoint object captures that information automatically. You can also specify it using the timestamp= argument. See the class documentation for details. The default data_type is DATA_INT (integer). Read the documentation for the various other data which cover float and counter values. ''' datapoint = DataPoint(key, value, data_type=DATA_FLOAT) datapoint.add(metadata_static) datapoint.add(metadata_dynamic) target.add(datapoint) agent.add(target) # Create target objects for WORK_1 target = TargetDataPoints('WORK_1') for quote in work_1_data: key, value = quote datapoint = DataPoint(key, value, data_type=DATA_FLOAT) datapoint.add(metadata_static) datapoint.add(metadata_dynamic) target.add(datapoint) agent.add(target) # Post the data to pattoo post = PostAgent(agent) post.post()
async def _serial_poller_async(tpp): """Poll OPCUA agent data. Args: tpp: TargetDataPoints object Returns: target_datapoints: TargetDataPoints object """ # Initialize key variables connected = False # Test for validity if isinstance(tpp, TargetPollingPoints) is False: return None if isinstance(tpp.target, OPCUAauth) is False: return None if tpp.valid is False: return None # Create URL for polling ip_target = tpp.target.ip_target ip_port = tpp.target.ip_port username = tpp.target.username password = tpp.target.password url = 'opc.tcp://{}:{}'.format(ip_target, ip_port) # Intialize data gathering target_datapoints = TargetDataPoints(ip_target) # Create a client object to connect to OPCUA server client = Client(url=url) client.set_user(username) client.set_password(password) # Connect try: await client.connect() connected = True except: log_message = ( 'Authentication for polling target {} is incorrect'.format(url)) log.log2warning(51011, log_message) pass if connected is True: for point in tpp.data: # Make sure we have the right data type if isinstance(point, PollingPoint) is False: log_message = ('''\ Invalid polling point {} for OPC UA URL {}'''.format(point, url)) log.log2info(51012, log_message) continue # Get data address = point.address try: node = client.get_node(address) value = await node.read_value() except BadNodeIdUnknown: log_message = ('''\ OPC UA node {} not found on server {}'''.format(address, url)) log.log2warning(51015, log_message) continue except: _exception = sys.exc_info() log_message = ('OPC UA server communication error') log.log2exception(51014, _exception, message=log_message) log_message = ('''\ Cannot get value from polling point {} for OPC UA URL {}\ '''.format(address, url)) log.log2info(51013, log_message) continue # Create datapoint if bool(point.multiplier) is True: if is_numeric(value) is True and (is_numeric(point.multiplier) is True): value = value * point.multiplier else: value = 0 datapoint = DataPoint(address, value) datapoint.add(DataPointMetadata('OPCUA Server', ip_target)) target_datapoints.add(datapoint) # Disconnect client await client.disconnect() return target_datapoints
def _serial_poller(drv): """Poll each spoke in parallel. Args: drv: Target to poll input_registers: Input registers to poll holding_registers: Holding registers to poll Returns: ddv: TargetDataPoints for the ip_target """ # Intialize data gathering ip_target = drv.target ddv = TargetDataPoints(ip_target) # Get list of type DataPoint datapoints = [] for _rv in drv.data: # Ignore invalid data if isinstance(_rv, RegisterVariable) is False: continue if _rv.valid is False: continue # Poll client = ModbusTcpClient(ip_target) if isinstance(_rv, InputRegisterVariable): try: response = client.read_input_registers(_rv.address, count=_rv.count, unit=_rv.unit) key = 'input_register' except ConnectionException as _err: log_message = ('''\ Cannot connect to target {} to retrieve input register {}, count {}, \ unit {}: {}'''.format(ip_target, _rv.register, _rv.count, _rv.unit, str(_err))) log.log2warning(65028, log_message) continue except: log_message = ('''\ Cause unknown failure with target {} getting input register {}, count {}, \ unit {}'''.format(ip_target, _rv.register, _rv.count, _rv.unit)) log.log2warning(65030, log_message) continue elif isinstance(_rv, HoldingRegisterVariable): try: response = client.read_holding_registers(_rv.address) key = 'holding_register' except ConnectionException: log_message = ('''\ Cannot connect to target {} to retrieve input register {}, count {}, \ unit {}'''.format(ip_target, _rv.register, _rv.count, _rv.unit)) log.log2warning(65032, log_message) continue except: log_message = ('''\ Cause unknown failure with target {} getting holding register {}, count {}, \ unit {}. [{}, {}, {}]\ '''.format(ip_target, _rv.register, _rv.count, _rv.unit, sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) log.log2warning(65031, log_message) continue # Process data if response.isError() is True: _log_modbus(ip_target, _rv, response) else: values = response.registers for data_index, _value in enumerate(values): # Do multiplication value = _value * _rv.multiplier # Create DataPoint and append new_key = ('{}_{}'.format(key, _rv.register + data_index)) datapoint = DataPoint(new_key, value, data_type=DATA_INT) datapoint.add(DataPointMetadata('unit', str(_rv.unit).zfill(3))) datapoints.append(datapoint) ddv.add(datapoints) # Return return ddv