def __init__(self, parent, child, app, config=None): """Initialize the class. Args: parent: Name of parent daemon child: Name of child daemon app: Flask App config: Config object Returns: None """ # Initialize key variables if config is None: _config = Config() else: _config = config # Apply inheritance Agent.__init__(self, parent, child=child, config=_config) self._app = app self._agent_api_variable = AgentAPIVariable( ip_bind_port=_config.ip_bind_port(), ip_listen_address=_config.ip_listen_address())
def log2exception(code, sys_exc_info, message=None, die=False): """Log trace message to file and STDOUT, but don't die. Args: code: Message code sys_exc_info: Tuple from exception from sys.exc_info die: Die if True Returns: None """ # Initialize key variables (exc_type, exc_value, exc_traceback) = sys_exc_info log_message = ('''\ Bug: Exception Type:{}, Exception Instance: {}, Stack Trace Object: {}]\ '''.format(exc_type, exc_value, exc_traceback)) log2warning(code, log_message) if bool(message) is True: log2warning(code, message) # Write trace to log file from pattoo_shared.configuration import Config config = Config() log_file = config.log_file() with open(log_file, 'a+') as _fh: traceback.print_tb(exc_traceback, file=_fh) # Die if bool(die) is True: log2die(code, log_message)
def create_app(self): """Create the test APP for flask. Args: None Returns: app: Flask object """ # Create APP and set configuration app = APP config = Config() app.config['TESTING'] = True app.config['LIVESERVER_PORT'] = config.agent_api_ip_bind_port() # app.config['errorlog'] = config.log_file_api() # app.config['accesslog'] = config.log_file_api() os.environ['FLASK_ENV'] = 'development' # Clear the flask cache cache = Cache(config={'CACHE_TYPE': 'null'}) cache.init_app(app) # Return return app
def test_agent_id(self): """Testing function agent_id.""" # Test directory = files._Directory(self.config) config = Config() expected = '{}{}agent_id'.format(config.daemon_directory(), os.sep) result = directory.agent_id() self.assertEqual(result, expected)
def test_lock(self): """Testing function lock.""" # Test directory = files._Directory(self.config) config = Config() expected = '{}{}lock'.format(config.system_daemon_directory(), os.sep) result = directory.lock() self.assertEqual(result, expected)
def test___init__(self): """Testing function __init__.""" # Test directory = files._Directory(self.config) config = Config() expected = config.daemon_directory() result = directory._root self.assertEqual(result, expected)
def _get(query): """Get pattoo API server GraphQL query results. Args: query: GraphQL query string Returns: result: Dict of JSON response """ # Initialize key variables success = False config = Config() result = None # Get the data from the GraphQL API url = config.web_api_server_url() try: response = requests.get(url, params={'query': query}) # Trigger HTTP errors if present response.raise_for_status() success = True except requests.exceptions.Timeout as err: # Maybe set up for a retry, or continue in a retry loop log_message = ('''\ Timeout when attempting to access {}. Message: {}\ '''.format(url, err)) log.log2die(20121, log_message) except requests.exceptions.TooManyRedirects as err: # Tell the user their URL was bad and try a different one log_message = ('''\ Too many redirects when attempting to access {}. Message: {}\ '''.format(url, err)) log.log2die(20116, log_message) except requests.exceptions.HTTPError as err: log_message = ('''\ HTTP error when attempting to access {}. Message: {}\ '''.format(url, err)) log.log2die(20118, log_message) except requests.exceptions.RequestException as err: # catastrophic error. bail. log_message = ('''\ Exception when attempting to access {}. Message: {}\ '''.format(url, err)) log.log2die(20119, log_message) except: log_message = ('''API Failure: [{}, {}, {}]\ '''.format(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) log.log2die(20120, log_message) # Process the data if bool(success) is True: result = response.json() return result
def __init__(self): """Initialize the class.""" # Pattoo libraries from pattoo_shared.configuration import Config # Define key variables app_name = 'pattoo' levels = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL } # Get the logging directory config = Config() log_file = config.log_file() config_log_level = config.log_level() # Set logging level if config_log_level in levels: log_level = levels[config_log_level] else: log_level = levels['debug'] # create logger with app_name self.logger_file = logging.getLogger('{}_file'.format(app_name)) self.logger_stdout = logging.getLogger('{}_console'.format(app_name)) # Set logging levels to file and stdout self.logger_stdout.setLevel(log_level) self.logger_file.setLevel(log_level) # create file handler which logs even debug messages file_handler = logging.FileHandler(log_file) file_handler.setLevel(log_level) # create console handler with a higher log level stdout_handler = logging.StreamHandler() stdout_handler.setLevel(log_level) # create formatter and add it to the handlers formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler.setFormatter(formatter) stdout_handler.setFormatter(formatter) # add the handlers to the logger self.logger_file.addHandler(file_handler) self.logger_stdout.addHandler(stdout_handler)
def test_index(self): """Testing method / function index.""" # Initialize key variables expected = 'The Pattoo Agent API is Operational.\n' # Create URL config = Config() agent_url = config.agent_api_server_url('') url = agent_url.replace('/receive/', '/status') # Check response with requests.get(url) as response: result = response.text self.assertEqual(expected, result)
def set_api_email(self): """Set email address for API to support encryption Args: None Returns: None """ # Add email address to Agent subclass econfig = Config() email_addr = econfig.api_email_address() self.set_email(email_addr)
class Test_File(unittest.TestCase): """Checks all functions and methods.""" ######################################################################### # General object setup ######################################################################### prefix = 'unittest' agent_hostname = 'pattoo_host' config = Config() def test___init__(self): """Testing function __init__.""" pass def test_pid(self): """Testing function pid.""" # Test filename = files._File(self.config) result = filename.pid(self.prefix) self.assertTrue(os.path.isdir(os.path.dirname(result))) def test_lock(self): """Testing function lock.""" # Test filename = files._File(self.config) result = filename.lock(self.prefix) self.assertTrue(os.path.isdir(os.path.dirname(result))) def test_agent_id(self): """Testing function agent_id.""" filename = files._File(self.config) result = filename.agent_id(self.prefix) self.assertTrue(os.path.isdir(os.path.dirname(result)))
def __init__(self, parent, child=None, config=None): """Initialize the class. Args: parent: Name of parent daemon child: Name of child daemon config: Config object Returns: None """ # Initialize key variables (Parent) if config is None: self.config = Config() else: self.config = config self.parent = parent self.pidfile_parent = files.pid_file(parent, self.config) self.lockfile_parent = files.lock_file(parent, self.config) # Initialize key variables (Child) if bool(child) is None: self._pidfile_child = None else: self._pidfile_child = files.pid_file(child, self.config)
def _make_agent_data(): """Create generate data to post to API server""" # Initialize key variables config = Config() polling_interval = 60 pattoo_agent_program = 1 pattoo_agent_polled_target = 2 pattoo_key = '3' pattoo_value = 4 # We want to make sure we get a different AgentID each time filename = files.agent_id_file(pattoo_agent_program, config) if os.path.isfile(filename) is True: os.remove(filename) # Setup AgentPolledData apd = AgentPolledData(pattoo_agent_program, polling_interval) # Initialize TargetDataPoints ddv = TargetDataPoints(pattoo_agent_polled_target) # Setup DataPoint data_type = DATA_INT variable = DataPoint(pattoo_key, pattoo_value, data_type=data_type) # Add data to TargetDataPoints ddv.add(variable) # Create a result apd.add(ddv) # Return agent data return apd
def __init__(self): """Initialize the class. Args: None Returns: None """ # Instantiate inheritance Config.__init__(self) # Get the configuration directory config_file = configuration.agent_config_filename(PATTOO_AGENT_SNMPD) self._agent_config = files.read_yaml_file(config_file)
def setUp(self): """Test setup""" # Setup base config and agent self._config = Config() self._agent = _create_agent() # Instantiation of test daemon self._daemon = MockGracefulDaemon(self._agent)
def _save_data(data, identifier): """Save data to cache file. Args: data: Dict to save identifier: Unique identifier for the source of the data. (AgentID) Returns: success: True: if successful """ # Initialize key variables success = False config = Config() cache_dir = config.agent_cache_directory(identifier) timestamp = int(time() * 1000) # Create a unique very long filename to reduce risk of filename = ('''{}{}{}_{}.json\ '''.format(cache_dir, os.sep, timestamp, identifier)) # Save data try: with open(filename, 'w') as f_handle: json.dump(data, f_handle) success = True except Exception as err: log_message = '{}'.format(err) log.log2warning(1030, log_message) except: (etype, evalue, etraceback) = sys.exc_info() log_message = ('''\ Cache-file save error: [{}, {}, {}]'''.format(etype, evalue, etraceback)) log.log2warning(1031, log_message) # Delete file if there is a failure. # Helps to protect against full file systems. if os.path.isfile(filename) is True and success is False: os.remove(filename) log_message = ('''\ Deleting corrupted cache file {} for identifier {}.\ '''.format(filename, identifier)) log.log2warning(1037, log_message) # Return return success
def setUp(self): """This will run each time before a test is performed """ print('setUp') gconfig = Config() # Get config for Pgpier # Create Pgpier object for the API api_gpg = files.set_gnupg(PATTOO_API_AGENT_NAME, gconfig, "*****@*****.**")
def __init__(self, identifier, data): """Initialize the class. Args: identifier: Unique identifier for the source of the data. (AgentID) data: dict of data to post Returns: None """ # Initialize key variables config = Config() # Get posting URL self._data = data self._identifier = identifier self._url = config.agent_api_server_url(identifier)
def test_encrypted_post(self): """Test that the API can receive and decrypt encrypted data from agent""" # Initialize key variables config = ServerConfig() # Get Pgpier object gconfig = Config() # Get config for Pgpier # Create Pgpier object for the agent agent_gpg = files.set_gnupg("test_encrypted_agent", gconfig, "*****@*****.**") # Make agent data agent_data = _make_agent_data() # Turn agent data into a dict to be compared to # the data received by the API expected = converter.posting_data_points( converter.agentdata_to_post(agent_data)) # Make encrypted post post_encrypted = EncryptedPostAgent(agent_data, agent_gpg) post_encrypted.post() # Read data from directory cache_directory = config.agent_cache_directory(PATTOO_API_AGENT_NAME) cache_data = files.read_json_files(cache_directory) # Test self.assertEqual(len(cache_data), 1) self.assertEqual(len(cache_data[0]), 2) result = cache_data[0][1] # Result and expected are not quite the same. 'expected' will have # lists of tuples where 'result' will have lists of lists for key, value in result.items(): if key != 'pattoo_datapoints': self.assertEqual(value, expected[key]) self.assertEqual(result['pattoo_datapoints']['datapoint_pairs'], expected['pattoo_datapoints']['datapoint_pairs']) # Test list of tuples for key, value in result['pattoo_datapoints']['key_value_pairs'].items( ): self.assertEqual( tuple(value), expected['pattoo_datapoints']['key_value_pairs'][int(key)]) # Revert cache_directory for filename in os.listdir(cache_directory): # Examine all the '.json' files in directory if filename.endswith('.json'): # Read file and add to string filepath = '{}{}{}'.format(cache_directory, os.sep, filename) os.remove(filepath)
def setUp(self): """Test setup""" # Setup base config and agent self._config = Config() self._agent = agent.Agent(parent=AGENT_NAME, config=self._config) # Instantiation of test daemon self._daemon = agent.BaseAgentDaemon(self._agent)
def setUp(self): """Test setup""" # Setup base config and agent self._config = Config() agent_name = general.random_agent_name() self._agent = agent.Agent(agent_name, config=self._config) # Instantiation of test daemon self._daemon = agent.AgentDaemon(self._agent)
def _create_agent(): """Creates new test agent. Args: None Return: result: new agent for testing """ # Return config = Config() name = general.random_agent_name() result = Agent(name, config=config) return result
class TestAgent(unittest.TestCase): """Checks all functions and methods.""" ######################################################################### # General object setup ######################################################################### config = Config() def test___init__(self): """Testing method or function named __init__.""" # Initialize key variables parent = 'parent' child = 'child' # Test - No Child tester = agent.Agent(parent, config=self.config) self.assertEqual(tester.parent, parent) expected = files.pid_file(parent, self.config) self.assertEqual(tester.pidfile_parent, expected) expected = files.lock_file(parent, self.config) self.assertEqual(tester.lockfile_parent, expected) expected = files.pid_file(None, self.config) self.assertEqual(tester._pidfile_child, expected) # Test - With Child tester = agent.Agent(parent, child=child, config=self.config) self.assertEqual(tester.parent, parent) expected = files.pid_file(parent, self.config) self.assertEqual(tester.pidfile_parent, expected) expected = files.lock_file(parent, self.config) self.assertEqual(tester.lockfile_parent, expected) expected = files.pid_file(child, self.config) self.assertEqual(tester._pidfile_child, expected) def test_name(self): """Testing method or function named name.""" # Initialize key variables parent = 'parent' # Test tester = agent.Agent(parent) self.assertEqual(tester.name(), parent) def test_run(self): """Testing method or function named query.""" pass
def __init__(self, identifier, data): """Initialize the class. Args: identifier: Unique identifier for the source of the data. (AgentID) data: dict of data to post Returns: None """ # Initialize key variables self.config = Config() # Get data and identifier self._data = data self._identifier = identifier
def _relay(url): """Relay data to pattoo server. Args: url: Pattoo spoked agent URL Returns: None """ # Initialize key variables config = Config() agent_id = files.get_agent_id(PATTOO_AGENT_LINUX_HUBD, config) # Initialize key variables passive = PassiveAgent(PATTOO_AGENT_LINUX_HUBD, agent_id, url) passive.relay()
class Test_Directory(unittest.TestCase): """Checks all functions and methods.""" ######################################################################### # General object setup ######################################################################### config = Config() def test___init__(self): """Testing function __init__.""" # Test directory = files._Directory(self.config) config = Config() expected = config.daemon_directory() result = directory._root self.assertEqual(result, expected) def test_pid(self): """Testing function pid.""" # Test directory = files._Directory(self.config) config = Config() expected = '{}{}pid'.format(config.system_daemon_directory(), os.sep) result = directory.pid() self.assertEqual(result, expected) def test_lock(self): """Testing function lock.""" # Test directory = files._Directory(self.config) config = Config() expected = '{}{}lock'.format(config.system_daemon_directory(), os.sep) result = directory.lock() self.assertEqual(result, expected) def test_agent_id(self): """Testing function agent_id.""" # Test directory = files._Directory(self.config) config = Config() expected = '{}{}agent_id'.format(config.daemon_directory(), os.sep) result = directory.agent_id() self.assertEqual(result, expected)
def arguments(): """Get the CLI arguments. Args: None Returns: args: NamedTuple of argument values """ # Get config config = Config() # Get cache directory directory = config.agent_cache_directory(PATTOO_API_AGENT_NAME) # Get arguments parser = argparse.ArgumentParser(description='''\ Program to ingest cached agent data from the {} directory into the database.\ '''.format(directory)) parser.add_argument('-b', '--batch_size', default=500, type=int, help='''\ The number of files to process at a time. Smaller batch sizes may help when \ you are memory or database connection constrained. Default=500''') parser.add_argument('-m', '--max_duration', default=3600, type=int, help='''\ The maximum time in seconds that the script should run. This reduces the risk \ of not keeping up with the cache data updates. Default=3600''') # Return args = parser.parse_args() return args
def main(): """Test all the pattoo-shared modules with unittests. Args: None Returns: None """ # Initialize key variables config = Config() # Set up parser parser = argparse.ArgumentParser() parser.add_argument('--start', help='Start daemon', action='store_true') parser.add_argument('--stop', help='Start daemon', action='store_true') parser.add_argument('--restart', help='Start daemon', action='store_true') parser.add_argument('--status', help='Daemon status', action='store_true') parser.add_argument('--agent_name', help='Agent name', type=str, required=True) args = parser.parse_args() # Daemon manipulation agent_ = Agent(args.agent_name, config=config) daemon = MockDaemon(agent_) if bool(args.start): daemon.start() elif bool(args.stop): daemon.stop() elif bool(args.restart): daemon.restart() elif bool(args.status): daemon.status() else: print('No command matches')
class Test_GnuPG(unittest.TestCase): """Test Pgpier integration.""" # Initialize config object config = Config() def test_set_gnupg(self): """Set Pgpier object test for agent.""" # Agent details agent_name = 'test_agent1' agent_email = '*****@*****.**' # Result result = files.set_gnupg(agent_name, self.config, agent_email) # If a Pgpier object was created, a key ID would # be set of type str self.assertIsNotNone(result.keyid) self.assertIsInstance(result.keyid, str) def test_get_gnupg(self): """Test retrieval of Pgpier object.""" # Create Pgpier object # Agent details agent_name = 'test_agent2' agent_email = '*****@*****.**' # Result files.set_gnupg(agent_name, self.config, agent_email) # Retrieve Pgpier object result = files.get_gnupg(agent_name, self.config) # Test if the Pgpier object was retrieved self.assertIsNotNone(result)
class TestEncryptedPostAgent(unittest.TestCase): """Test EncryptedPostAgent""" # Initialize # Create Pgpier objects agent_gpg = set_gnupg('test_agent0', Config(), '*****@*****.**') api_gpg = set_gnupg('test_api0', Config(), '*****@*****.**') # Variables that will be modified by callback functions agent_publickey = None agent_email = None symmetric_key = None nonce = None def test_agent(self): """Test agent post and purge""" # Create agent data agentdata = ta.test_agent() # Get agent variables _data = converter.agentdata_to_post(agentdata) data = converter.posting_data_points(_data) # Create agent encrypted_agent = phttp.EncryptedPostAgent(agentdata, self.agent_gpg) # Define callback functions # Key exchange callback for post request to process key exchange def exchange_post_callback(request, context): """Exchange callback for post request mock""" # Retrieve agent info from received request object json_data = request.json() json_dict = json.loads(json_data) self.agent_publickey = json_dict['pattoo_agent_key'] self.agent_email = json_dict['pattoo_agent_email'] # Import agent public key self.api_gpg.imp_pub_key(self.agent_publickey) # Trust public keys to enable encryption with traded keys agent_fp = self.api_gpg.email_to_key(self.agent_email) self.api_gpg.trust_key(agent_fp) # Send accepted response context.status_code = 202 return "Noted" # Key exchange callback for post request to process key exchange def exchange_get_callback(request, context): """Exchange callback for post request mock""" # Status is OK context.status_code = 200 # Generate nonce self.nonce = hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest() # Prepare API info to send to agent api_publickey = self.api_gpg.exp_pub_key() self.api_gpg.set_email() api_email_addr = self.api_gpg.email_addr # Encrypt nonce encrypted_nonce = self.api_gpg.encrypt_data( self.nonce, self.agent_gpg.fingerprint) json_response = { 'data': { 'api_email': api_email_addr, 'api_key': api_publickey, 'encrypted_nonce': encrypted_nonce } } # Send data return json_response # Validation callback to respond to agent validation post request def validation_callback(request, context): """Validation callback for request mock""" # Retrieve agent info from received request object json_data = request.json() json_dict = json.loads(json_data) encrypted_nonce = json_dict['encrypted_nonce'] encrypted_sym_key = json_dict['encrypted_sym_key'] # Validate by decrypting the encrypted symmetric key # then using the symmetric key to decrypt the nonce # and check if it is the same as the one that was sent passphrase = self.api_gpg.passphrase symmetric_key = self.api_gpg.decrypt_data(encrypted_sym_key, passphrase) nonce = self.api_gpg.symmetric_decrypt(encrypted_nonce, symmetric_key) if nonce == self.nonce: self.symmetric_key = symmetric_key context.status_code = 200 else: context.status_code = 409 return 'Result' # Encrypted post callback to respond to agent encrypted data def encrypted_callback(request, context): """Encrypted post callback for request mock""" # Retrieve encrypted data from received request object json_data = request.json() json_dict = json.loads(json_data) encrypted_data = json_dict['encrypted_data'] # Decrypt data decrypted_data = self.api_gpg.symmetric_decrypt( encrypted_data, self.symmetric_key) # Unload data_dict = json.loads(decrypted_data) recv_data = data_dict['data'] # Check that decrypted data is the same as the received # The two dictionaries are hashed then the values are compared agent_data = data if len(agent_data) == len(recv_data): # Data received and decrypted successfully context.status_code = 202 else: # Decryption failed context.status_code = 409 return "Noted" # Mock each requests with requests_mock.Mocker() as m: # Mock agent sending info to API server m.post('http://127.0.0.6:50505/pattoo/api/v1/agent/key', text=exchange_post_callback) # Mock agent receiving API info m.get('http://127.0.0.6:50505/pattoo/api/v1/agent/key', json=exchange_get_callback) # Mock agent validation m.post('http://127.0.0.6:50505/pattoo/api/v1/agent/validation', text=validation_callback) m.post('http://127.0.0.6:50505/pattoo/api/v1/agent/encrypted', text=encrypted_callback) # Run function success = encrypted_agent.post() # Test self.assertTrue(success) # Get agent identifier from data identifier = agentdata.agent_id # Save data to cache phttp._save_data(data, identifier) # Encrypted purge encrypted_agent.purge() # Check that both the post and purge exchanged keys and # send data self.assertEqual(m.call_count, 8)