def __init__(self, parent, child, app, config=None):
"""Initialize the class.
Args:
parent: Name of parent daemon
child: Name of child daemon
app: Flask App
config: Config object
Returns:
None
"""
# Initialize key variables
if config is None:
_config = Config()
else:
_config = config
# Apply inheritance
Agent.__init__(self, parent, child=child, config=_config)
self._app = app
self._agent_api_variable = AgentAPIVariable(
ip_bind_port=_config.ip_bind_port(),
ip_listen_address=_config.ip_listen_address())
def log2exception(code, sys_exc_info, message=None, die=False):
"""Log trace message to file and STDOUT, but don't die.
Args:
code: Message code
sys_exc_info: Tuple from exception from sys.exc_info
die: Die if True
Returns:
None
"""
# Initialize key variables
(exc_type, exc_value, exc_traceback) = sys_exc_info
log_message = ('''\
Bug: Exception Type:{}, Exception Instance: {}, Stack Trace Object: {}]\
'''.format(exc_type, exc_value, exc_traceback))
log2warning(code, log_message)
if bool(message) is True:
log2warning(code, message)
# Write trace to log file
from pattoo_shared.configuration import Config
config = Config()
log_file = config.log_file()
with open(log_file, 'a+') as _fh:
traceback.print_tb(exc_traceback, file=_fh)
# Die
if bool(die) is True:
log2die(code, log_message)
def create_app(self):
"""Create the test APP for flask.
Args:
None
Returns:
app: Flask object
"""
# Create APP and set configuration
app = APP
config = Config()
app.config['TESTING'] = True
app.config['LIVESERVER_PORT'] = config.agent_api_ip_bind_port()
# app.config['errorlog'] = config.log_file_api()
# app.config['accesslog'] = config.log_file_api()
os.environ['FLASK_ENV'] = 'development'
# Clear the flask cache
cache = Cache(config={'CACHE_TYPE': 'null'})
cache.init_app(app)
# Return
return app
def test_agent_id(self):
"""Testing function agent_id."""
# Test
directory = files._Directory(self.config)
config = Config()
expected = '{}{}agent_id'.format(config.daemon_directory(), os.sep)
result = directory.agent_id()
self.assertEqual(result, expected)
def test_lock(self):
"""Testing function lock."""
# Test
directory = files._Directory(self.config)
config = Config()
expected = '{}{}lock'.format(config.system_daemon_directory(), os.sep)
result = directory.lock()
self.assertEqual(result, expected)
def test___init__(self):
"""Testing function __init__."""
# Test
directory = files._Directory(self.config)
config = Config()
expected = config.daemon_directory()
result = directory._root
self.assertEqual(result, expected)
def _get(query):
"""Get pattoo API server GraphQL query results.
Args:
query: GraphQL query string
Returns:
result: Dict of JSON response
"""
# Initialize key variables
success = False
config = Config()
result = None
# Get the data from the GraphQL API
url = config.web_api_server_url()
try:
response = requests.get(url, params={'query': query})
# Trigger HTTP errors if present
response.raise_for_status()
success = True
except requests.exceptions.Timeout as err:
# Maybe set up for a retry, or continue in a retry loop
log_message = ('''\
Timeout when attempting to access {}. Message: {}\
'''.format(url, err))
log.log2die(20121, log_message)
except requests.exceptions.TooManyRedirects as err:
# Tell the user their URL was bad and try a different one
log_message = ('''\
Too many redirects when attempting to access {}. Message: {}\
'''.format(url, err))
log.log2die(20116, log_message)
except requests.exceptions.HTTPError as err:
log_message = ('''\
HTTP error when attempting to access {}. Message: {}\
'''.format(url, err))
log.log2die(20118, log_message)
except requests.exceptions.RequestException as err:
# catastrophic error. bail.
log_message = ('''\
Exception when attempting to access {}. Message: {}\
'''.format(url, err))
log.log2die(20119, log_message)
except:
log_message = ('''API Failure: [{}, {}, {}]\
'''.format(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2]))
log.log2die(20120, log_message)
# Process the data
if bool(success) is True:
result = response.json()
return result
def __init__(self):
"""Initialize the class."""
# Pattoo libraries
from pattoo_shared.configuration import Config
# Define key variables
app_name = 'pattoo'
levels = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL
}
# Get the logging directory
config = Config()
log_file = config.log_file()
config_log_level = config.log_level()
# Set logging level
if config_log_level in levels:
log_level = levels[config_log_level]
else:
log_level = levels['debug']
# create logger with app_name
self.logger_file = logging.getLogger('{}_file'.format(app_name))
self.logger_stdout = logging.getLogger('{}_console'.format(app_name))
# Set logging levels to file and stdout
self.logger_stdout.setLevel(log_level)
self.logger_file.setLevel(log_level)
# create file handler which logs even debug messages
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(log_level)
# create console handler with a higher log level
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(log_level)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
# add the handlers to the logger
self.logger_file.addHandler(file_handler)
self.logger_stdout.addHandler(stdout_handler)
def test_index(self):
"""Testing method / function index."""
# Initialize key variables
expected = 'The Pattoo Agent API is Operational.\n'
# Create URL
config = Config()
agent_url = config.agent_api_server_url('')
url = agent_url.replace('/receive/', '/status')
# Check response
with requests.get(url) as response:
result = response.text
self.assertEqual(expected, result)
def set_api_email(self):
"""Set email address for API to support encryption
Args:
None
Returns:
None
"""
# Add email address to Agent subclass
econfig = Config()
email_addr = econfig.api_email_address()
self.set_email(email_addr)
class Test_File(unittest.TestCase):
"""Checks all functions and methods."""
#########################################################################
# General object setup
#########################################################################
prefix = 'unittest'
agent_hostname = 'pattoo_host'
config = Config()
def test___init__(self):
"""Testing function __init__."""
pass
def test_pid(self):
"""Testing function pid."""
# Test
filename = files._File(self.config)
result = filename.pid(self.prefix)
self.assertTrue(os.path.isdir(os.path.dirname(result)))
def test_lock(self):
"""Testing function lock."""
# Test
filename = files._File(self.config)
result = filename.lock(self.prefix)
self.assertTrue(os.path.isdir(os.path.dirname(result)))
def test_agent_id(self):
"""Testing function agent_id."""
filename = files._File(self.config)
result = filename.agent_id(self.prefix)
self.assertTrue(os.path.isdir(os.path.dirname(result)))
def __init__(self, parent, child=None, config=None):
"""Initialize the class.
Args:
parent: Name of parent daemon
child: Name of child daemon
config: Config object
Returns:
None
"""
# Initialize key variables (Parent)
if config is None:
self.config = Config()
else:
self.config = config
self.parent = parent
self.pidfile_parent = files.pid_file(parent, self.config)
self.lockfile_parent = files.lock_file(parent, self.config)
# Initialize key variables (Child)
if bool(child) is None:
self._pidfile_child = None
else:
self._pidfile_child = files.pid_file(child, self.config)
def _make_agent_data():
"""Create generate data to post to API server"""
# Initialize key variables
config = Config()
polling_interval = 60
pattoo_agent_program = 1
pattoo_agent_polled_target = 2
pattoo_key = '3'
pattoo_value = 4
# We want to make sure we get a different AgentID each time
filename = files.agent_id_file(pattoo_agent_program, config)
if os.path.isfile(filename) is True:
os.remove(filename)
# Setup AgentPolledData
apd = AgentPolledData(pattoo_agent_program, polling_interval)
# Initialize TargetDataPoints
ddv = TargetDataPoints(pattoo_agent_polled_target)
# Setup DataPoint
data_type = DATA_INT
variable = DataPoint(pattoo_key, pattoo_value, data_type=data_type)
# Add data to TargetDataPoints
ddv.add(variable)
# Create a result
apd.add(ddv)
# Return agent data
return apd
def __init__(self):
"""Initialize the class.
Args:
None
Returns:
None
"""
# Instantiate inheritance
Config.__init__(self)
# Get the configuration directory
config_file = configuration.agent_config_filename(PATTOO_AGENT_SNMPD)
self._agent_config = files.read_yaml_file(config_file)
def setUp(self):
"""Test setup"""
# Setup base config and agent
self._config = Config()
self._agent = _create_agent()
# Instantiation of test daemon
self._daemon = MockGracefulDaemon(self._agent)
def _save_data(data, identifier):
"""Save data to cache file.
Args:
data: Dict to save
identifier: Unique identifier for the source of the data. (AgentID)
Returns:
success: True: if successful
"""
# Initialize key variables
success = False
config = Config()
cache_dir = config.agent_cache_directory(identifier)
timestamp = int(time() * 1000)
# Create a unique very long filename to reduce risk of
filename = ('''{}{}{}_{}.json\
'''.format(cache_dir, os.sep, timestamp, identifier))
# Save data
try:
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
success = True
except Exception as err:
log_message = '{}'.format(err)
log.log2warning(1030, log_message)
except:
(etype, evalue, etraceback) = sys.exc_info()
log_message = ('''\
Cache-file save error: [{}, {}, {}]'''.format(etype, evalue, etraceback))
log.log2warning(1031, log_message)
# Delete file if there is a failure.
# Helps to protect against full file systems.
if os.path.isfile(filename) is True and success is False:
os.remove(filename)
log_message = ('''\
Deleting corrupted cache file {} for identifier {}.\
'''.format(filename, identifier))
log.log2warning(1037, log_message)
# Return
return success
def setUp(self):
"""This will run each time before a test is performed
"""
print('setUp')
gconfig = Config() # Get config for Pgpier
# Create Pgpier object for the API
api_gpg = files.set_gnupg(PATTOO_API_AGENT_NAME, gconfig,
"*****@*****.**")
def __init__(self, identifier, data):
"""Initialize the class.
Args:
identifier: Unique identifier for the source of the data. (AgentID)
data: dict of data to post
Returns:
None
"""
# Initialize key variables
config = Config()
# Get posting URL
self._data = data
self._identifier = identifier
self._url = config.agent_api_server_url(identifier)
def test_encrypted_post(self):
"""Test that the API can receive and decrypt
encrypted data from agent"""
# Initialize key variables
config = ServerConfig()
# Get Pgpier object
gconfig = Config() # Get config for Pgpier
# Create Pgpier object for the agent
agent_gpg = files.set_gnupg("test_encrypted_agent", gconfig,
"*****@*****.**")
# Make agent data
agent_data = _make_agent_data()
# Turn agent data into a dict to be compared to
# the data received by the API
expected = converter.posting_data_points(
converter.agentdata_to_post(agent_data))
# Make encrypted post
post_encrypted = EncryptedPostAgent(agent_data, agent_gpg)
post_encrypted.post()
# Read data from directory
cache_directory = config.agent_cache_directory(PATTOO_API_AGENT_NAME)
cache_data = files.read_json_files(cache_directory)
# Test
self.assertEqual(len(cache_data), 1)
self.assertEqual(len(cache_data[0]), 2)
result = cache_data[0][1]
# Result and expected are not quite the same. 'expected' will have
# lists of tuples where 'result' will have lists of lists
for key, value in result.items():
if key != 'pattoo_datapoints':
self.assertEqual(value, expected[key])
self.assertEqual(result['pattoo_datapoints']['datapoint_pairs'],
expected['pattoo_datapoints']['datapoint_pairs'])
# Test list of tuples
for key, value in result['pattoo_datapoints']['key_value_pairs'].items(
):
self.assertEqual(
tuple(value),
expected['pattoo_datapoints']['key_value_pairs'][int(key)])
# Revert cache_directory
for filename in os.listdir(cache_directory):
# Examine all the '.json' files in directory
if filename.endswith('.json'):
# Read file and add to string
filepath = '{}{}{}'.format(cache_directory, os.sep, filename)
os.remove(filepath)
def setUp(self):
"""Test setup"""
# Setup base config and agent
self._config = Config()
self._agent = agent.Agent(parent=AGENT_NAME, config=self._config)
# Instantiation of test daemon
self._daemon = agent.BaseAgentDaemon(self._agent)
def setUp(self):
"""Test setup"""
# Setup base config and agent
self._config = Config()
agent_name = general.random_agent_name()
self._agent = agent.Agent(agent_name, config=self._config)
# Instantiation of test daemon
self._daemon = agent.AgentDaemon(self._agent)
def _create_agent():
"""Creates new test agent.
Args:
None
Return:
result: new agent for testing
"""
# Return
config = Config()
name = general.random_agent_name()
result = Agent(name, config=config)
return result
class TestAgent(unittest.TestCase):
"""Checks all functions and methods."""
#########################################################################
# General object setup
#########################################################################
config = Config()
def test___init__(self):
"""Testing method or function named __init__."""
# Initialize key variables
parent = 'parent'
child = 'child'
# Test - No Child
tester = agent.Agent(parent, config=self.config)
self.assertEqual(tester.parent, parent)
expected = files.pid_file(parent, self.config)
self.assertEqual(tester.pidfile_parent, expected)
expected = files.lock_file(parent, self.config)
self.assertEqual(tester.lockfile_parent, expected)
expected = files.pid_file(None, self.config)
self.assertEqual(tester._pidfile_child, expected)
# Test - With Child
tester = agent.Agent(parent, child=child, config=self.config)
self.assertEqual(tester.parent, parent)
expected = files.pid_file(parent, self.config)
self.assertEqual(tester.pidfile_parent, expected)
expected = files.lock_file(parent, self.config)
self.assertEqual(tester.lockfile_parent, expected)
expected = files.pid_file(child, self.config)
self.assertEqual(tester._pidfile_child, expected)
def test_name(self):
"""Testing method or function named name."""
# Initialize key variables
parent = 'parent'
# Test
tester = agent.Agent(parent)
self.assertEqual(tester.name(), parent)
def test_run(self):
"""Testing method or function named query."""
pass
def __init__(self, identifier, data):
"""Initialize the class.
Args:
identifier: Unique identifier for the source of the data. (AgentID)
data: dict of data to post
Returns:
None
"""
# Initialize key variables
self.config = Config()
# Get data and identifier
self._data = data
self._identifier = identifier
def _relay(url):
"""Relay data to pattoo server.
Args:
url: Pattoo spoked agent URL
Returns:
None
"""
# Initialize key variables
config = Config()
agent_id = files.get_agent_id(PATTOO_AGENT_LINUX_HUBD, config)
# Initialize key variables
passive = PassiveAgent(PATTOO_AGENT_LINUX_HUBD, agent_id, url)
passive.relay()
class Test_Directory(unittest.TestCase):
"""Checks all functions and methods."""
#########################################################################
# General object setup
#########################################################################
config = Config()
def test___init__(self):
"""Testing function __init__."""
# Test
directory = files._Directory(self.config)
config = Config()
expected = config.daemon_directory()
result = directory._root
self.assertEqual(result, expected)
def test_pid(self):
"""Testing function pid."""
# Test
directory = files._Directory(self.config)
config = Config()
expected = '{}{}pid'.format(config.system_daemon_directory(), os.sep)
result = directory.pid()
self.assertEqual(result, expected)
def test_lock(self):
"""Testing function lock."""
# Test
directory = files._Directory(self.config)
config = Config()
expected = '{}{}lock'.format(config.system_daemon_directory(), os.sep)
result = directory.lock()
self.assertEqual(result, expected)
def test_agent_id(self):
"""Testing function agent_id."""
# Test
directory = files._Directory(self.config)
config = Config()
expected = '{}{}agent_id'.format(config.daemon_directory(), os.sep)
result = directory.agent_id()
self.assertEqual(result, expected)
def arguments():
"""Get the CLI arguments.
Args:
None
Returns:
args: NamedTuple of argument values
"""
# Get config
config = Config()
# Get cache directory
directory = config.agent_cache_directory(PATTOO_API_AGENT_NAME)
# Get arguments
parser = argparse.ArgumentParser(description='''\
Program to ingest cached agent data from the {} directory into the database.\
'''.format(directory))
parser.add_argument('-b',
'--batch_size',
default=500,
type=int,
help='''\
The number of files to process at a time. Smaller batch sizes may help when \
you are memory or database connection constrained. Default=500''')
parser.add_argument('-m',
'--max_duration',
default=3600,
type=int,
help='''\
The maximum time in seconds that the script should run. This reduces the risk \
of not keeping up with the cache data updates. Default=3600''')
# Return
args = parser.parse_args()
return args
def main():
"""Test all the pattoo-shared modules with unittests.
Args:
None
Returns:
None
"""
# Initialize key variables
config = Config()
# Set up parser
parser = argparse.ArgumentParser()
parser.add_argument('--start', help='Start daemon', action='store_true')
parser.add_argument('--stop', help='Start daemon', action='store_true')
parser.add_argument('--restart', help='Start daemon', action='store_true')
parser.add_argument('--status', help='Daemon status', action='store_true')
parser.add_argument('--agent_name',
help='Agent name',
type=str,
required=True)
args = parser.parse_args()
# Daemon manipulation
agent_ = Agent(args.agent_name, config=config)
daemon = MockDaemon(agent_)
if bool(args.start):
daemon.start()
elif bool(args.stop):
daemon.stop()
elif bool(args.restart):
daemon.restart()
elif bool(args.status):
daemon.status()
else:
print('No command matches')
class Test_GnuPG(unittest.TestCase):
"""Test Pgpier integration."""
# Initialize config object
config = Config()
def test_set_gnupg(self):
"""Set Pgpier object test for agent."""
# Agent details
agent_name = 'test_agent1'
agent_email = '*****@*****.**'
# Result
result = files.set_gnupg(agent_name, self.config, agent_email)
# If a Pgpier object was created, a key ID would
# be set of type str
self.assertIsNotNone(result.keyid)
self.assertIsInstance(result.keyid, str)
def test_get_gnupg(self):
"""Test retrieval of Pgpier object."""
# Create Pgpier object
# Agent details
agent_name = 'test_agent2'
agent_email = '*****@*****.**'
# Result
files.set_gnupg(agent_name, self.config, agent_email)
# Retrieve Pgpier object
result = files.get_gnupg(agent_name, self.config)
# Test if the Pgpier object was retrieved
self.assertIsNotNone(result)
class TestEncryptedPostAgent(unittest.TestCase):
"""Test EncryptedPostAgent"""
# Initialize
# Create Pgpier objects
agent_gpg = set_gnupg('test_agent0', Config(), '*****@*****.**')
api_gpg = set_gnupg('test_api0', Config(), '*****@*****.**')
# Variables that will be modified by callback functions
agent_publickey = None
agent_email = None
symmetric_key = None
nonce = None
def test_agent(self):
"""Test agent post and purge"""
# Create agent data
agentdata = ta.test_agent()
# Get agent variables
_data = converter.agentdata_to_post(agentdata)
data = converter.posting_data_points(_data)
# Create agent
encrypted_agent = phttp.EncryptedPostAgent(agentdata, self.agent_gpg)
# Define callback functions
# Key exchange callback for post request to process key exchange
def exchange_post_callback(request, context):
"""Exchange callback for post request mock"""
# Retrieve agent info from received request object
json_data = request.json()
json_dict = json.loads(json_data)
self.agent_publickey = json_dict['pattoo_agent_key']
self.agent_email = json_dict['pattoo_agent_email']
# Import agent public key
self.api_gpg.imp_pub_key(self.agent_publickey)
# Trust public keys to enable encryption with traded keys
agent_fp = self.api_gpg.email_to_key(self.agent_email)
self.api_gpg.trust_key(agent_fp)
# Send accepted response
context.status_code = 202
return "Noted"
# Key exchange callback for post request to process key exchange
def exchange_get_callback(request, context):
"""Exchange callback for post request mock"""
# Status is OK
context.status_code = 200
# Generate nonce
self.nonce = hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()
# Prepare API info to send to agent
api_publickey = self.api_gpg.exp_pub_key()
self.api_gpg.set_email()
api_email_addr = self.api_gpg.email_addr
# Encrypt nonce
encrypted_nonce = self.api_gpg.encrypt_data(
self.nonce, self.agent_gpg.fingerprint)
json_response = {
'data': {
'api_email': api_email_addr,
'api_key': api_publickey,
'encrypted_nonce': encrypted_nonce
}
}
# Send data
return json_response
# Validation callback to respond to agent validation post request
def validation_callback(request, context):
"""Validation callback for request mock"""
# Retrieve agent info from received request object
json_data = request.json()
json_dict = json.loads(json_data)
encrypted_nonce = json_dict['encrypted_nonce']
encrypted_sym_key = json_dict['encrypted_sym_key']
# Validate by decrypting the encrypted symmetric key
# then using the symmetric key to decrypt the nonce
# and check if it is the same as the one that was sent
passphrase = self.api_gpg.passphrase
symmetric_key = self.api_gpg.decrypt_data(encrypted_sym_key,
passphrase)
nonce = self.api_gpg.symmetric_decrypt(encrypted_nonce,
symmetric_key)
if nonce == self.nonce:
self.symmetric_key = symmetric_key
context.status_code = 200
else:
context.status_code = 409
return 'Result'
# Encrypted post callback to respond to agent encrypted data
def encrypted_callback(request, context):
"""Encrypted post callback for request mock"""
# Retrieve encrypted data from received request object
json_data = request.json()
json_dict = json.loads(json_data)
encrypted_data = json_dict['encrypted_data']
# Decrypt data
decrypted_data = self.api_gpg.symmetric_decrypt(
encrypted_data, self.symmetric_key)
# Unload
data_dict = json.loads(decrypted_data)
recv_data = data_dict['data']
# Check that decrypted data is the same as the received
# The two dictionaries are hashed then the values are compared
agent_data = data
if len(agent_data) == len(recv_data):
# Data received and decrypted successfully
context.status_code = 202
else:
# Decryption failed
context.status_code = 409
return "Noted"
# Mock each requests
with requests_mock.Mocker() as m:
# Mock agent sending info to API server
m.post('http://127.0.0.6:50505/pattoo/api/v1/agent/key',
text=exchange_post_callback)
# Mock agent receiving API info
m.get('http://127.0.0.6:50505/pattoo/api/v1/agent/key',
json=exchange_get_callback)
# Mock agent validation
m.post('http://127.0.0.6:50505/pattoo/api/v1/agent/validation',
text=validation_callback)
m.post('http://127.0.0.6:50505/pattoo/api/v1/agent/encrypted',
text=encrypted_callback)
# Run function
success = encrypted_agent.post()
# Test
self.assertTrue(success)
# Get agent identifier from data
identifier = agentdata.agent_id
# Save data to cache
phttp._save_data(data, identifier)
# Encrypted purge
encrypted_agent.purge()
# Check that both the post and purge exchanged keys and
# send data
self.assertEqual(m.call_count, 8)
