def _get_api_public_key(session, exchange_url):
"""Use previously established session to get the API server's public key.
Args:
session: Request Session object
exchange_url: URL to use for key exchange
Returns:
result: JSON key data from API server
"""
# Predefine failure response
result = None
status = None
# Get API information
try:
response = session.get(exchange_url)
status = response.status_code
except:
_exception = sys.exc_info()
log_message = ('Key exchange failure')
log.log2exception(1106, _exception, message=log_message)
# Checks that the API sent over information
if status == 200:
# Process API server response
result = response.json()
else:
log_message = (
'Cannot get public key from API server. Status: {}'.format(status))
log.log2info(1057, log_message)
return result
def test_log2exception(self):
"""Testing function log2exception."""
# Test
try:
float(None)
except:
_exception = sys.exc_info()
log.log2exception(self.code, _exception)
def _send_symmetric_key(session, encryption, url, symmetric_key, data):
"""Send symmetric_key to the remote API server.
Args:
session: Request Session object
encryption: Encryption object
url: URL to use for exchanging the symmetric key
symmetric_key: Symmetric key
data: Data to post
Returns:
success: True if successful
"""
# Predefine failure response
success = False
status = None
# Process API server information
api_email = data['api_email']
api_key = data['api_key']
encrypted_nonce = data['encrypted_nonce']
# Import API public key
encryption.pimport(api_key)
api_fingerprint = encryption.fingerprint(api_email)
encryption.trust(api_fingerprint)
# Decrypt nonce
decrypted_nonce = encryption.decrypt(encrypted_nonce)
# Create JSON to post
data_ = json.dumps({
'encrypted_nonce':
encryption.sencrypt(decrypted_nonce, symmetric_key),
'encrypted_sym_key':
encryption.encrypt(symmetric_key, api_fingerprint)
})
# POST data to API
try:
response = session.post(url, json=data_)
status = response.status_code
except:
_exception = sys.exc_info()
log_message = ('Symmetric key exchange failure')
log.log2exception(1098, _exception, message=log_message)
# Check that the transaction was validated
if status == 200:
success = True
else:
log_message = '''\
Cannot exchange symmetric keys with API server. Status: {}'''.format(status)
log.log2info(1099, log_message)
return success
def datapoint(graphql_id):
"""Get translations for the GraphQL ID of a datapoint query.
Args:
graphql_id: GraphQL ID
Returns:
result: DataPoint object
"""
# Initialize key variables
query = '''\
{
datapoint(id: "IDENTIFIER") {
id
idxDatapoint
agent {
agentProgram
agentPolledTarget
idxPairXlateGroup
pairXlateGroup{
id
}
}
glueDatapoint {
edges {
node {
pair {
key
value
}
}
}
}
}
}
'''.replace('IDENTIFIER', graphql_id)
# Get data from API server
data = None
# Get data from remote system
try:
data = get(query)
except:
_exception = sys.exc_info()
log_message = ('Cannot connect to pattoo web API')
log.log2exception(80014, _exception, message=log_message)
# Return
result = DataPoint(data)
return result
def crypt_receive():
"""Receive encrypted data from agent
Args:
None
Returns:
message (str): Reception result
response (int): HTTP response code
"""
# If a symmetric key has already been established, skip
if 'symm_key' not in session:
log_message = 'No session symmetric key'
log.log2info(20171, log_message)
return (log_message, 208)
if request.method == 'POST':
try:
# Get data from agent
data_dict = json.loads(request.get_json(silent=False))
except:
_exception = sys.exc_info()
log_message = 'Client sent corrupted validation JSON data'
log.log2exception(20169, _exception, message=log_message)
return (log_message, 500)
# Symmetrically decrypt data
data = encryption.sdecrypt(data_dict['encrypted_data'],
session['symm_key'])
# Extract posted data and source
try:
final_data = json.loads(data)
except:
_exception = sys.exc_info()
log_message = 'Decrypted data extraction failed'
log.log2exception(20174, _exception, message=log_message)
abort(500, description=log_message)
# Save data
success = _save_data(final_data['data'], final_data['source'])
if bool(success) is False:
abort(500, description='Invalid JSON data received.')
# Return
log_message = 'Decrypted and received'
log.log2info(20184, log_message)
return (log_message, 202)
# Otherwise abort
return ('Proceed to key exchange first', 400)
def re_raise(self):
"""Extend the re_raise method.
Args:
None
Returns:
None
"""
# Log message
log.log2exception(20114, (self._etype, self._evalue, self._etraceback))
# Process traceback
raise self._error_exception.with_traceback(self._etraceback)
def _send_agent_public_key(session, encryption, exchange_url):
"""Send public key to the remote API server.
Args:
session: Request Session object
encryption: Encryption object
exchange_url: URL to use for key exchange
Returns:
success: True is successful
"""
# Predefine failure response
success = False
status = None
# Data for POST
send_data = {
'pattoo_agent_email': encryption.email,
'pattoo_agent_key': encryption.pexport()
}
# Convert dict to str
send_data = json.dumps(send_data)
try:
# Send over data
response = session.post(exchange_url, json=send_data)
status = response.status_code
except:
_exception = sys.exc_info()
log_message = ('Key exchange failure')
log.log2exception(1077, _exception, message=log_message)
# Checks that sent data was accepted
if status in [202, 208]:
success = True
else:
log_message = (
'Cannot send public key to API server. Status: {}'.format(status))
log.log2info(1069, log_message)
return success
def _process_kvps_exception(pattoo_db_records):
"""Get all the key-value pairs found.
Traps any exceptions and return them for processing. Very helpful in
troubleshooting multiprocessing
Args:
pattoo_db_records: List of dicts read from cache files.
Returns:
None
"""
# Initialize key variables
result = []
'''
Sleep for a short random time. We have seen where on very fast systems
SQLAlchemy will hang the creation of multiprocessing subprocesses. The
typical behaviour is the creation of one fewer
pattoo.db._add_engine_pidguard() log messages than agents to process.
These messages correspond to the creation of a subprocess which immediately
invalidates a parent process's DB connection that will cause errors
if used, which provided the clue to the source of the problem.
Though SQLAlchemy isn't used by key_value_pairs. It's added as a
future precaution in case it does.
'''
time.sleep((random.random() / 10) + 0.1)
# Execute
try:
result = get.key_value_pairs(pattoo_db_records)
except Exception as error:
_exception = sys.exc_info()
log.log2exception(20133, _exception)
return ExceptionWrapper(error)
except:
_exception = sys.exc_info()
log.log2exception_die(20111, _exception)
# Return
return result
def _process_data_exception(pattoo_db_records):
"""Insert all data values for an agent into database.
Traps any exceptions and return them for processing. Very helpful in
troubleshooting multiprocessing
Args:
pattoo_db_records: List of dicts read from cache files.
Returns:
None
"""
# Initialize
'''
Sleep for a short random time. We have seen where on very fast systems
SQLAlchemy will hang the creation of multiprocessing subprocesses. The
typical behaviour is the creation of one fewer
pattoo.db._add_engine_pidguard() log messages than agents to process.
These messages correspond to the creation of a subprocess which immediately
invalidates a parent process's DB connection that will cause errors
if used, which provided the clue to the source of the problem.
'''
time.sleep((random.random() / 10) + 0.1)
# Execute
try:
process_db_records(pattoo_db_records)
except Exception as error:
_exception = sys.exc_info()
log.log2exception(20132, _exception)
return ExceptionWrapper(error)
except:
_exception = sys.exc_info()
log.log2exception_die(20109, _exception)
# Return
return None
def key_exchange():
"""Process public key exhange.
Args:
None
Returns:
result: Various responses
"""
# Initialize key variables
required_keys = ['pattoo_agent_email', 'pattoo_agent_key']
# If a symmetric key has already been established, skip
if 'symm_key' in session:
log_message = 'Symmetric key already set.'
log.log2info(20148, log_message)
return (log_message, 208)
# Get data from incoming agent POST
if request.method == 'POST':
try:
# Get data from agent
data_dict = json.loads(request.get_json(silent=False))
except:
_exception = sys.exc_info()
log_message = 'Client sent corrupted JSON data'
log.log2exception(20167, _exception, message=log_message)
return (log_message, 500)
# Check for minimal keys
for key in required_keys:
if key not in data_dict.keys():
log_message = '''\
Required JSON key "{}" missing in key exchange'''.format(key)
log.log2warning(20164, log_message)
abort(404)
# Save email in session
session['email'] = data_dict['pattoo_agent_email']
# Save agent public key in keyring
encryption.pimport(data_dict['pattoo_agent_key'])
return ('Key received', 202)
# Get data from incoming agent POST
if request.method == 'GET':
if 'email' in session:
# Generate server nonce
session['nonce'] = hashlib.sha256(str(
uuid.uuid4()).encode()).hexdigest()
# Retrieve information from session. Set previously in POST
agent_fingerprint = encryption.fingerprint(session['email'])
# Trust agent key
encryption.trust(agent_fingerprint)
# Encrypt api nonce with agent public key
encrypted_nonce = encryption.encrypt(session['nonce'],
agent_fingerprint)
data_dict = {
'api_email': encryption.email,
'api_key': encryption.pexport(),
'encrypted_nonce': encrypted_nonce
}
# Send api email, public key and encrypted nonce
log_message = 'API information sent'
return jsonify(data_dict)
# Otherwise send error message
return ('Send email and key first', 403)
# Return aborted status
abort(400)
def _save_data(data, source):
"""Handle the agent posting route.
Args:
data: Data dict received from agents
Returns:
success: True if successful
"""
# Initialize key variables
success = False
prefix = 'Invalid posted data.'
# Read configuration
config = configuration.ConfigAgentAPId()
cache_dir = config.agent_cache_directory(PATTOO_API_AGENT_NAME)
# Abort if data isn't a list
if isinstance(data, dict) is False:
log_message = '{} Not a dictionary'.format(prefix)
log.log2warning(20024, log_message)
abort(404)
if len(data) != len(CACHE_KEYS):
log_message = ('''\
{} Incorrect length. Expected length of {}'''.format(prefix, len(CACHE_KEYS)))
log.log2warning(20019, log_message)
return success
# Basic integrity check of required JSON fields
for key in data.keys():
if key not in CACHE_KEYS:
log_message = '{} Invalid key'.format(prefix)
log.log2warning(20018, log_message)
return success
# Extract key values from posting
try:
timestamp = data['pattoo_agent_timestamp']
except:
_exception = sys.exc_info()
log_message = ('API Failure')
log.log2exception(20025, _exception, message=log_message)
return success
# Create filename. Add a suffix in the event the source is posting
# frequently.
suffix = str(randrange(100000)).zfill(6)
json_path = ('{}{}{}_{}_{}.json'.format(cache_dir, os.sep, timestamp,
source, suffix))
# Create cache file
try:
with open(json_path, 'w+') as temp_file:
json.dump(data, temp_file)
except Exception as err:
log_message = '{}'.format(err)
log.log2warning(20016, log_message)
return success
except:
_exception = sys.exc_info()
log_message = ('API Failure')
log.log2exception(20017, _exception, message=log_message)
return success
# Return
success = True
return success
def validation():
"""Validate remote agent.
Process:
1) Decrypt the symmetric key received from the agent
2) Decrypting the nonce from agent
3) Verify that nonce is the same as originally sent.
4) Store symmetric key in session to be used for future decryption.
6) Delete the agent public key
Args:
None
Returns:
message (str): Validation response message
response (int): HTTP response code
"""
# If a symmetric key has already been established, skip
if 'symm_key' in session:
log_message = 'Symmetric key already set.'
log.log2info(20165, log_message)
return (log_message, 208)
# If no nonce is set, inform agent to exchange keys
if 'nonce' not in session:
return ('Proceed to key exchange first', 403)
# Get data from incoming agent POST
if request.method == 'POST':
try:
# Get data from agent
data_dict = json.loads(request.get_json(silent=False))
except:
_exception = sys.exc_info()
log_message = 'Client sent corrupted validation JSON data'
log.log2exception(20168, _exception, message=log_message)
return (log_message, 500)
# Decrypt symmetric key
symmetric_key = encryption.decrypt(data_dict['encrypted_sym_key'])
# Symmetrically decrypt nonce
nonce = encryption.sdecrypt(data_dict['encrypted_nonce'],
symmetric_key)
# Checks if the decrypted nonce matches one sent
if nonce != session['nonce']:
log_message = 'Nonce does not match'
log.log2info(20166, log_message)
return (log_message, 401)
# Delete agent public key
encryption.pdelete(encryption.fingerprint(session['email']))
# Session parameter cleanup
session['symm_key'] = symmetric_key
session.pop('email', None)
session.pop('nonce', None)
# Return response
log_message = 'Symmetric key saved'
log.log2info(20173, log_message)
return (log_message, 200)
# Otherwise abort
abort(404)
async def _serial_poller_async(tpp):
"""Poll OPCUA agent data.
Args:
tpp: TargetDataPoints object
Returns:
target_datapoints: TargetDataPoints object
"""
# Initialize key variables
connected = False
# Test for validity
if isinstance(tpp, TargetPollingPoints) is False:
return None
if isinstance(tpp.target, OPCUAauth) is False:
return None
if tpp.valid is False:
return None
# Create URL for polling
ip_target = tpp.target.ip_target
ip_port = tpp.target.ip_port
username = tpp.target.username
password = tpp.target.password
url = 'opc.tcp://{}:{}'.format(ip_target, ip_port)
# Intialize data gathering
target_datapoints = TargetDataPoints(ip_target)
# Create a client object to connect to OPCUA server
client = Client(url=url)
client.set_user(username)
client.set_password(password)
# Connect
try:
await client.connect()
connected = True
except:
log_message = (
'Authentication for polling target {} is incorrect'.format(url))
log.log2warning(51011, log_message)
pass
if connected is True:
for point in tpp.data:
# Make sure we have the right data type
if isinstance(point, PollingPoint) is False:
log_message = ('''\
Invalid polling point {} for OPC UA URL {}'''.format(point, url))
log.log2info(51012, log_message)
continue
# Get data
address = point.address
try:
node = client.get_node(address)
value = await node.read_value()
except BadNodeIdUnknown:
log_message = ('''\
OPC UA node {} not found on server {}'''.format(address, url))
log.log2warning(51015, log_message)
continue
except:
_exception = sys.exc_info()
log_message = ('OPC UA server communication error')
log.log2exception(51014, _exception, message=log_message)
log_message = ('''\
Cannot get value from polling point {} for OPC UA URL {}\
'''.format(address, url))
log.log2info(51013, log_message)
continue
# Create datapoint
if bool(point.multiplier) is True:
if is_numeric(value) is True and (is_numeric(point.multiplier)
is True):
value = value * point.multiplier
else:
value = 0
datapoint = DataPoint(address, value)
datapoint.add(DataPointMetadata('OPCUA Server', ip_target))
target_datapoints.add(datapoint)
# Disconnect client
await client.disconnect()
return target_datapoints
def datapoints_agent(graphql_id, screen=None):
"""Get translations for the GraphQL ID of a datapointAgent query.
Args:
graphql_id: GraphQL ID
screen: GraphQL filter for screening results
Returns:
result: DataPoint object
"""
# Initialize key variables
query = '''\
{
agent(id: "IDENTIFIER") {
datapointAgent SCREEN {
edges {
cursor
node {
id
idxDatapoint
idxAgent
agent {
agentProgram
agentPolledTarget
idxPairXlateGroup
pairXlateGroup{
id
}
}
glueDatapoint {
edges {
node {
pair {
key
value
}
}
}
}
}
}
pageInfo {
startCursor
endCursor
hasNextPage
hasPreviousPage
}
}
}
}
'''.replace('IDENTIFIER', graphql_id)
if screen is None:
query = query.replace('SCREEN', '')
else:
query = query.replace('SCREEN', screen)
# Get data from API server
data = None
# Get data from remote system
try:
data = get(query)
except:
_exception = sys.exc_info()
log_message = ('Cannot connect to pattoo web API')
log.log2exception(80016, _exception, message=log_message)
# Return
result = DataPointsAgent(data)
return result
def crypt_receive():
"""Receive encrypted data from agent
Args:
None
Returns:
message (str): Reception result
response (int): HTTP response code
"""
# Read configuration
config = Config()
cache_dir = config.agent_cache_directory(PATTOO_API_AGENT_NAME)
try:
# Retrieves Pgpier class
gpg = get_gnupg(PATTOO_API_AGENT_NAME, config)
#Sets key ID
gpg.set_keyid()
# Checks if a Pgpier object exists
if gpg is None:
raise Exception('Could not retrieve Pgpier for {}'.format(
PATTOO_API_AGENT_NAME))
except Exception as e:
response = 500
message = 'Server error'
log_msg = 'Could not retrieve Pgpier: >>>{}<<<'.format(e)
log.log2warning(20175, log_msg)
return message, response
# Predefined error message and response
response = 400
message = 'Proceed to key exchange first'
# Block connection if a symmetric key was not stored
if 'symm_key' not in session:
message = 'No symmetric key'
response = 403
return message, response
if request.method == 'POST':
# Get data from agent
data_json = request.get_json(silent=False)
data_dict = json.loads(data_json)
# Retrieved symmetrically encrypted data
encrypted_data = data_dict['encrypted_data']
# Symmetrically decrypt data
data = gpg.symmetric_decrypt(encrypted_data, session['symm_key'])
# Initialize key variables
prefix = 'Invalid posted data.'
posted_data = None
source = None
# Extract posted data and source
try:
data_extract = json.loads(data)
posted_data = data_extract['data']
source = data_extract['source']
except Exception as e:
log_message = 'Decrypted data extraction failed: {}'\
.format(e)
log.log2warning(20176, log_message)
log_message = 'Decrypted data extraction successful'
log.log2info(20177, log_message)
# Abort if posted_data isn't a list
if isinstance(posted_data, dict) is False:
log_message = '{} Not a dictionary'.format(prefix)
log.log2warning(20178, log_message)
abort(404)
if len(posted_data) != len(CACHE_KEYS):
log_message = ('''{} Incorrect length. Expected length of {}
'''.format(prefix, len(CACHE_KEYS)))
log.log2warning(20179, log_message)
abort(404)
for key in posted_data.keys():
if key not in CACHE_KEYS:
log_message = '{} Invalid key'.format(prefix)
log.log2warning(20180, log_message)
abort(404)
# Extract key values from posting
try:
timestamp = posted_data['pattoo_agent_timestamp']
except:
_exception = sys.exc_info()
log_message = ('API Failure')
log.log2exception(20181, _exception, message=log_message)
abort(404)
# Create filename. Add a suffix in the event the source is posting
# frequently.
suffix = str(randrange(100000)).zfill(6)
json_path = ('{}{}{}_{}_{}.json'.format(cache_dir, os.sep, timestamp,
source, suffix))
# Create cache file
try:
with open(json_path, 'w+') as temp_file:
json.dump(posted_data, temp_file)
except Exception as err:
log_message = '{}'.format(err)
log.log2warning(20182, log_message)
abort(404)
except:
_exception = sys.exc_info()
log_message = ('API Failure')
log.log2exception(20183, _exception, message=log_message)
abort(404)
# Return
message = 'Decrypted and received'
response = 202
log.log2info(20184, message)
return message, response
def encrypted_post(metadata, save=True):
"""Post encrypted data to the API server.
First, the data is checked for its validity. Sencondly,
the data and agent ID is stored in a dictionary with
the key value pairs. The dictionary is converted to a
string so that is can be encrypted. The encrypted data
is then paired with a key, as a dictionary, distinguishing
the data as encrypted. The dictionary is then converted
to a string so it can be added to the request method
as json. A response from the API server tells if the data
was received and decrypted successfully.
Args:
metadata: _EncrypedPost object where:
encryption: encrypt.Encryption object
session: Requests session object
symmetric_key: Symmetric key
encryption_url: API URL to post the data to
data: Data to post as a dict
identifier: Agent identifier
save: If True, save data to cache if API server is inaccessible
Returns:
success: True if successful
"""
# Initialize key variables
success = False
status = None
# Fail if nothing to post
if isinstance(metadata.data, dict) is False or bool(
metadata.data) is False:
return success
# Prepare data for posting
data = json.dumps({'data': metadata.data, 'source': metadata.identifier})
# Symmetrically encrypt data
encrypted_data = metadata.encryption.sencrypt(data, metadata.symmetric_key)
# Post data save to cache if this fails
try:
response = metadata.session.post(
metadata.encryption_url,
json=json.dumps({'encrypted_data': encrypted_data}))
status = response.status_code
except:
_exception = sys.exc_info()
log_message = ('Encrypted posting failure')
log.log2exception(1075, _exception, message=log_message)
if save is True:
# Save data to cache
_save_data(metadata.data, metadata.identifier)
else:
# Proceed normally if there is a failure.
# This will be logged later
pass
# Checks if data was posted successfully
if status == 202:
log_message = 'Posted to API. Response "{}" from URL: "{}"'.format(
status, metadata.encryption_url)
log.log2debug(1059, log_message)
# The data was accepted successfully
success = True
else:
log_message = 'Error posting. Response "{}" from URL: "{}"'.format(
status, metadata.encryption_url)
log.log2warning(1058, log_message)
return success
def receive(source):
"""Handle the agent posting route.
Args:
source: Unique Identifier of an pattoo agent
Returns:
Text response of Received
"""
# Initialize key variables
prefix = 'Invalid posted data.'
# Read configuration
config = Config()
cache_dir = config.agent_cache_directory(PATTOO_API_AGENT_NAME)
# Get JSON from incoming agent POST
try:
posted_data = request.json
except:
# Don't crash if we cannot convert JSON
abort(404)
# Abort if posted_data isn't a list
if isinstance(posted_data, dict) is False:
log_message = '{} Not a dictionary'.format(prefix)
log.log2warning(20024, log_message)
abort(404)
if len(posted_data) != len(CACHE_KEYS):
log_message = ('''\
{} Incorrect length. Expected length of {}'''.format(prefix, len(CACHE_KEYS)))
log.log2warning(20019, log_message)
abort(404)
for key in posted_data.keys():
if key not in CACHE_KEYS:
log_message = '{} Invalid key'.format(prefix)
log.log2warning(20018, log_message)
abort(404)
# Extract key values from posting
try:
timestamp = posted_data['pattoo_agent_timestamp']
except:
_exception = sys.exc_info()
log_message = ('API Failure')
log.log2exception(20025, _exception, message=log_message)
abort(404)
# Create filename. Add a suffix in the event the source is posting
# frequently.
suffix = str(randrange(100000)).zfill(6)
json_path = ('{}{}{}_{}_{}.json'.format(cache_dir, os.sep, timestamp,
source, suffix))
# Create cache file
try:
with open(json_path, 'w+') as temp_file:
json.dump(posted_data, temp_file)
except Exception as err:
log_message = '{}'.format(err)
log.log2warning(20016, log_message)
abort(404)
except:
_exception = sys.exc_info()
log_message = ('API Failure')
log.log2exception(20017, _exception, message=log_message)
abort(404)
# Return
return 'OK'
def post(url, data, identifier, save=True):
"""Post data to central server.
Args:
url: URL to receive posted data
identifier: Unique identifier for the source of the data. (AgentID)
data: Data dict to post. If None, then uses self._post_data (
Used for testing and cache purging)
save: When True, save data to cache directory if posting fails
Returns:
success: True: if successful
"""
# Initialize key variables
success = False
response = False
# Fail if nothing to post
if isinstance(data, dict) is False or bool(data) is False:
return success
# Post data save to cache if this fails
try:
result = requests.post(url, json=data)
response = True
except:
_exception = sys.exc_info()
log_message = ('Data posting failure')
log.log2exception(1097, _exception, message=log_message)
if save is True:
# Save data to cache
_save_data(data, identifier)
else:
# Proceed normally if there is a failure.
# This will be logged later
pass
# Define success
if response is True:
if result.status_code == 200:
success = True
else:
log_message = ('''\
HTTP {} error for identifier "{}" posted to server {}\
'''.format(result.status_code, identifier, url))
log.log2warning(1017, log_message)
# Save data to cache, remote webserver isn't
# working properly
_save_data(data, identifier)
# Log message
if success is True:
log_message = ('''\
Data for identifier "{}" posted to server {}\
'''.format(identifier, url))
log.log2debug(1027, log_message)
else:
log_message = ('''\
Data for identifier "{}" failed to post to server {}\
'''.format(identifier, url))
log.log2warning(1028, log_message)
# Return
return success
