def _get_api_public_key(session, exchange_url): """Use previously established session to get the API server's public key. Args: session: Request Session object exchange_url: URL to use for key exchange Returns: result: JSON key data from API server """ # Predefine failure response result = None status = None # Get API information try: response = session.get(exchange_url) status = response.status_code except: _exception = sys.exc_info() log_message = ('Key exchange failure') log.log2exception(1106, _exception, message=log_message) # Checks that the API sent over information if status == 200: # Process API server response result = response.json() else: log_message = ( 'Cannot get public key from API server. Status: {}'.format(status)) log.log2info(1057, log_message) return result
def test_log2exception(self): """Testing function log2exception.""" # Test try: float(None) except: _exception = sys.exc_info() log.log2exception(self.code, _exception)
def _send_symmetric_key(session, encryption, url, symmetric_key, data): """Send symmetric_key to the remote API server. Args: session: Request Session object encryption: Encryption object url: URL to use for exchanging the symmetric key symmetric_key: Symmetric key data: Data to post Returns: success: True if successful """ # Predefine failure response success = False status = None # Process API server information api_email = data['api_email'] api_key = data['api_key'] encrypted_nonce = data['encrypted_nonce'] # Import API public key encryption.pimport(api_key) api_fingerprint = encryption.fingerprint(api_email) encryption.trust(api_fingerprint) # Decrypt nonce decrypted_nonce = encryption.decrypt(encrypted_nonce) # Create JSON to post data_ = json.dumps({ 'encrypted_nonce': encryption.sencrypt(decrypted_nonce, symmetric_key), 'encrypted_sym_key': encryption.encrypt(symmetric_key, api_fingerprint) }) # POST data to API try: response = session.post(url, json=data_) status = response.status_code except: _exception = sys.exc_info() log_message = ('Symmetric key exchange failure') log.log2exception(1098, _exception, message=log_message) # Check that the transaction was validated if status == 200: success = True else: log_message = '''\ Cannot exchange symmetric keys with API server. Status: {}'''.format(status) log.log2info(1099, log_message) return success
def datapoint(graphql_id): """Get translations for the GraphQL ID of a datapoint query. Args: graphql_id: GraphQL ID Returns: result: DataPoint object """ # Initialize key variables query = '''\ { datapoint(id: "IDENTIFIER") { id idxDatapoint agent { agentProgram agentPolledTarget idxPairXlateGroup pairXlateGroup{ id } } glueDatapoint { edges { node { pair { key value } } } } } } '''.replace('IDENTIFIER', graphql_id) # Get data from API server data = None # Get data from remote system try: data = get(query) except: _exception = sys.exc_info() log_message = ('Cannot connect to pattoo web API') log.log2exception(80014, _exception, message=log_message) # Return result = DataPoint(data) return result
def crypt_receive(): """Receive encrypted data from agent Args: None Returns: message (str): Reception result response (int): HTTP response code """ # If a symmetric key has already been established, skip if 'symm_key' not in session: log_message = 'No session symmetric key' log.log2info(20171, log_message) return (log_message, 208) if request.method == 'POST': try: # Get data from agent data_dict = json.loads(request.get_json(silent=False)) except: _exception = sys.exc_info() log_message = 'Client sent corrupted validation JSON data' log.log2exception(20169, _exception, message=log_message) return (log_message, 500) # Symmetrically decrypt data data = encryption.sdecrypt(data_dict['encrypted_data'], session['symm_key']) # Extract posted data and source try: final_data = json.loads(data) except: _exception = sys.exc_info() log_message = 'Decrypted data extraction failed' log.log2exception(20174, _exception, message=log_message) abort(500, description=log_message) # Save data success = _save_data(final_data['data'], final_data['source']) if bool(success) is False: abort(500, description='Invalid JSON data received.') # Return log_message = 'Decrypted and received' log.log2info(20184, log_message) return (log_message, 202) # Otherwise abort return ('Proceed to key exchange first', 400)
def re_raise(self): """Extend the re_raise method. Args: None Returns: None """ # Log message log.log2exception(20114, (self._etype, self._evalue, self._etraceback)) # Process traceback raise self._error_exception.with_traceback(self._etraceback)
def _send_agent_public_key(session, encryption, exchange_url): """Send public key to the remote API server. Args: session: Request Session object encryption: Encryption object exchange_url: URL to use for key exchange Returns: success: True is successful """ # Predefine failure response success = False status = None # Data for POST send_data = { 'pattoo_agent_email': encryption.email, 'pattoo_agent_key': encryption.pexport() } # Convert dict to str send_data = json.dumps(send_data) try: # Send over data response = session.post(exchange_url, json=send_data) status = response.status_code except: _exception = sys.exc_info() log_message = ('Key exchange failure') log.log2exception(1077, _exception, message=log_message) # Checks that sent data was accepted if status in [202, 208]: success = True else: log_message = ( 'Cannot send public key to API server. Status: {}'.format(status)) log.log2info(1069, log_message) return success
def _process_kvps_exception(pattoo_db_records): """Get all the key-value pairs found. Traps any exceptions and return them for processing. Very helpful in troubleshooting multiprocessing Args: pattoo_db_records: List of dicts read from cache files. Returns: None """ # Initialize key variables result = [] ''' Sleep for a short random time. We have seen where on very fast systems SQLAlchemy will hang the creation of multiprocessing subprocesses. The typical behaviour is the creation of one fewer pattoo.db._add_engine_pidguard() log messages than agents to process. These messages correspond to the creation of a subprocess which immediately invalidates a parent process's DB connection that will cause errors if used, which provided the clue to the source of the problem. Though SQLAlchemy isn't used by key_value_pairs. It's added as a future precaution in case it does. ''' time.sleep((random.random() / 10) + 0.1) # Execute try: result = get.key_value_pairs(pattoo_db_records) except Exception as error: _exception = sys.exc_info() log.log2exception(20133, _exception) return ExceptionWrapper(error) except: _exception = sys.exc_info() log.log2exception_die(20111, _exception) # Return return result
def _process_data_exception(pattoo_db_records): """Insert all data values for an agent into database. Traps any exceptions and return them for processing. Very helpful in troubleshooting multiprocessing Args: pattoo_db_records: List of dicts read from cache files. Returns: None """ # Initialize ''' Sleep for a short random time. We have seen where on very fast systems SQLAlchemy will hang the creation of multiprocessing subprocesses. The typical behaviour is the creation of one fewer pattoo.db._add_engine_pidguard() log messages than agents to process. These messages correspond to the creation of a subprocess which immediately invalidates a parent process's DB connection that will cause errors if used, which provided the clue to the source of the problem. ''' time.sleep((random.random() / 10) + 0.1) # Execute try: process_db_records(pattoo_db_records) except Exception as error: _exception = sys.exc_info() log.log2exception(20132, _exception) return ExceptionWrapper(error) except: _exception = sys.exc_info() log.log2exception_die(20109, _exception) # Return return None
def key_exchange(): """Process public key exhange. Args: None Returns: result: Various responses """ # Initialize key variables required_keys = ['pattoo_agent_email', 'pattoo_agent_key'] # If a symmetric key has already been established, skip if 'symm_key' in session: log_message = 'Symmetric key already set.' log.log2info(20148, log_message) return (log_message, 208) # Get data from incoming agent POST if request.method == 'POST': try: # Get data from agent data_dict = json.loads(request.get_json(silent=False)) except: _exception = sys.exc_info() log_message = 'Client sent corrupted JSON data' log.log2exception(20167, _exception, message=log_message) return (log_message, 500) # Check for minimal keys for key in required_keys: if key not in data_dict.keys(): log_message = '''\ Required JSON key "{}" missing in key exchange'''.format(key) log.log2warning(20164, log_message) abort(404) # Save email in session session['email'] = data_dict['pattoo_agent_email'] # Save agent public key in keyring encryption.pimport(data_dict['pattoo_agent_key']) return ('Key received', 202) # Get data from incoming agent POST if request.method == 'GET': if 'email' in session: # Generate server nonce session['nonce'] = hashlib.sha256(str( uuid.uuid4()).encode()).hexdigest() # Retrieve information from session. Set previously in POST agent_fingerprint = encryption.fingerprint(session['email']) # Trust agent key encryption.trust(agent_fingerprint) # Encrypt api nonce with agent public key encrypted_nonce = encryption.encrypt(session['nonce'], agent_fingerprint) data_dict = { 'api_email': encryption.email, 'api_key': encryption.pexport(), 'encrypted_nonce': encrypted_nonce } # Send api email, public key and encrypted nonce log_message = 'API information sent' return jsonify(data_dict) # Otherwise send error message return ('Send email and key first', 403) # Return aborted status abort(400)
def _save_data(data, source): """Handle the agent posting route. Args: data: Data dict received from agents Returns: success: True if successful """ # Initialize key variables success = False prefix = 'Invalid posted data.' # Read configuration config = configuration.ConfigAgentAPId() cache_dir = config.agent_cache_directory(PATTOO_API_AGENT_NAME) # Abort if data isn't a list if isinstance(data, dict) is False: log_message = '{} Not a dictionary'.format(prefix) log.log2warning(20024, log_message) abort(404) if len(data) != len(CACHE_KEYS): log_message = ('''\ {} Incorrect length. Expected length of {}'''.format(prefix, len(CACHE_KEYS))) log.log2warning(20019, log_message) return success # Basic integrity check of required JSON fields for key in data.keys(): if key not in CACHE_KEYS: log_message = '{} Invalid key'.format(prefix) log.log2warning(20018, log_message) return success # Extract key values from posting try: timestamp = data['pattoo_agent_timestamp'] except: _exception = sys.exc_info() log_message = ('API Failure') log.log2exception(20025, _exception, message=log_message) return success # Create filename. Add a suffix in the event the source is posting # frequently. suffix = str(randrange(100000)).zfill(6) json_path = ('{}{}{}_{}_{}.json'.format(cache_dir, os.sep, timestamp, source, suffix)) # Create cache file try: with open(json_path, 'w+') as temp_file: json.dump(data, temp_file) except Exception as err: log_message = '{}'.format(err) log.log2warning(20016, log_message) return success except: _exception = sys.exc_info() log_message = ('API Failure') log.log2exception(20017, _exception, message=log_message) return success # Return success = True return success
def validation(): """Validate remote agent. Process: 1) Decrypt the symmetric key received from the agent 2) Decrypting the nonce from agent 3) Verify that nonce is the same as originally sent. 4) Store symmetric key in session to be used for future decryption. 6) Delete the agent public key Args: None Returns: message (str): Validation response message response (int): HTTP response code """ # If a symmetric key has already been established, skip if 'symm_key' in session: log_message = 'Symmetric key already set.' log.log2info(20165, log_message) return (log_message, 208) # If no nonce is set, inform agent to exchange keys if 'nonce' not in session: return ('Proceed to key exchange first', 403) # Get data from incoming agent POST if request.method == 'POST': try: # Get data from agent data_dict = json.loads(request.get_json(silent=False)) except: _exception = sys.exc_info() log_message = 'Client sent corrupted validation JSON data' log.log2exception(20168, _exception, message=log_message) return (log_message, 500) # Decrypt symmetric key symmetric_key = encryption.decrypt(data_dict['encrypted_sym_key']) # Symmetrically decrypt nonce nonce = encryption.sdecrypt(data_dict['encrypted_nonce'], symmetric_key) # Checks if the decrypted nonce matches one sent if nonce != session['nonce']: log_message = 'Nonce does not match' log.log2info(20166, log_message) return (log_message, 401) # Delete agent public key encryption.pdelete(encryption.fingerprint(session['email'])) # Session parameter cleanup session['symm_key'] = symmetric_key session.pop('email', None) session.pop('nonce', None) # Return response log_message = 'Symmetric key saved' log.log2info(20173, log_message) return (log_message, 200) # Otherwise abort abort(404)
async def _serial_poller_async(tpp): """Poll OPCUA agent data. Args: tpp: TargetDataPoints object Returns: target_datapoints: TargetDataPoints object """ # Initialize key variables connected = False # Test for validity if isinstance(tpp, TargetPollingPoints) is False: return None if isinstance(tpp.target, OPCUAauth) is False: return None if tpp.valid is False: return None # Create URL for polling ip_target = tpp.target.ip_target ip_port = tpp.target.ip_port username = tpp.target.username password = tpp.target.password url = 'opc.tcp://{}:{}'.format(ip_target, ip_port) # Intialize data gathering target_datapoints = TargetDataPoints(ip_target) # Create a client object to connect to OPCUA server client = Client(url=url) client.set_user(username) client.set_password(password) # Connect try: await client.connect() connected = True except: log_message = ( 'Authentication for polling target {} is incorrect'.format(url)) log.log2warning(51011, log_message) pass if connected is True: for point in tpp.data: # Make sure we have the right data type if isinstance(point, PollingPoint) is False: log_message = ('''\ Invalid polling point {} for OPC UA URL {}'''.format(point, url)) log.log2info(51012, log_message) continue # Get data address = point.address try: node = client.get_node(address) value = await node.read_value() except BadNodeIdUnknown: log_message = ('''\ OPC UA node {} not found on server {}'''.format(address, url)) log.log2warning(51015, log_message) continue except: _exception = sys.exc_info() log_message = ('OPC UA server communication error') log.log2exception(51014, _exception, message=log_message) log_message = ('''\ Cannot get value from polling point {} for OPC UA URL {}\ '''.format(address, url)) log.log2info(51013, log_message) continue # Create datapoint if bool(point.multiplier) is True: if is_numeric(value) is True and (is_numeric(point.multiplier) is True): value = value * point.multiplier else: value = 0 datapoint = DataPoint(address, value) datapoint.add(DataPointMetadata('OPCUA Server', ip_target)) target_datapoints.add(datapoint) # Disconnect client await client.disconnect() return target_datapoints
def datapoints_agent(graphql_id, screen=None): """Get translations for the GraphQL ID of a datapointAgent query. Args: graphql_id: GraphQL ID screen: GraphQL filter for screening results Returns: result: DataPoint object """ # Initialize key variables query = '''\ { agent(id: "IDENTIFIER") { datapointAgent SCREEN { edges { cursor node { id idxDatapoint idxAgent agent { agentProgram agentPolledTarget idxPairXlateGroup pairXlateGroup{ id } } glueDatapoint { edges { node { pair { key value } } } } } } pageInfo { startCursor endCursor hasNextPage hasPreviousPage } } } } '''.replace('IDENTIFIER', graphql_id) if screen is None: query = query.replace('SCREEN', '') else: query = query.replace('SCREEN', screen) # Get data from API server data = None # Get data from remote system try: data = get(query) except: _exception = sys.exc_info() log_message = ('Cannot connect to pattoo web API') log.log2exception(80016, _exception, message=log_message) # Return result = DataPointsAgent(data) return result
def crypt_receive(): """Receive encrypted data from agent Args: None Returns: message (str): Reception result response (int): HTTP response code """ # Read configuration config = Config() cache_dir = config.agent_cache_directory(PATTOO_API_AGENT_NAME) try: # Retrieves Pgpier class gpg = get_gnupg(PATTOO_API_AGENT_NAME, config) #Sets key ID gpg.set_keyid() # Checks if a Pgpier object exists if gpg is None: raise Exception('Could not retrieve Pgpier for {}'.format( PATTOO_API_AGENT_NAME)) except Exception as e: response = 500 message = 'Server error' log_msg = 'Could not retrieve Pgpier: >>>{}<<<'.format(e) log.log2warning(20175, log_msg) return message, response # Predefined error message and response response = 400 message = 'Proceed to key exchange first' # Block connection if a symmetric key was not stored if 'symm_key' not in session: message = 'No symmetric key' response = 403 return message, response if request.method == 'POST': # Get data from agent data_json = request.get_json(silent=False) data_dict = json.loads(data_json) # Retrieved symmetrically encrypted data encrypted_data = data_dict['encrypted_data'] # Symmetrically decrypt data data = gpg.symmetric_decrypt(encrypted_data, session['symm_key']) # Initialize key variables prefix = 'Invalid posted data.' posted_data = None source = None # Extract posted data and source try: data_extract = json.loads(data) posted_data = data_extract['data'] source = data_extract['source'] except Exception as e: log_message = 'Decrypted data extraction failed: {}'\ .format(e) log.log2warning(20176, log_message) log_message = 'Decrypted data extraction successful' log.log2info(20177, log_message) # Abort if posted_data isn't a list if isinstance(posted_data, dict) is False: log_message = '{} Not a dictionary'.format(prefix) log.log2warning(20178, log_message) abort(404) if len(posted_data) != len(CACHE_KEYS): log_message = ('''{} Incorrect length. Expected length of {} '''.format(prefix, len(CACHE_KEYS))) log.log2warning(20179, log_message) abort(404) for key in posted_data.keys(): if key not in CACHE_KEYS: log_message = '{} Invalid key'.format(prefix) log.log2warning(20180, log_message) abort(404) # Extract key values from posting try: timestamp = posted_data['pattoo_agent_timestamp'] except: _exception = sys.exc_info() log_message = ('API Failure') log.log2exception(20181, _exception, message=log_message) abort(404) # Create filename. Add a suffix in the event the source is posting # frequently. suffix = str(randrange(100000)).zfill(6) json_path = ('{}{}{}_{}_{}.json'.format(cache_dir, os.sep, timestamp, source, suffix)) # Create cache file try: with open(json_path, 'w+') as temp_file: json.dump(posted_data, temp_file) except Exception as err: log_message = '{}'.format(err) log.log2warning(20182, log_message) abort(404) except: _exception = sys.exc_info() log_message = ('API Failure') log.log2exception(20183, _exception, message=log_message) abort(404) # Return message = 'Decrypted and received' response = 202 log.log2info(20184, message) return message, response
def encrypted_post(metadata, save=True): """Post encrypted data to the API server. First, the data is checked for its validity. Sencondly, the data and agent ID is stored in a dictionary with the key value pairs. The dictionary is converted to a string so that is can be encrypted. The encrypted data is then paired with a key, as a dictionary, distinguishing the data as encrypted. The dictionary is then converted to a string so it can be added to the request method as json. A response from the API server tells if the data was received and decrypted successfully. Args: metadata: _EncrypedPost object where: encryption: encrypt.Encryption object session: Requests session object symmetric_key: Symmetric key encryption_url: API URL to post the data to data: Data to post as a dict identifier: Agent identifier save: If True, save data to cache if API server is inaccessible Returns: success: True if successful """ # Initialize key variables success = False status = None # Fail if nothing to post if isinstance(metadata.data, dict) is False or bool( metadata.data) is False: return success # Prepare data for posting data = json.dumps({'data': metadata.data, 'source': metadata.identifier}) # Symmetrically encrypt data encrypted_data = metadata.encryption.sencrypt(data, metadata.symmetric_key) # Post data save to cache if this fails try: response = metadata.session.post( metadata.encryption_url, json=json.dumps({'encrypted_data': encrypted_data})) status = response.status_code except: _exception = sys.exc_info() log_message = ('Encrypted posting failure') log.log2exception(1075, _exception, message=log_message) if save is True: # Save data to cache _save_data(metadata.data, metadata.identifier) else: # Proceed normally if there is a failure. # This will be logged later pass # Checks if data was posted successfully if status == 202: log_message = 'Posted to API. Response "{}" from URL: "{}"'.format( status, metadata.encryption_url) log.log2debug(1059, log_message) # The data was accepted successfully success = True else: log_message = 'Error posting. Response "{}" from URL: "{}"'.format( status, metadata.encryption_url) log.log2warning(1058, log_message) return success
def receive(source): """Handle the agent posting route. Args: source: Unique Identifier of an pattoo agent Returns: Text response of Received """ # Initialize key variables prefix = 'Invalid posted data.' # Read configuration config = Config() cache_dir = config.agent_cache_directory(PATTOO_API_AGENT_NAME) # Get JSON from incoming agent POST try: posted_data = request.json except: # Don't crash if we cannot convert JSON abort(404) # Abort if posted_data isn't a list if isinstance(posted_data, dict) is False: log_message = '{} Not a dictionary'.format(prefix) log.log2warning(20024, log_message) abort(404) if len(posted_data) != len(CACHE_KEYS): log_message = ('''\ {} Incorrect length. Expected length of {}'''.format(prefix, len(CACHE_KEYS))) log.log2warning(20019, log_message) abort(404) for key in posted_data.keys(): if key not in CACHE_KEYS: log_message = '{} Invalid key'.format(prefix) log.log2warning(20018, log_message) abort(404) # Extract key values from posting try: timestamp = posted_data['pattoo_agent_timestamp'] except: _exception = sys.exc_info() log_message = ('API Failure') log.log2exception(20025, _exception, message=log_message) abort(404) # Create filename. Add a suffix in the event the source is posting # frequently. suffix = str(randrange(100000)).zfill(6) json_path = ('{}{}{}_{}_{}.json'.format(cache_dir, os.sep, timestamp, source, suffix)) # Create cache file try: with open(json_path, 'w+') as temp_file: json.dump(posted_data, temp_file) except Exception as err: log_message = '{}'.format(err) log.log2warning(20016, log_message) abort(404) except: _exception = sys.exc_info() log_message = ('API Failure') log.log2exception(20017, _exception, message=log_message) abort(404) # Return return 'OK'
def post(url, data, identifier, save=True): """Post data to central server. Args: url: URL to receive posted data identifier: Unique identifier for the source of the data. (AgentID) data: Data dict to post. If None, then uses self._post_data ( Used for testing and cache purging) save: When True, save data to cache directory if posting fails Returns: success: True: if successful """ # Initialize key variables success = False response = False # Fail if nothing to post if isinstance(data, dict) is False or bool(data) is False: return success # Post data save to cache if this fails try: result = requests.post(url, json=data) response = True except: _exception = sys.exc_info() log_message = ('Data posting failure') log.log2exception(1097, _exception, message=log_message) if save is True: # Save data to cache _save_data(data, identifier) else: # Proceed normally if there is a failure. # This will be logged later pass # Define success if response is True: if result.status_code == 200: success = True else: log_message = ('''\ HTTP {} error for identifier "{}" posted to server {}\ '''.format(result.status_code, identifier, url)) log.log2warning(1017, log_message) # Save data to cache, remote webserver isn't # working properly _save_data(data, identifier) # Log message if success is True: log_message = ('''\ Data for identifier "{}" posted to server {}\ '''.format(identifier, url)) log.log2debug(1027, log_message) else: log_message = ('''\ Data for identifier "{}" failed to post to server {}\ '''.format(identifier, url)) log.log2warning(1028, log_message) # Return return success