def set_configuration(self, configuration): """ Set the runtime configuration. @type configuration: cuivre.client.configuration.Configuration @param configuration: The new configuration. """ self.conf = configuration try: self.check_data_directory() except DataDirectoryExistenceException: try: self.create_data_directory() except OSError: raise except (DataDirectoryReadException, DataDirectoryWriteException): raise self.database = Database(self.conf.db_file_path) try: self.conf.check_config_file() except cuivre.client.configuration.ConfigFileExistenceException: try: self.conf.write_configuration_file() except cuivre.client.configuration.ConfigFileWriteException: raise except cuivre.client.configuration.ConfigFileReadException: raise except cuivre.client.configuration.ConfigFileWriteException: pass try: self.database.check_database_file() except cuivre.client.database.ExistenceException: try: self.database.create_database() except cuivre.client.database.CreationException: raise except ( cuivre.client.database.ReadException, cuivre.client.database.WriteException ): raise
class Client(object): """Implements information checking and retrieval from Convergence notaries.""" def __init__(self, configuration=None): """ Initialize the client's configuration. @type configuration: cuivre.client.configuration.Configuration @param configuration: The custom configuration to use, or None to make the client create a default one. """ if configuration is None: configuration = Configuration() self.set_configuration(configuration) self.callbacks = {} def set_configuration(self, configuration): """ Set the runtime configuration. @type configuration: cuivre.client.configuration.Configuration @param configuration: The new configuration. """ self.conf = configuration try: self.check_data_directory() except DataDirectoryExistenceException: try: self.create_data_directory() except OSError: raise except (DataDirectoryReadException, DataDirectoryWriteException): raise self.database = Database(self.conf.db_file_path) try: self.conf.check_config_file() except cuivre.client.configuration.ConfigFileExistenceException: try: self.conf.write_configuration_file() except cuivre.client.configuration.ConfigFileWriteException: raise except cuivre.client.configuration.ConfigFileReadException: raise except cuivre.client.configuration.ConfigFileWriteException: pass try: self.database.check_database_file() except cuivre.client.database.ExistenceException: try: self.database.create_database() except cuivre.client.database.CreationException: raise except ( cuivre.client.database.ReadException, cuivre.client.database.WriteException ): raise def check_data_directory(self): """Check access permissions on the data directory.""" if not os.access(self.conf.data_dir, os.F_OK): raise DataDirectoryExistenceException( 'Configuration directory does not exist.') if not os.access(self.conf.data_dir, os.R_OK): raise DataDirectoryReadException( 'Configuration directory is not readable.') if not os.access(self.conf.data_dir, os.W_OK): raise DataDirectoryWriteException( 'Configuration directory is not writeable.') def create_data_directory(self): """Create the data directory if it doesn't already exist.""" if not os.access(self.conf.data_dir, os.F_OK): try: os.makedirs(self.conf.data_dir, mode=0700) except OSError: raise def set_callback(self, cb_slot, cb): cb = self.callbacks[cb_slot] = cb def get_callback(self, cb_slot): cb = self.callbacks[CB_NOTARY_ENTITY_CONNECTION_CONFIGURATION] def add_notary_entity_from_bundle_file(self, bundle_path): """ Add a notary to the database from a notary bundle file. @type bundle_path: string @param bundle_path: Path to a notary bundle file. @rtype: NotaryBundle @return: A bundle object from the information in the specified file. """ try: return self.database.add_notary_entity_from_bundle_file( file_path=bundle_path) except cuivre.bundle.BrokenBundle: raise except cuivre.client.database.DuplicateNotaryException as e: raise NotaryAlreadyInDatabase(e.msg) def disable_notary(self, notary_id): """ Disable the specified notary. @type notary_id: int @param notary_id: Database id of the notary to disable. """ self.database.disable_notary(notary_id) def enable_notary(self, notary_id): """ Enable the specified notary. @type notary_id: int @param notary_id: Database id of the notary to enable. """ self.database.enable_notary(notary_id) def get_notaries(self): """ Return all the notaries currently in the database. @rtype: dict @return: A dictionary of the notary entities in the database, with notary ids as keys. """ notaries = self.database.get_notaries() return notaries def remove_notary(self, notary_id): """ Remove the specified notary from the database. @type notary_id: int @param notary_id: Database id of the notary to remove. """ self.database.remove_notary_entity(notary_id) def update_notary(self, notary_id, bundle_location): """ Update the notary's information with data from the specified source. @type notary_id: int @param notary_id: Database id of the notary to remove. @type bundle_location: str @param bundle_location: HTTPS URL of the notary's bundle file. """ if bundle_location is not None: r = cuivre.client.HTTPSGet(self) msg = r.get(bundle_location) bundle_text_data = msg.body bundle = cuivre.bundle.NotaryBundle(text=bundle_text_data) new_notary_entity = bundle.notary_entity new_notary_entity.id = notary_id self.database.update_notary_entity(new_notary_entity) return new_notary_entity def update_notaries(self): """Update all the notaries currently in the database.""" notaries = self.database.get_notary_entities() for n in notaries: self.update_notary(n.id, n.bundle_location) def get_certificate_validation_report(self, host, port=cuivre.HTTPS_DEFAULT_PORT, certificate=None): """ Validate the certificate against common verification criteria and the network perspective supplied by the installed Convergence notaries. All the information is compiled in a certificate validation report, recording data and outcome of each verification step, and the overall certificate validation based upon these steps. The fingerprint validation step is cached locally in the database, reducing network latency issues and requests overload on notary servers. @type host: str @param host: The host of the SSL server. @type port: int @param port: The port on which the SSL server is listening. @type certificate: M2Crypto.X509.X509 @param certificate: The certificate to validate. @rtype : CertificateValidationReport @return: A CertificateValidationReport object describing each step of the verification process. """ if port == None: port = cuivre.HTTPS_DEFAULT_PORT report = cuivre.client.certificate_report.CertificateValidationReport( host, port, self, certificate ) fingerprint = report.fingerprint report.compile_ssl_server_report_stanza() if not report.ssl_server_stanza.validation: raise CertificateNotForSSLServer( 'The certificate is not to be used for an SSL server.') report.compile_time_constraints_report_stanza() if not report.time_constraints_stanza.validation: self.database.delete_cache_entry(host, port) if not report.time_constraints_stanza.not_before_validation: raise ValidityNotBeforeError( 'The validity of the certificate is forward in time.') if not report.time_constraints_stanza.not_after_validation: raise ValidityNotAfterError('The certificate has expired.') report.compile_fingerprint_cache_report_stanza() cache = report.fingerprint_cache_stanza if cache.hit and cache.validation: if report.fingerprint_cache_stanza.expiration is not None and \ report.fingerprint_cache_stanza.expiration < report.creation: self.database.delete_cache_entry(host, port) cache.validation = False if not cache.hit: report.compile_convergence_report_stanza() if report.convergence_stanza.validation: self.database.write_cache(fingerprint, host, port, report.not_after, report.creation) report.validate() return report def get_fingerprint_validation_report(self, fingerprint, host, port=cuivre.HTTPS_DEFAULT_PORT): """ Validate the fingerprint against the network perspective supplied by the installed Convergence notaries. All the information is compiled in a fingerprint validation report, recording data and outcome of each notary connection, and the overall fingerprint validation based upon the selected policy. The fingerprint validation step is cached locally in the database, reducing network latency issues and requests overload on notary servers. @type fingerprint: string @param fingerprint: The fingerprint to validate. (Ex.: 00:11:22:...) @type host: str @param host: The host of the SSL server. @type port: int @param port: The port on which the SSL server is listening. @rtype : FingerprintValidationReport @return: A FingerprintValidationReport object describing connections and verification results. """ if port == None: port = cuivre.HTTPS_DEFAULT_PORT report = cuivre.client.fingerprint_report.FingerprintValidationReport( host, port, self, fingerprint ) report.compile_fingerprint_cache_report_stanza() cache = report.fingerprint_cache_stanza if cache.hit and cache.validation: if report.fingerprint_cache_stanza.expiration is not None and \ report.fingerprint_cache_stanza.expiration < report.creation: self.database.delete_cache_entry(host, port) cache.validation = False if not cache.hit: report.compile_convergence_report_stanza() if report.convergence_stanza.validation: self.database.write_cache(fingerprint, host, port, None, report.creation) report.validate() return report def validate_fingerprint(self, fingerprint, host, port, report=None): """ Validate a certificate fingerprint by checking the network perspective of the registered and trusted Convergence notaries, for the specified server host and port. This method is to be called by report objects, during the validation process, which take advantage of the local cache. If called directly in user code, it can be used to compile a new report stanza for the specified fingerprint. @type fingerprint: string @param fingerprint: The fingerprint to validate. (Ex.: 00:11:22:...) @type host: string @param host: The host of the SSL server. @type port: integer @param port: The port on which the SSL server is listening. @type report: ConvergenceValidationReportStanza @param report: The report stanza to compile about this validation. @rtype : ConvergenceValidationReportStanza @return: A ConvergenceValidationReportStanza object describing connections and verification results. """ enabled_notary_entities = self.database.get_notaries(only_enabled=True) if len(enabled_notary_entities) < 1: raise NoEnabledNotariesException('No enabled notaries in the database.') # If we're anonymizing, extract a relay notary entity from the list of # enabled ones. relay_notary_entity = None if self.conf.anonymization: if len(enabled_notary_entities) < 2: raise CannotUseAnonymizedQueries( 'Can\'t use anonymized notary queries, as that needs at least two ' 'notary entities.' ) relay_id = random.choice(enabled_notary_entities.keys()) relay_notary_entity = enabled_notary_entities.pop(relay_id) # Randomly pick at most 'max_notaries' notaries. available_notaries_ids = enabled_notary_entities.keys() random.shuffle(available_notaries_ids) available_notaries_ids = available_notaries_ids[0:3] picked_notaries = {} for i in available_notaries_ids: picked_notaries[i] = enabled_notary_entities.pop(i) del available_notaries_ids del enabled_notary_entities # Create a query context, set its fields to runtime configuration values, # and populate it with information about all spawned connections. ctx = cuivre.client.types.CuivreQueryContext() # Set Convergence protocol parameters. ctx.anonymization = self.conf.anonymization ctx.policy = self.conf.convergence_policy # Set SSL/TLS protocol parameters. ctx.ssl_protocol = self.conf.ssl_protocol ctx.ssl_cipher_list = self.conf.ssl_cipher_list # Set relay notary entity data, if any. if relay_notary_entity: ctx.relay_notary_entity_name = relay_notary_entity.name # Initialize M2Crypto threading support. M2Crypto.threading.init() # Iterate on the selected notary entities. for notary_entity in picked_notaries.itervalues(): cb = self.get_callback(CB_NOTARY_ENTITY_CONNECTION_CONFIGURATION) if cb is not None: cb(notary_entity) # Iterate on the selected notary entity's hosts. for notary_host in notary_entity.hosts.values(): cb = self.get_callback(CB_NOTARY_CONNECTION_CONFIGURATION) if cb is not None: cb(notary_host) # For each connection, create a new notary entity structure with only # one target notary host, thus parallelizing the connections. connection_notary_entity = cuivre.client.types.NotaryEntityData( notary_entity.id, notary_entity.name, notary_entity.region, notary_entity.bundle_location, {notary_host.id : notary_host} ) # Notary connection data with the notary entity's only host, and a # complete relay notary entity, or None if not anonymizing. connection_data = cuivre.client.types.NotaryHostConnectionData( connection_notary_entity, self.conf.notary_connection_timeout, relay_notary_entity, self.conf.relay_connection_timeout ) # Insert the connection data into the query context. ctx.add_notary_to_connection_mapping(connection_data) # Spawn a notary connection thread. t = cuivre.client.notary_connection.NotaryConnectionHandler( ctx, connection_data ) ctx.connections_counters['lock'].acquire() ctx.connections_counters['setup'] += 1 ctx.connections_counters['lock'].release() cb = self.get_callback(CB_NOTARY_CONNECTION_START) if cb is not None: cb(connection_data) t.start() try: while True: events = ctx.epoll['epoll'].poll(1) for fd, event in events: if event & select.EPOLLHUP: ctx.connections_counters['lock'].acquire() ctx.connections_counters['open'] -= 1 ctx.connections_counters['lock'].release() ctx.epoll['lock'].acquire() ctx.epoll['epoll'].unregister(fd) ctx.epoll['lock'].release() elif event & select.EPOLLIN: s = ctx.fds_to_connections['connections'][fd] bit = s.ssl_conn.recv(SOCKET_READ_BUFFER_LENGTH) if bit is not None: s.msg_in += bit rest = None try: rest = s.http_msg_in.parse(s.msg_in) except cuivre.http.BadHTTPMessageFromServer: ctx.epoll['lock'].acquire() ctx.epoll['epoll'].unregister(fd) ctx.epoll['lock'].release() s.ssl_conn.shutdown(socket.SHUT_RDWR) s.ssl_conn.close() ctx.connections_counters['lock'].acquire() ctx.connections_counters['open'] -= 1 ctx.connections_counters['lock'].release() if rest is not None: s.msg_in = rest if s.http_msg_in.parsed_body: ctx.epoll['lock'].acquire() ctx.epoll['epoll'].unregister(fd) ctx.epoll['lock'].release() s.ssl_conn.shutdown(socket.SHUT_RDWR) s.ssl_conn.close() ctx.connections_counters['lock'].acquire() ctx.connections_counters['open'] -= 1 ctx.connections_counters['lock'].release() if event & select.EPOLLRDHUP: if bit is None or \ bit is not None and len(bit) < 1: ctx.epoll['lock'].acquire() ctx.epoll['epoll'].unregister(fd) ctx.epoll['lock'].release() s.ssl_conn.shutdown(socket.SHUT_RDWR) s.ssl_conn.close() ctx.connections_counters['lock'].acquire() ctx.connections_counters['open'] -= 1 ctx.connections_counters['lock'].release() elif event & select.EPOLLOUT: s = ctx.fds_to_connections['connections'][fd] sock = s.ssl_conn # Prepare request message. if s.msg_out == None: post_data = 'fingerprint={}'.format(fingerprint) hr = cuivre.http.HTTPRequestMessage() hr.method = cuivre.http.HTTP_METHOD_POST_STRING hr.resource = '/target/{}+{}'.format(host, port) hr.http_version = cuivre.http.HTTP_VERSION_STRING_10 if ctx.anonymization: hr.headers['Host'] = '{}:{}'.format( s.notary_entity.hosts.values()[0].host, cuivre.CONVERGENCE_PROTOCOL_PORT) else: hr.headers['Host'] = '{}:{}'.format( s.notary_entity.hosts.values()[0].host, s.notary_entity.hosts.values()[0].ssl_port) hr.headers['Content-Type'] = 'application/x-www-form-urlencoded' hr.headers['Connection'] = 'Close' hr.set_body(post_data) s.msg_out = str(hr) # Actually send the request. q = sock.send(s.msg_out) s.msg_out = s.msg_out[q:] ctx.epoll['lock'].acquire() if len(s.msg_out) == 0: ctx.epoll['epoll'].modify(fd, select.EPOLLIN | select.EPOLLRDHUP) elif len(s.msg_out) > 0: ctx.epoll['epoll'].modify(fd, select.EPOLLOUT | select.EPOLLHUP) ctx.epoll['lock'].release() # Bottom of WHILE cycle. if ctx.connections_counters['open'] == 0 and \ ctx.connections_counters['setup'] == 0: break finally: for n in ctx.notaries_to_connections.itervalues(): for c in n['connections'].itervalues(): cb = self.get_callback(CB_NOTARY_CONNECTION_CLOSE) if cb is not None: cb(c) c.thread.join() ctx.epoll['epoll'].close() M2Crypto.threading.cleanup() # Check for errors, resolve connections' statuses to usable form. # # Apply Convergence policy of authentication failure in regards to: # - connections # - certificate fingerprint agreement if report is None: report = \ cuivre.client.fingerprint_report.ConvergenceValidationReportStanza() report.anonymization = ctx.anonymization report.policy = ctx.policy if ctx.anonymization: report.relay_notary_entity_name = ctx.relay_notary_entity_name # Counters for applying the policy. notary_verification_answers = { 'verified' : 0, 'failure' : 0, 'ignore' : 0 } for n in ctx.notaries_to_connections.values(): connection_data = n['connections'].values()[0] notary_entity_name = connection_data.notary_entity.name report.add_notary_entity(notary_entity_name) # All connections, the established one and all others for this notary. for nhc in n['connections'].itervalues(): relay_host = None relay_port = None if nhc.relay_notary_entity: relay_host = nhc.relay_notary_entity.hosts.values()[0].host relay_port = nhc.relay_notary_entity.hosts.values()[0].http_port notary_host = nhc.notary_entity.hosts.values()[0].host notary_port = nhc.notary_entity.hosts.values()[0].ssl_port if ctx.anonymization: notary_port = cuivre.CONVERGENCE_PROTOCOL_PORT notary_host_id = nhc.notary_entity.hosts.values()[0].id # Insert the notary connection in the report. report.add_notary_connection( notary_entity_name, relay_host, relay_port, notary_host_id, notary_host, notary_port ) # Parse the established connection's other data fields. if not n['connection']: notary_verification_answers['failure'] += 1 continue connection_data = n['connection'] ssl_protocol = connection_data.ssl_protocol ssl_cipher_suite = connection_data.ssl_cipher_suite # Parse the notary response body. m = connection_data.http_msg_in status_code = int(m.status_code) if status_code == cuivre.http.HTTP_STATUS_CODE_OK: pass elif status_code == cuivre.http.HTTP_STATUS_CODE_CONFLICT: notary_verification_answers['failure'] += 1 elif status_code == cuivre.http.HTTP_STATUS_CODE_SEE_OTHER: notary_verification_answers['ignore'] += 1 elif status_code == cuivre.http.HTTP_STATUS_CODE_BAD_REQUEST: notary_verification_answers['failure'] += 1 elif status_code == cuivre.http.HTTP_STATUS_SERVICE_UNAVAILABLE: notary_verification_answers['failure'] += 1 else: notary_verification_answers['failure'] += 1 # If the response contains a JSON list of fingerprints (200 and 409 codes) # parse it and look for the last one in time. Then, verify the response's # signature with the notary host's public key. observed_fingerprint = None if status_code == cuivre.http.HTTP_STATUS_CODE_OK or \ status_code == cuivre.http.HTTP_STATUS_CODE_CONFLICT: j = json.loads(m.body) # Extract the fingerprint currently observed by the notary. if 'fingerprintList' in j: last = (None, 0) for i in j['fingerprintList']: if int(i['timestamp']['finish']) > last[1] or last[0] == None: last = (i['fingerprint'], int(i['timestamp']['finish'])) observed_fingerprint = last[0] # Verify by ourselves the notary's OK response. if status_code == cuivre.http.HTTP_STATUS_CODE_OK: if fingerprint != observed_fingerprint: # Warning: the certificate observed by the notary differs from the # one we're currently observing. notary_verification_answers['failure'] += 1 else: notary_verification_answers['verified'] += 1 # Verify response signature. signature_verification = False public_key_pem = \ connection_data.notary_entity.hosts.values()[0].public_key_pem if cuivre.client.utils.verify_response_signature(j, public_key_pem): signature_verification = True else: observed_fingerprint = None signature_verification = None # Set the established connection for this notary in the report. notary_host_id = connection_data.notary_entity.hosts.values()[0].id notary_fingerprint = cuivre.client.utils.put_colon_separator( connection_data.ssl_certificate_fingerprint ) report.set_notary_established_connection( notary_entity_name, notary_host_id, notary_fingerprint, ssl_protocol, ssl_cipher_suite, status_code, observed_fingerprint, signature_verification ) # Apply Convergence policy. q = notary_verification_answers['verified'] + \ notary_verification_answers['failure'] verification_success = False if ctx.policy == cuivre.POLICY_ONE: if notary_verification_answers['verified'] >= 1: verification_success = True elif ctx.policy == cuivre.POLICY_MAJORITY: if notary_verification_answers['verified'] >= q/2: verification_success = True elif ctx.policy == cuivre.POLICY_UNANIMITY: if notary_verification_answers['verified'] == q: verification_success = True report.validation = verification_success return report def get_fingerprints_cache(self): """ Proxy method calling the get_fingerprints_cache() method on the database object, fetching all cached fingerprints. @rtype: list @return: Current cache lines. """ rows = self.database.get_fingerprints_cache() return rows def empty_cache(self): """ Proxy method calling the empty_cache() method on the database object. @rtype: list @return: The cache lines that were deleted. """ rows = self.database.empty_cache() return rows def delete_cache_entry(self, host, port=cuivre.HTTPS_DEFAULT_PORT): """ Proxy method calling the delete_cache_entry() method on the database object. @type host: string @param host: The cached fingerprint's matching host. @type port: int @param port: The server's listening port. @rtype: list @return: The cache lines that were deleted. """ rows = self.database.delete_cache_entry(host, port) return rows