def set_configuration(self, configuration):
"""
Set the runtime configuration.
@type configuration: cuivre.client.configuration.Configuration
@param configuration: The new configuration.
"""
self.conf = configuration
try:
self.check_data_directory()
except DataDirectoryExistenceException:
try:
self.create_data_directory()
except OSError:
raise
except (DataDirectoryReadException, DataDirectoryWriteException):
raise
self.database = Database(self.conf.db_file_path)
try:
self.conf.check_config_file()
except cuivre.client.configuration.ConfigFileExistenceException:
try:
self.conf.write_configuration_file()
except cuivre.client.configuration.ConfigFileWriteException:
raise
except cuivre.client.configuration.ConfigFileReadException:
raise
except cuivre.client.configuration.ConfigFileWriteException:
pass
try:
self.database.check_database_file()
except cuivre.client.database.ExistenceException:
try:
self.database.create_database()
except cuivre.client.database.CreationException:
raise
except (
cuivre.client.database.ReadException,
cuivre.client.database.WriteException
):
raise
class Client(object):
"""Implements information checking and retrieval from Convergence notaries."""
def __init__(self, configuration=None):
"""
Initialize the client's configuration.
@type configuration: cuivre.client.configuration.Configuration
@param configuration: The custom configuration to use, or None to make the
client create a default one.
"""
if configuration is None:
configuration = Configuration()
self.set_configuration(configuration)
self.callbacks = {}
def set_configuration(self, configuration):
"""
Set the runtime configuration.
@type configuration: cuivre.client.configuration.Configuration
@param configuration: The new configuration.
"""
self.conf = configuration
try:
self.check_data_directory()
except DataDirectoryExistenceException:
try:
self.create_data_directory()
except OSError:
raise
except (DataDirectoryReadException, DataDirectoryWriteException):
raise
self.database = Database(self.conf.db_file_path)
try:
self.conf.check_config_file()
except cuivre.client.configuration.ConfigFileExistenceException:
try:
self.conf.write_configuration_file()
except cuivre.client.configuration.ConfigFileWriteException:
raise
except cuivre.client.configuration.ConfigFileReadException:
raise
except cuivre.client.configuration.ConfigFileWriteException:
pass
try:
self.database.check_database_file()
except cuivre.client.database.ExistenceException:
try:
self.database.create_database()
except cuivre.client.database.CreationException:
raise
except (
cuivre.client.database.ReadException,
cuivre.client.database.WriteException
):
raise
def check_data_directory(self):
"""Check access permissions on the data directory."""
if not os.access(self.conf.data_dir, os.F_OK):
raise DataDirectoryExistenceException(
'Configuration directory does not exist.')
if not os.access(self.conf.data_dir, os.R_OK):
raise DataDirectoryReadException(
'Configuration directory is not readable.')
if not os.access(self.conf.data_dir, os.W_OK):
raise DataDirectoryWriteException(
'Configuration directory is not writeable.')
def create_data_directory(self):
"""Create the data directory if it doesn't already exist."""
if not os.access(self.conf.data_dir, os.F_OK):
try:
os.makedirs(self.conf.data_dir, mode=0700)
except OSError:
raise
def set_callback(self, cb_slot, cb):
cb = self.callbacks[cb_slot] = cb
def get_callback(self, cb_slot):
cb = self.callbacks[CB_NOTARY_ENTITY_CONNECTION_CONFIGURATION]
def add_notary_entity_from_bundle_file(self, bundle_path):
"""
Add a notary to the database from a notary bundle file.
@type bundle_path: string
@param bundle_path: Path to a notary bundle file.
@rtype: NotaryBundle
@return: A bundle object from the information in the specified file.
"""
try:
return self.database.add_notary_entity_from_bundle_file(
file_path=bundle_path)
except cuivre.bundle.BrokenBundle:
raise
except cuivre.client.database.DuplicateNotaryException as e:
raise NotaryAlreadyInDatabase(e.msg)
def disable_notary(self, notary_id):
"""
Disable the specified notary.
@type notary_id: int
@param notary_id: Database id of the notary to disable.
"""
self.database.disable_notary(notary_id)
def enable_notary(self, notary_id):
"""
Enable the specified notary.
@type notary_id: int
@param notary_id: Database id of the notary to enable.
"""
self.database.enable_notary(notary_id)
def get_notaries(self):
"""
Return all the notaries currently in the database.
@rtype: dict
@return: A dictionary of the notary entities in the database, with notary
ids as keys.
"""
notaries = self.database.get_notaries()
return notaries
def remove_notary(self, notary_id):
"""
Remove the specified notary from the database.
@type notary_id: int
@param notary_id: Database id of the notary to remove.
"""
self.database.remove_notary_entity(notary_id)
def update_notary(self, notary_id, bundle_location):
"""
Update the notary's information with data from the specified source.
@type notary_id: int
@param notary_id: Database id of the notary to remove.
@type bundle_location: str
@param bundle_location: HTTPS URL of the notary's bundle file.
"""
if bundle_location is not None:
r = cuivre.client.HTTPSGet(self)
msg = r.get(bundle_location)
bundle_text_data = msg.body
bundle = cuivre.bundle.NotaryBundle(text=bundle_text_data)
new_notary_entity = bundle.notary_entity
new_notary_entity.id = notary_id
self.database.update_notary_entity(new_notary_entity)
return new_notary_entity
def update_notaries(self):
"""Update all the notaries currently in the database."""
notaries = self.database.get_notary_entities()
for n in notaries:
self.update_notary(n.id, n.bundle_location)
def get_certificate_validation_report(self,
host,
port=cuivre.HTTPS_DEFAULT_PORT,
certificate=None):
"""
Validate the certificate against common verification criteria and the
network perspective supplied by the installed Convergence notaries. All the
information is compiled in a certificate validation report, recording data
and outcome of each verification step, and the overall certificate
validation based upon these steps. The fingerprint validation step is cached
locally in the database, reducing network latency issues and requests
overload on notary servers.
@type host: str
@param host: The host of the SSL server.
@type port: int
@param port: The port on which the SSL server is listening.
@type certificate: M2Crypto.X509.X509
@param certificate: The certificate to validate.
@rtype : CertificateValidationReport
@return: A CertificateValidationReport object describing each step of the
verification process.
"""
if port == None:
port = cuivre.HTTPS_DEFAULT_PORT
report = cuivre.client.certificate_report.CertificateValidationReport(
host,
port,
self,
certificate
)
fingerprint = report.fingerprint
report.compile_ssl_server_report_stanza()
if not report.ssl_server_stanza.validation:
raise CertificateNotForSSLServer(
'The certificate is not to be used for an SSL server.')
report.compile_time_constraints_report_stanza()
if not report.time_constraints_stanza.validation:
self.database.delete_cache_entry(host, port)
if not report.time_constraints_stanza.not_before_validation:
raise ValidityNotBeforeError(
'The validity of the certificate is forward in time.')
if not report.time_constraints_stanza.not_after_validation:
raise ValidityNotAfterError('The certificate has expired.')
report.compile_fingerprint_cache_report_stanza()
cache = report.fingerprint_cache_stanza
if cache.hit and cache.validation:
if report.fingerprint_cache_stanza.expiration is not None and \
report.fingerprint_cache_stanza.expiration < report.creation:
self.database.delete_cache_entry(host, port)
cache.validation = False
if not cache.hit:
report.compile_convergence_report_stanza()
if report.convergence_stanza.validation:
self.database.write_cache(fingerprint, host, port, report.not_after,
report.creation)
report.validate()
return report
def get_fingerprint_validation_report(self,
fingerprint,
host,
port=cuivre.HTTPS_DEFAULT_PORT):
"""
Validate the fingerprint against the network perspective supplied by the
installed Convergence notaries. All the information is compiled in a
fingerprint validation report, recording data and outcome of each notary
connection, and the overall fingerprint validation based upon the selected
policy. The fingerprint validation step is cached locally in the database,
reducing network latency issues and requests overload on notary servers.
@type fingerprint: string
@param fingerprint: The fingerprint to validate. (Ex.: 00:11:22:...)
@type host: str
@param host: The host of the SSL server.
@type port: int
@param port: The port on which the SSL server is listening.
@rtype : FingerprintValidationReport
@return: A FingerprintValidationReport object describing connections and
verification results.
"""
if port == None:
port = cuivre.HTTPS_DEFAULT_PORT
report = cuivre.client.fingerprint_report.FingerprintValidationReport(
host,
port,
self,
fingerprint
)
report.compile_fingerprint_cache_report_stanza()
cache = report.fingerprint_cache_stanza
if cache.hit and cache.validation:
if report.fingerprint_cache_stanza.expiration is not None and \
report.fingerprint_cache_stanza.expiration < report.creation:
self.database.delete_cache_entry(host, port)
cache.validation = False
if not cache.hit:
report.compile_convergence_report_stanza()
if report.convergence_stanza.validation:
self.database.write_cache(fingerprint, host, port, None,
report.creation)
report.validate()
return report
def validate_fingerprint(self, fingerprint, host, port, report=None):
"""
Validate a certificate fingerprint by checking the network perspective of
the registered and trusted Convergence notaries, for the specified server
host and port. This method is to be called by report objects, during the
validation process, which take advantage of the local cache. If called
directly in user code, it can be used to compile a new report stanza for the
specified fingerprint.
@type fingerprint: string
@param fingerprint: The fingerprint to validate. (Ex.: 00:11:22:...)
@type host: string
@param host: The host of the SSL server.
@type port: integer
@param port: The port on which the SSL server is listening.
@type report: ConvergenceValidationReportStanza
@param report: The report stanza to compile about this validation.
@rtype : ConvergenceValidationReportStanza
@return: A ConvergenceValidationReportStanza object describing connections
and verification results.
"""
enabled_notary_entities = self.database.get_notaries(only_enabled=True)
if len(enabled_notary_entities) < 1:
raise NoEnabledNotariesException('No enabled notaries in the database.')
# If we're anonymizing, extract a relay notary entity from the list of
# enabled ones.
relay_notary_entity = None
if self.conf.anonymization:
if len(enabled_notary_entities) < 2:
raise CannotUseAnonymizedQueries(
'Can\'t use anonymized notary queries, as that needs at least two '
'notary entities.'
)
relay_id = random.choice(enabled_notary_entities.keys())
relay_notary_entity = enabled_notary_entities.pop(relay_id)
# Randomly pick at most 'max_notaries' notaries.
available_notaries_ids = enabled_notary_entities.keys()
random.shuffle(available_notaries_ids)
available_notaries_ids = available_notaries_ids[0:3]
picked_notaries = {}
for i in available_notaries_ids:
picked_notaries[i] = enabled_notary_entities.pop(i)
del available_notaries_ids
del enabled_notary_entities
# Create a query context, set its fields to runtime configuration values,
# and populate it with information about all spawned connections.
ctx = cuivre.client.types.CuivreQueryContext()
# Set Convergence protocol parameters.
ctx.anonymization = self.conf.anonymization
ctx.policy = self.conf.convergence_policy
# Set SSL/TLS protocol parameters.
ctx.ssl_protocol = self.conf.ssl_protocol
ctx.ssl_cipher_list = self.conf.ssl_cipher_list
# Set relay notary entity data, if any.
if relay_notary_entity:
ctx.relay_notary_entity_name = relay_notary_entity.name
# Initialize M2Crypto threading support.
M2Crypto.threading.init()
# Iterate on the selected notary entities.
for notary_entity in picked_notaries.itervalues():
cb = self.get_callback(CB_NOTARY_ENTITY_CONNECTION_CONFIGURATION)
if cb is not None:
cb(notary_entity)
# Iterate on the selected notary entity's hosts.
for notary_host in notary_entity.hosts.values():
cb = self.get_callback(CB_NOTARY_CONNECTION_CONFIGURATION)
if cb is not None:
cb(notary_host)
# For each connection, create a new notary entity structure with only
# one target notary host, thus parallelizing the connections.
connection_notary_entity = cuivre.client.types.NotaryEntityData(
notary_entity.id,
notary_entity.name,
notary_entity.region,
notary_entity.bundle_location,
{notary_host.id : notary_host}
)
# Notary connection data with the notary entity's only host, and a
# complete relay notary entity, or None if not anonymizing.
connection_data = cuivre.client.types.NotaryHostConnectionData(
connection_notary_entity,
self.conf.notary_connection_timeout,
relay_notary_entity,
self.conf.relay_connection_timeout
)
# Insert the connection data into the query context.
ctx.add_notary_to_connection_mapping(connection_data)
# Spawn a notary connection thread.
t = cuivre.client.notary_connection.NotaryConnectionHandler(
ctx, connection_data
)
ctx.connections_counters['lock'].acquire()
ctx.connections_counters['setup'] += 1
ctx.connections_counters['lock'].release()
cb = self.get_callback(CB_NOTARY_CONNECTION_START)
if cb is not None:
cb(connection_data)
t.start()
try:
while True:
events = ctx.epoll['epoll'].poll(1)
for fd, event in events:
if event & select.EPOLLHUP:
ctx.connections_counters['lock'].acquire()
ctx.connections_counters['open'] -= 1
ctx.connections_counters['lock'].release()
ctx.epoll['lock'].acquire()
ctx.epoll['epoll'].unregister(fd)
ctx.epoll['lock'].release()
elif event & select.EPOLLIN:
s = ctx.fds_to_connections['connections'][fd]
bit = s.ssl_conn.recv(SOCKET_READ_BUFFER_LENGTH)
if bit is not None:
s.msg_in += bit
rest = None
try:
rest = s.http_msg_in.parse(s.msg_in)
except cuivre.http.BadHTTPMessageFromServer:
ctx.epoll['lock'].acquire()
ctx.epoll['epoll'].unregister(fd)
ctx.epoll['lock'].release()
s.ssl_conn.shutdown(socket.SHUT_RDWR)
s.ssl_conn.close()
ctx.connections_counters['lock'].acquire()
ctx.connections_counters['open'] -= 1
ctx.connections_counters['lock'].release()
if rest is not None:
s.msg_in = rest
if s.http_msg_in.parsed_body:
ctx.epoll['lock'].acquire()
ctx.epoll['epoll'].unregister(fd)
ctx.epoll['lock'].release()
s.ssl_conn.shutdown(socket.SHUT_RDWR)
s.ssl_conn.close()
ctx.connections_counters['lock'].acquire()
ctx.connections_counters['open'] -= 1
ctx.connections_counters['lock'].release()
if event & select.EPOLLRDHUP:
if bit is None or \
bit is not None and len(bit) < 1:
ctx.epoll['lock'].acquire()
ctx.epoll['epoll'].unregister(fd)
ctx.epoll['lock'].release()
s.ssl_conn.shutdown(socket.SHUT_RDWR)
s.ssl_conn.close()
ctx.connections_counters['lock'].acquire()
ctx.connections_counters['open'] -= 1
ctx.connections_counters['lock'].release()
elif event & select.EPOLLOUT:
s = ctx.fds_to_connections['connections'][fd]
sock = s.ssl_conn
# Prepare request message.
if s.msg_out == None:
post_data = 'fingerprint={}'.format(fingerprint)
hr = cuivre.http.HTTPRequestMessage()
hr.method = cuivre.http.HTTP_METHOD_POST_STRING
hr.resource = '/target/{}+{}'.format(host, port)
hr.http_version = cuivre.http.HTTP_VERSION_STRING_10
if ctx.anonymization:
hr.headers['Host'] = '{}:{}'.format(
s.notary_entity.hosts.values()[0].host,
cuivre.CONVERGENCE_PROTOCOL_PORT)
else:
hr.headers['Host'] = '{}:{}'.format(
s.notary_entity.hosts.values()[0].host,
s.notary_entity.hosts.values()[0].ssl_port)
hr.headers['Content-Type'] = 'application/x-www-form-urlencoded'
hr.headers['Connection'] = 'Close'
hr.set_body(post_data)
s.msg_out = str(hr)
# Actually send the request.
q = sock.send(s.msg_out)
s.msg_out = s.msg_out[q:]
ctx.epoll['lock'].acquire()
if len(s.msg_out) == 0:
ctx.epoll['epoll'].modify(fd,
select.EPOLLIN | select.EPOLLRDHUP)
elif len(s.msg_out) > 0:
ctx.epoll['epoll'].modify(fd,
select.EPOLLOUT | select.EPOLLHUP)
ctx.epoll['lock'].release()
# Bottom of WHILE cycle.
if ctx.connections_counters['open'] == 0 and \
ctx.connections_counters['setup'] == 0:
break
finally:
for n in ctx.notaries_to_connections.itervalues():
for c in n['connections'].itervalues():
cb = self.get_callback(CB_NOTARY_CONNECTION_CLOSE)
if cb is not None:
cb(c)
c.thread.join()
ctx.epoll['epoll'].close()
M2Crypto.threading.cleanup()
# Check for errors, resolve connections' statuses to usable form.
#
# Apply Convergence policy of authentication failure in regards to:
# - connections
# - certificate fingerprint agreement
if report is None:
report = \
cuivre.client.fingerprint_report.ConvergenceValidationReportStanza()
report.anonymization = ctx.anonymization
report.policy = ctx.policy
if ctx.anonymization:
report.relay_notary_entity_name = ctx.relay_notary_entity_name
# Counters for applying the policy.
notary_verification_answers = {
'verified' : 0,
'failure' : 0,
'ignore' : 0
}
for n in ctx.notaries_to_connections.values():
connection_data = n['connections'].values()[0]
notary_entity_name = connection_data.notary_entity.name
report.add_notary_entity(notary_entity_name)
# All connections, the established one and all others for this notary.
for nhc in n['connections'].itervalues():
relay_host = None
relay_port = None
if nhc.relay_notary_entity:
relay_host = nhc.relay_notary_entity.hosts.values()[0].host
relay_port = nhc.relay_notary_entity.hosts.values()[0].http_port
notary_host = nhc.notary_entity.hosts.values()[0].host
notary_port = nhc.notary_entity.hosts.values()[0].ssl_port
if ctx.anonymization:
notary_port = cuivre.CONVERGENCE_PROTOCOL_PORT
notary_host_id = nhc.notary_entity.hosts.values()[0].id
# Insert the notary connection in the report.
report.add_notary_connection(
notary_entity_name,
relay_host, relay_port,
notary_host_id, notary_host, notary_port
)
# Parse the established connection's other data fields.
if not n['connection']:
notary_verification_answers['failure'] += 1
continue
connection_data = n['connection']
ssl_protocol = connection_data.ssl_protocol
ssl_cipher_suite = connection_data.ssl_cipher_suite
# Parse the notary response body.
m = connection_data.http_msg_in
status_code = int(m.status_code)
if status_code == cuivre.http.HTTP_STATUS_CODE_OK:
pass
elif status_code == cuivre.http.HTTP_STATUS_CODE_CONFLICT:
notary_verification_answers['failure'] += 1
elif status_code == cuivre.http.HTTP_STATUS_CODE_SEE_OTHER:
notary_verification_answers['ignore'] += 1
elif status_code == cuivre.http.HTTP_STATUS_CODE_BAD_REQUEST:
notary_verification_answers['failure'] += 1
elif status_code == cuivre.http.HTTP_STATUS_SERVICE_UNAVAILABLE:
notary_verification_answers['failure'] += 1
else:
notary_verification_answers['failure'] += 1
# If the response contains a JSON list of fingerprints (200 and 409 codes)
# parse it and look for the last one in time. Then, verify the response's
# signature with the notary host's public key.
observed_fingerprint = None
if status_code == cuivre.http.HTTP_STATUS_CODE_OK or \
status_code == cuivre.http.HTTP_STATUS_CODE_CONFLICT:
j = json.loads(m.body)
# Extract the fingerprint currently observed by the notary.
if 'fingerprintList' in j:
last = (None, 0)
for i in j['fingerprintList']:
if int(i['timestamp']['finish']) > last[1] or last[0] == None:
last = (i['fingerprint'], int(i['timestamp']['finish']))
observed_fingerprint = last[0]
# Verify by ourselves the notary's OK response.
if status_code == cuivre.http.HTTP_STATUS_CODE_OK:
if fingerprint != observed_fingerprint:
# Warning: the certificate observed by the notary differs from the
# one we're currently observing.
notary_verification_answers['failure'] += 1
else:
notary_verification_answers['verified'] += 1
# Verify response signature.
signature_verification = False
public_key_pem = \
connection_data.notary_entity.hosts.values()[0].public_key_pem
if cuivre.client.utils.verify_response_signature(j, public_key_pem):
signature_verification = True
else:
observed_fingerprint = None
signature_verification = None
# Set the established connection for this notary in the report.
notary_host_id = connection_data.notary_entity.hosts.values()[0].id
notary_fingerprint = cuivre.client.utils.put_colon_separator(
connection_data.ssl_certificate_fingerprint
)
report.set_notary_established_connection(
notary_entity_name,
notary_host_id,
notary_fingerprint,
ssl_protocol,
ssl_cipher_suite,
status_code,
observed_fingerprint,
signature_verification
)
# Apply Convergence policy.
q = notary_verification_answers['verified'] + \
notary_verification_answers['failure']
verification_success = False
if ctx.policy == cuivre.POLICY_ONE:
if notary_verification_answers['verified'] >= 1:
verification_success = True
elif ctx.policy == cuivre.POLICY_MAJORITY:
if notary_verification_answers['verified'] >= q/2:
verification_success = True
elif ctx.policy == cuivre.POLICY_UNANIMITY:
if notary_verification_answers['verified'] == q:
verification_success = True
report.validation = verification_success
return report
def get_fingerprints_cache(self):
"""
Proxy method calling the get_fingerprints_cache() method on the database
object, fetching all cached fingerprints.
@rtype: list
@return: Current cache lines.
"""
rows = self.database.get_fingerprints_cache()
return rows
def empty_cache(self):
"""
Proxy method calling the empty_cache() method on the database object.
@rtype: list
@return: The cache lines that were deleted.
"""
rows = self.database.empty_cache()
return rows
def delete_cache_entry(self, host, port=cuivre.HTTPS_DEFAULT_PORT):
"""
Proxy method calling the delete_cache_entry() method on the database object.
@type host: string
@param host: The cached fingerprint's matching host.
@type port: int
@param port: The server's listening port.
@rtype: list
@return: The cache lines that were deleted.
"""
rows = self.database.delete_cache_entry(host, port)
return rows
