def _do_auth_hmac_cb(self, uuid): """Perform mutual HMAC_CB authentication.""" wwwauth = create_option_header('HMAC_CB', realm=uuid) headers = [('WWW-Authenticate', wwwauth)] auth = self.environ.get('HTTP_AUTHORIZATION') if auth is None: raise HTTPReturn(http.UNAUTHORIZED, headers) try: method, options = parse_option_header(auth) except ValueError: raise HTTPReturn(http.UNAUTHORIZED, headers) if method != 'HMAC_CB': raise HTTPReturn(http.UNAUTHORIZED, headers) if 'name' in options: # pair step 1 - ask user for permission to pair name = options['name'] if not self.allow_pairing: raise HTTPReturn('403 Pairing Disabled') bus = instance(MessageBusServer) kxid = self.crypto.random(16).encode('hex') pin = '%06d' % (self.crypto.randint(bits=31) % 1000000) approved = bus.call_method(None, 'get_pairing_approval', name, uuid, pin, kxid, timeout=60) if not approved: raise HTTPReturn('403 Approval Denied') restrictions = {} self.key_exchanges[kxid] = (time.time(), restrictions, pin) wwwauth = create_option_header('HMAC_CB', kxid=kxid) headers = [('WWW-Authenticate', wwwauth)] raise HTTPReturn(http.UNAUTHORIZED, headers) elif 'kxid' in options: # pair step 2 - check auth and do the actual pairing kxid = options['kxid'] if kxid not in self.key_exchanges: raise HTTPReturn(http.FORBIDDEN) starttime, restrictions, pin = self.key_exchanges.pop(kxid) signature = base64.try_decode(options.get('signature', '')) if not signature: raise HTTPReturn(http.FORBIDDEN) now = time.time() if now - starttime > 60: raise HTTPReturn('403 Request Timeout') cb = self.environ['SSL_CHANNEL_BINDING_TLS_UNIQUE'] check = self.crypto.hmac(adjust_pin(pin, +1), cb, 'sha1') if check != signature: raise HTTPReturn('403 Invalid PIN') bus = instance(MessageBusServer) bus.send_signal(None, 'PairingComplete', kxid) # Prove to the other side we also know the PIN signature = self.crypto.hmac(adjust_pin(pin, -1), cb, 'sha1') signature = base64.encode(signature) authinfo = create_option_header('HMAC_CB', kxid=kxid, signature=signature) self.headers.append(('Authentication-Info', authinfo)) else: raise HTTPReturn(http.UNAUTHORIZED, headers)
def locator_is_available(self): """Return whether or not the locator is available. There are platforms where we don't have a locator at the moment. """ locator = instance(Locator) return len(locator.sources) > 0
def get_vault(self, uuid): """Return the vault with UUID *uuid*. The result value is a dictionary containing the vault metadata, or ``None`` if the vault was not found. """ return instance(Model).get_vault(uuid)
def add_version(self, vault, version): """Add a new version to a vault. The *version* parameter must be a dictionary. The version is a new version and should not contain and "id" key yet. """ return instance(Model).add_version(vault, version)
def get_config(self): """Return the configuration object. The configuration object is a dictionary that can be used by frontends to store configuration data. """ return instance(Model).get_config()
def _do_auth_rsa_cb(self, uuid): """Perform mutual RSA_CB authentication.""" wwwauth = create_option_header('RSA_CB', realm=uuid) headers = [('WWW-Authenticate', wwwauth)] auth = self.environ.get('HTTP_AUTHORIZATION') if auth is None: raise HTTPReturn(http.UNAUTHORIZED, headers) try: method, opts = parse_option_header(auth) except ValueError: raise HTTPReturn(http.UNAUTHORIZED, headers) if method != 'RSA_CB': raise HTTPReturn(http.UNAUTHORIZED, headers) if 'node' not in opts or not check_uuid4(opts['node']): raise HTTPReturn(http.UNAUTHORIZED, headers) if 'signature' not in opts or not base64.check(opts['signature']): raise HTTPReturn(http.UNAUTHORIZED, headers) model = instance(Model) cert = model.get_certificate(uuid, opts['node']) if cert is None: raise HTTPReturn(http.UNAUTHORIZED, headers) signature = base64.decode(opts['signature']) pubkey = base64.decode(cert['payload']['keys']['auth']['key']) cb = self.environ['SSL_CHANNEL_BINDING_TLS_UNIQUE'] if not self.crypto.rsa_verify(cb, signature, pubkey, 'pss-sha1'): raise HTTPReturn(http.UNAUTHORIZED, headers) # The peer was authenticated. Authenticate ourselves as well. privkey = model.get_auth_key(uuid) vault = model.get_vault(uuid) node = vault['node'] signature = self.crypto.rsa_sign(cb, privkey, 'pss-sha1') signature = base64.encode(signature) auth = create_option_header('RSA_CB', node=node, signature=signature) self.headers.append(('Authentication-Info', auth))
def get_version_history(self, vault, uuid): """Get the history of a version. This returns a ordered list with the linear history all the way from the current newest instance of the version, back to the first version. """ return instance(Model).get_version_history(vault, uuid)
def delete_vault(self, vault): """Delete a vault and all its items. The *vault* parameter must be a vault metadata dictionary returned by :meth:`get_vault`. """ return instance(Model).delete_vault(vault)
def delete_version(self, vault, version): """Delete a version from a vault. This create a special updated version of the record with a "deleted" flag set. By default, deleted versions do not show up in the output of :meth:`get_versions`. """ return instance(Model).delete_version(vault, version)
def password_strength(self, method, *args): """Return the strength of a password that was generated by :meth:`generate_password`. The return value is an integer indicating the entropy of the password in bits. """ return instance(PasswordGenerator).strength(method, *args)
def update_version(self, vault, version): """Update an existing version. The *version* parameter should be a dictionary. It must have an "id" of a version that already exists. The version will become the latest version of the specific id. """ return instance(Model).update_version(vault, version)
def exec_(self): ctrlapi = instance(ControlApiClient) config = ctrlapi.get_config('frontends.qt') if config is None: config = ctrlapi.create_config({'name': 'frontends.qt'}) self._config = config mainwindow = singleton(MainWindow) mainwindow.show() return super(Bluepass, self).exec_()
def lock_vault(self, uuid): """Lock a vault. This destroys the decrypted private keys and any other decrypted items that are cached. It is not an error to lock a vault that is already locked. """ return instance(Model).lock_vault(uuid)
def set_allow_pairing(self, timeout): """Be visible on the network for *timeout* seconds. When visible, other instances of Bluepass will be able to find us, and initiate a pairing request. The pairing request will still have to be approved, and PIN codes needs to be exchanged. """ publisher = instance(SyncAPIPublisher) publisher.set_allow_pairing(timeout)
def __init__(self): super(SocketAPIHandler, self).__init__() self.crypto = instance(CryptoProvider) self.logger = logging.getLogger(__name__) instance(Model).add_callback(self._event_callback) instance(Locator).add_callback(self._event_callback) instance(SyncAPIPublisher).add_callback(self._event_callback) self.pairdata = {}
def pair_neighbor_step1(self, node, source): """Start a new pairing process. A pairing process is started with node *node* residing in source *source*. The return value is a string containing a random cookie that identifies the current request. """ locator = instance(Locator) neighbor = locator.get_neighbor(node, source) if neighbor is None: raise PairingError('NotFound', 'No such neighbor') visible = neighbor['properties'].get('visible') if not visible: raise PairingError('NotFound', 'Node not visible') vault = neighbor['vault'] model = instance(Model) if model.get_vault(vault): raise PairingError('Exists', 'Vault already exists') # Don't keep the GUI blocked while we wait for remote approval. cookie = self.crypto.random(16).encode('hex') self.early_response(cookie) name = misc.gethostname() for addr in neighbor['addresses']: client = SyncAPIClient(addr) try: client.connect() except SyncAPIError as e: continue # try next address try: kxid = client.pair_step1(vault, name) except SyncAPIError as e: status = e[0] detail = e.asdict() else: status = 'OK' detail = {} self.pairdata[cookie] = (kxid, neighbor, addr) self.connection.send_signal('PairNeighborStep1Completed', cookie, status, detail) client.close() break
def generate_password(self, method, *args): """Generate a password. The *method* specifies the method. It can currently be "diceware" or "random". The "diceware" method takes one argument: an integer with the number of words to generate. The "random" method takes two arguments: th size in character, and an alphabest in the form of a regular expression character set (e.g. [a-zA-Z0-9]). """ return instance(PasswordGenerator).generate(method, *args)
def unlock_vault(self, uuid, password): """Unlock a vault. The vault *uuid* is unlocked using *password*. This decrypts the private keys that are stored in the database and stored them in memory. On error, an exception is raised. It is not an error to unlock a vault that is already unlocked. """ return instance(Model).unlock_vault(uuid, password)
def update_vault(self, vault): """Update a vault's metadata. The *vault* parameter must be a dictionary. The recommended way to use this function is to use :meth:`get_vault` to retrieve the metadata, make updates, make updates to it, and the use this method to save the updates. On success, nothing is returned. On error, an exception is raised. """ return instance(Model).update_vault(uuid, vault)
def sync_inbound(self, env): uuid = env['mapper.vault'] if not check_uuid4(uuid): raise HTTPReturn(http.NOT_FOUND) model = instance(Model) vault = model.get_vault(uuid) if vault is None: raise HTTPReturn(http.NOT_FOUND) self._do_auth_rsa_cb(uuid) items = self.entity if items is None or not isinstance(items, list): raise HTTPReturn(http.BAD_REQUEST) model.import_items(uuid, items)
def test_pair_and_sync(self): # Create two databases and two models database1 = Database(self.tempfile()) model1 = create(Model, database1) assert instance(Model) is model1 vault1 = model1.create_vault('Vault1', 'Passw0rd') database2 = Database(self.tempfile()) model2 = Model(database2) vault2 = model2.create_vault('Vault2', 'Passw0rd', uuid=vault1['id']) # Start a message bus server and client connection lsock = socket.socket() lsock.bind(('localhost', 0)) lsock.listen(2) mbserver = create(MessageBusServer, lsock, 'S3cret', None) #mbserver.set_trace('/tmp/server.txt') mbserver.start() csock = socket.socket() csock.connect(lsock.getsockname()) mbhandler = MBTestHandler() mbclient = MessageBusConnection(csock, 'S3cret', mbhandler) #mbclient.set_trace('/tmp/client.txt') # Start the syncapi lsock = socket.socket() lsock.bind(('localhost', 0)) lsock.listen(2) address = lsock.getsockname() syncapp = SyncAPIApplication() syncapp.allow_pairing = True server = SyncAPIServer(lsock, syncapp) server.start() # Pair with vault1 client = SyncAPIClient(lsock.getsockname()) client.connect() kxid = client.pair_step1(vault1['id'], 'foo') assert kxid is not None assert mbhandler.name == 'foo' certinfo = { 'name': 'node2', 'node': vault2['node'] } keys = certinfo['keys'] = {} for key in vault2['keys']: keys[key] = { 'key': vault2['keys'][key]['public'], 'keytype': vault2['keys'][key]['keytype'] } peercert = client.pair_step2(vault1['id'], kxid, mbhandler.pin, certinfo) assert isinstance(peercert, dict) assert model1.check_certinfo(peercert)[0] model2.add_certificate(vault2['id'], peercert) # Sync version1 = model1.add_version(vault1['id'], {'foo': 'bar'}) client.sync(vault1['id'], model2) version2 = model2.get_version(vault1['id'], version1['id']) assert version2 is not None assert version2['foo'] == 'bar'
def pair_neighbor_step2(self, cookie, pin, name, password): """Complete a pairing process. The *cookie* argument are returned by :meth:`pair_neighbor_step1`. The *pin* argument is the PIN code that the remote Bluepass instance showed to its user. The *name* and *password* arguments specify the name and password of the paired vault that is created in the local instance. Paired vaults will automatically be kept up to date. Changes made in a paired vault in once Bluepass instance will automatically be synced to other instances by the Bluepass backend. To get notified of new versions that were added, listen for the ``VersionsAdded`` signal. """ if cookie not in self.pairdata: raise PairingError('NotFound', 'No such key exchange ID') kxid, neighbor, addr = self.pairdata.pop(cookie) # Again don't keep the GUI blocked while we pair and do a full sync self.early_response() model = instance(Model) vault = model.create_vault(name, password, neighbor['vault'], notify=False) certinfo = { 'node': vault['node'], 'name': misc.gethostname() } keys = certinfo['keys'] = {} for key in vault['keys']: keys[key] = { 'key': vault['keys'][key]['public'], 'keytype': vault['keys'][key]['keytype'] } client = SyncAPIClient(addr) client.connect() try: peercert = client.pair_step2(vault['id'], kxid, pin, certinfo) except SyncAPIError as e: status = e[0] detail = e.asdict() model.delete_vault(vault) else: status = 'OK' detail = {} model.add_certificate(vault['id'], peercert) client.sync(vault['id'], model, notify=False) model.raise_event('VaultAdded', vault) self.connection.send_signal('PairNeighborStep2Completed', cookie, status, detail) client.close()
def get_version(self, vault, uuid): """Return a version from a vault. The latest version identified by *uuid* is returned from *vault*. The version is returned as a dictionary. If the version does not exist, ``None`` is returned. In Bluepass, vaults contain versions. Think of a version as an arbitrary object that is versioned and encrypted. A version has at least "id" and "_type" keys. The "id" will stay constant over the entire lifetime of the version. Newer versions supersede older versions. This method call returns the newest instance of the version. Versions are the unit of synchronization in our peer to peer replication protocol. They are also the unit of encryption. Both passwords are groups are stored as versions. """ return instance(Model).get_version(vault, uuid)
def sync_outbound(self, env): uuid = env['mapper.vault'] if not check_uuid4(uuid): raise HTTPReturn(http.NOT_FOUND) model = instance(Model) vault = model.get_vault(uuid) if vault is None: raise HTTPReturn(http.NOT_FOUND) self._do_auth_rsa_cb(uuid) args = parse_qs(env.get('QUERY_STRING', '')) vector = args.get('vector', [''])[0] if vector: try: vector = parse_vector(vector) except ValueError: raise HTTPReturn(http.BAD_REQUEST) items = model.get_items(uuid, vector) myvector = model.get_vector(uuid) self.headers.append(('X-Vector', dump_vector(myvector))) return items
def pair(self, env): uuid = env['mapper.vault'] if not check_uuid4(uuid): raise HTTPReturn(http.NOT_FOUND) model = instance(Model) vault = model.get_vault(uuid) if not vault: raise HTTPReturn(http.NOT_FOUND) self._do_auth_hmac_cb(uuid) # Sign the certificate request that was sent to tus certinfo = self.entity if not certinfo or not isinstance(certinfo, dict): raise HTTPReturn(http.BAD_REQUEST) model.add_certificate(uuid, certinfo) # And send our own certificate request in return certinfo = { 'node': vault['node'], 'name': socket.gethostname() } certkeys = certinfo['keys'] = {} for key in vault['keys']: certkeys[key] = { 'key': vault['keys'][key]['public'], 'keytype': vault['keys'][key]['keytype'] } return certinfo
def get_versions(self, vault): """Return the newest instances for all versions in a vault. The return value is a list of dictionaries. """ return instance(Model).get_versions(vault)
def _event_callback(self, event, *args): # Forward the event over the message bus. instance(MessageBusServer).send_signal(None, event, *args)
def backend(self): return instance(BackendProxy)
def mainWindow(self): return instance(MainWindow)
def get_neighbors(self): """Return current neighbords on the network. The return value is a list of dictionaries. """ return instance(Locator).get_neighbors()
def update_config(self, config): """Update the configuration object.""" return instance(Model).update_config(config)
class SocketAPIHandler(MessageBusHandler): """A message bus handler that implements our socket API.""" # NOTE: all methods run in separate greenlets! def __init__(self): super(SocketAPIHandler, self).__init__() self.crypto = instance(CryptoProvider) self.logger = logging.getLogger(__name__) instance(Model).add_callback(self._event_callback) instance(Locator).add_callback(self._event_callback) instance(SyncAPIPublisher).add_callback(self._event_callback) self.pairdata = {} def _event_callback(self, event, *args): # Forward the event over the message bus. instance(MessageBusServer).send_signal(None, event, *args) # Version @method() def get_version_info(self): """Get version information. Returns a dictionary containing at least the key "version". """ version_info = {} for name in dir(_version): if name.startswith('_'): continue version_info[name] = getattr(_version, name) return version_info # Model methods @method() def get_config(self): """Return the configuration object. The configuration object is a dictionary that can be used by frontends to store configuration data. """ return instance(Model).get_config() @method() def update_config(self, config): """Update the configuration object.""" return instance(Model).update_config(config) @method() def create_vault(self, name, password, async=False): """Create a new vault. The vault will have the name *name*. The vault's private keys will be encrypted with *password*. The *async* parameter specifies if the vault creation needs to be asynchronous. If it is set to False, then the vault is created synchronously and it is returned as a dictionary. If async is set to True, then this will return the UUID of the vault as a stirng. Once the vault has been created, the ``VaultCreationComplete`` signal will be raised. The signal has three arguments: the UUID, a status code, and a detailed message. Creating a vault requires the backend to generate 3 RSA keys. This can be a time consuming process. Therefore it is recommended to use asynchronous vault creation in user interfaces. """ # Vault creation is time consuming because 3 RSA keys have # to be generated. Therefore an async variant is provided. model = instance(Model) if not async: return model.create_vault(name, password) uuid = self.crypto.randuuid() self.early_response(uuid) try: vault = model.create_vault(name, password, uuid) except StructuredError as e: status = e[0] detail = e.asdict() except Exception: e = StructuredError.uncaught_exception() status = e[0] detail = e.asdict() else: status = 'OK' detail = vault self.connection.send_signal('VaultCreationComplete', uuid, status, detail)
def vault_is_locked(self, uuid): """Return whether or not the vault *uuid* is locked.""" return instance(Model).vault_is_locked(uuid)
def get_vault_statistics(self, uuid): """Return statistics about a vault. The return value is a dictionary. """ return instance(Model).get_vault_statistics(uuid)
def _run(self): """Main execution loop, runs in its own greenlet.""" logger = logging.getLogger(__name__) locator = instance(Locator) self.locator = locator model = instance(Model) model.add_callback(self._event_callback) nodename = self._get_hostname() vaults = model.get_vaults() for vault in vaults: locator.register(vault['node'], nodename, vault['id'], vault['name'], self.server.address) logger.debug('published node %s', vault['node']) self.published_nodes.add(vault['node']) stopped = False while not stopped: timeout = self.allow_pairing_until - time.time() \ if self.allow_pairing else None self.queue_notempty.wait(timeout) self.queue_notempty.clear() while self.queue: event, args = self.queue.pop(0) logger.debug('processing event: %s', event) if event == 'allow_pairing': timeout = args[0] if timeout > 0: self.allow_pairing = True self.allow_pairing_until = time.time() + timeout for node in self.published_nodes: locator.set_property(node, 'visible', 'true') logger.debug('make node %s visible', node) instance(SyncAPIApplication).allow_pairing = True self.raise_event('AllowPairingStarted', timeout) else: self.allow_pairing = False self.allow_pairing_until = None for node in self.published_nodes: locator.set_property(node, 'visible', 'false') logger.debug('make node %s invisible (user)', node) instance(SyncAPIApplication).allow_pairing = False self.raise_event('AllowPairingEnded') elif event == 'VaultAdded': vault = args[0] node = vault['node'] if node in self.published_nodes: logger.error( 'got VaultAdded signal for published node') continue properties = {} if self.allow_pairing: properties['visible'] = 'true' locator.register(node, nodename, vault['id'], vault['name'], self.server.address, properties) self.published_nodes.add(node) logger.debug('published node %s', node) elif event == 'VaultRemoved': vault = args[0] node = vault['node'] if node not in self.published_nodes: logger.error( 'got VaultRemoved signal for unpublished node') continue locator.unregister(node) self.published_nodes.remove(node) logger.debug('unpublished node %s', node) elif event == 'stop': stopped = True now = time.time() if self.allow_pairing and now >= self.allow_pairing_until: for node in self.published_nodes: self.locator.set_property(node, 'visible', 'false') logger.debug('make node %s invisible (timeout)', node) self.allow_pairing = False self.allow_pairing_until = None instance(SyncAPIApplication).allow_pairing = False self.raise_event('AllowPairingEnded') logger.debug('done processing queue, sleeping') logger.debug('shutting down publisher')
def get_vaults(self): """Return a list of all vaults. The result value is a list if dictionaries containing vault metadata. """ return instance(Model).get_vaults()
def backend(self): return instance(ControlApiClient)
def _run(self): """This runs the synchronization loop.""" logger = self.logger model = instance(Model) model.add_callback(self._event_callback) locator = instance(Locator) locator.add_callback(self._event_callback) neighbors = locator.get_neighbors() mynodes = set((v['node'] for v in model.get_vaults())) myvaults = set((v['id'] for v in model.get_vaults())) while True: # Determine how long we need to wait now = time.time() timeout = self.interval for neighbor in neighbors: node = neighbor['node'] vault = neighbor['vault'] if node in mynodes or vault not in myvaults \ or not model.get_certificate(vault, node): continue last_sync = self.last_sync.get(node, 0) timeout = min(timeout, max(0, last_sync + self.interval - now)) # Now wait for a timeout, or an event. self.queue_notempty.wait(timeout) self.queue_notempty.clear() # Build a list of nodes that we need to sync with. # # We sync to nodes that are are not ours, whose vault we also # have, and where there is a certificate. In addition, at least # one of the following three needs to be true: # # 1. The last sync to this node is > interval seconds ago. # 2. A version was added locally to the node's vault # 3. The node resides at an address that we are already syncing # with. # # Regarding #3, we organize the nodes by network address, and try # to sync all nodes over a single connection. So the nodes in #3 # are almost "free" to do so that's they are included. now = time.time() neighbors = locator.get_neighbors() mynodes = set((v['node'] for v in model.get_vaults())) myvaults = set((v['id'] for v in model.get_vaults())) byaddress = {} sync_nodes = set() sync_vaults = set() # First process events. while self.queue: event, args = self.queue.pop(0) if event == 'NeighborDiscovered': neighbor = args[0] # As an optimization do not sync with new neighbors that # are discovered while we are running, because we known # that when they are started up they will sync with us. self.last_sync[neighbor['node']] = now elif event == 'VersionsAdded': vault, versions = args # As an optimization, only push out a list of added # versions in case it is generated locally, because we know # the originator will push the update to everybody else. for version in versions: item = model.get_version_item(vault, version['id']) if item['origin']['node'] not in mynodes: continue logger.debug('local update, syncing to all nodes for ' 'vault %s', vault) sync_vaults.add(vault) break # Now build a list of nodes including a "byaddress" list. for neighbor in neighbors: node = neighbor['node'] vault = neighbor['vault'] if node in mynodes or vault not in myvaults \ or not model.get_certificate(vault, node): # Never sync with these nodes... continue last_sync = self.last_sync.get(node, 0) timeout = last_sync + self.interval < now if timeout or vault in sync_vaults: for addr in neighbor['addresses']: key = addr['id'] if key not in byaddress: byaddress[key] = (addr['family'], addr, []) byaddress[key][2].append(neighbor) sync_nodes.add(node) continue # See if we are already syncing with an address, and if so, # include /that address only/ in the sync job. for addr in neighbor['addresses']: key = addr['id'] if key in byaddress: byaddress[key][2].append(neighbor) sync_nodes.add(node) if not sync_nodes: # Nothing to do... continue logger.debug('total nodes to sync: %d', len(sync_nodes)) # Now sync to the nodes. Try to reuse the network connection for # multiple nodes. We sort the addresses on location source so that # we will be able to give different priorites to different sources # later. nnodes = nconnections = 0 addresses = sorted(byaddress.itervalues(), key=lambda x: x[0]) for source,addr,neighbors in addresses: client = None for neighbor in neighbors: node = neighbor['node'] if node not in sync_nodes: continue # already synced logger.debug('syncing with node %s', node) if client is None: client = SyncAPIClient(addr) try: client.connect() except SyncAPIError as e: logger.error('could not connect to %s: %s', addr, str(e)) client.close() break logger.debug('connected to %s', addr) nconnections += 1 vault = neighbor['vault'] starttime = time.time() try: client.sync(vault, model) except SyncAPIError: logger.error('failed to sync vault %s at %s', vault, addr) client.close() client = None else: logger.debug('succesfully synced vault %s at %s', vault, addr) nnodes += 1 sync_nodes.remove(node) self.last_sync[node] = starttime if client: client.close() if not sync_nodes: break # we are done logger.debug('synced to %d nodes using %d network connections', nnodes, nconnections) if sync_nodes: logger.debug('failed to sync with %d nodes', len(sync_nodes)) logger.debug('syncer loop terminated')
def _run(self): """This runs the synchronization loop.""" logger = self.logger model = instance(Model) model.add_callback(self._event_callback) locator = instance(Locator) locator.add_callback(self._event_callback) neighbors = locator.get_neighbors() mynodes = set((v['node'] for v in model.get_vaults())) myvaults = set((v['id'] for v in model.get_vaults())) while True: # Determine how long we need to wait now = time.time() timeout = self.interval for neighbor in neighbors: node = neighbor['node'] vault = neighbor['vault'] if node in mynodes or vault not in myvaults \ or not model.get_certificate(vault, node): continue last_sync = self.last_sync.get(node, 0) timeout = min(timeout, max(0, last_sync + self.interval - now)) # Now wait for a timeout, or an event. self.queue_notempty.wait(timeout) self.queue_notempty.clear() # Build a list of nodes that we need to sync with. # # We sync to nodes that are are not ours, whose vault we also # have, and where there is a certificate. In addition, at least # one of the following three needs to be true: # # 1. The last sync to this node is > interval seconds ago. # 2. A version was added locally to the node's vault # 3. The node resides at an address that we are already syncing # with. # # Regarding #3, we organize the nodes by network address, and try # to sync all nodes over a single connection. So the nodes in #3 # are almost "free" to do so that's they are included. now = time.time() neighbors = locator.get_neighbors() mynodes = set((v['node'] for v in model.get_vaults())) myvaults = set((v['id'] for v in model.get_vaults())) byaddress = {} sync_nodes = set() sync_vaults = set() # First process events. while self.queue: event, args = self.queue.pop(0) if event == 'NeighborDiscovered': neighbor = args[0] # As an optimization do not sync with new neighbors that # are discovered while we are running, because we known # that when they are started up they will sync with us. self.last_sync[neighbor['node']] = now elif event == 'VersionsAdded': vault, versions = args # As an optimization, only push out a list of added # versions in case it is generated locally, because we know # the originator will push the update to everybody else. for version in versions: item = model.get_version_item(vault, version['id']) if item['origin']['node'] not in mynodes: continue logger.debug( 'local update, syncing to all nodes for ' 'vault %s', vault) sync_vaults.add(vault) break # Now build a list of nodes including a "byaddress" list. for neighbor in neighbors: node = neighbor['node'] vault = neighbor['vault'] if node in mynodes or vault not in myvaults \ or not model.get_certificate(vault, node): # Never sync with these nodes... continue last_sync = self.last_sync.get(node, 0) timeout = last_sync + self.interval < now if timeout or vault in sync_vaults: for addr in neighbor['addresses']: key = addr['id'] if key not in byaddress: byaddress[key] = (addr['family'], addr, []) byaddress[key][2].append(neighbor) sync_nodes.add(node) continue # See if we are already syncing with an address, and if so, # include /that address only/ in the sync job. for addr in neighbor['addresses']: key = addr['id'] if key in byaddress: byaddress[key][2].append(neighbor) sync_nodes.add(node) if not sync_nodes: # Nothing to do... continue logger.debug('total nodes to sync: %d', len(sync_nodes)) # Now sync to the nodes. Try to reuse the network connection for # multiple nodes. We sort the addresses on location source so that # we will be able to give different priorites to different sources # later. nnodes = nconnections = 0 addresses = sorted(byaddress.itervalues(), key=lambda x: x[0]) for source, addr, neighbors in addresses: client = None for neighbor in neighbors: node = neighbor['node'] if node not in sync_nodes: continue # already synced logger.debug('syncing with node %s', node) if client is None: client = SyncAPIClient(addr) try: client.connect() except SyncAPIError as e: logger.error('could not connect to %s: %s', addr, str(e)) client.close() break logger.debug('connected to %s', addr) nconnections += 1 vault = neighbor['vault'] starttime = time.time() try: client.sync(vault, model) except SyncAPIError: logger.error('failed to sync vault %s at %s', vault, addr) client.close() client = None else: logger.debug('succesfully synced vault %s at %s', vault, addr) nnodes += 1 sync_nodes.remove(node) self.last_sync[node] = starttime if client: client.close() if not sync_nodes: break # we are done logger.debug('synced to %d nodes using %d network connections', nnodes, nconnections) if sync_nodes: logger.debug('failed to sync with %d nodes', len(sync_nodes)) logger.debug('syncer loop terminated')