def remove(self, other_nodes, backups=0, root=None, force=False): """Called when the node was removed""" if force: for other in other_nodes: if other == self.url: continue node = SubNode(other) node.take_over(self.url, other_nodes, backups=backups, root=root) else: req = Request.blank( self.url + '/remove-self', json={'other': other_nodes, 'name': self.url, 'backups': backups}) print forward(req, root=root).body.strip()
def node_added(self, req): """Responds to ``POST /node-added`` This is called to ask this node to take over from any other nodes, as appropriate. Takes a request with the JSON body: `other`: list of all nodes. `name`: the name of this node. Responds with a text description of what it did. """ self.assert_is_internal(req) status = Response(content_type='text/plain') data = req.json dbs = [] for other_node in data['other']: req_data = data.copy() req_data['name'] = other_node url = urlparse.urljoin(req.application_url, '/' + other_node) status.write('Deprecating from %s\n' % url) query = Request.blank(url + '/query-deprecate', json=req_data, method='POST') query.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(query) assert resp.status_code == 200, str(resp) resp_data = resp.json for db_data in resp_data['deprecated']: status.write(' deprecated: %(path)s\n' % db_data) dbs.append((other_node, db_data)) for other_node, db_data in dbs: status.write('Copying database %s from %s\n' % (db_data['path'], other_node)) url = urlparse.urljoin(req.application_url, '/' + other_node) copier = Request.blank(url + urllib.quote(db_data['path']) + '?copy') copier.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(copier) assert resp.status_code == 200, str(resp) ## FIXME: the terribleness! fp = StringIO(resp.body) db = self.storage.for_user(db_data['domain'], db_data['username'], db_data['bucket']) db.decode_db(fp) status.write(' copied %i bytes\n' % resp.content_length) deleter = Request.blank(url + db_data['path'] + '?delete') deleter.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(deleter) assert resp.status_code < 300, str(resp) status.write(' deleted\n') status.write('done.\n') return status
def take_over(self, bad_node, other_nodes, backups=0, root=None): """Called when the node should take over from `bad_node`""" req = Request.blank( self.url + '/take-over', json={'other': other_nodes, 'bad': bad_node, 'name': self.url, 'backups': backups}) body = forward(req, root=root).body.strip() if body: print body
def post_backup(self, req, db, backup, last_pos): """Handles backups from a POST request. Forwards the request, as come from the given database, and with the known last position of this database. `backup` is the node to back up to. """ url = urlparse.urljoin(req.application_url, '/' + backup) url += urllib.quote(req.path_info) if req.query_string: ## FIXME: not sure if 'since' should propagate, or maybe it doesn't matter? url += '?' + req.query_string backup_req = Request.blank(url, method='POST') backup_req.GET['backup-from-pos'] = str(last_pos) backup_req.GET['source'] = req.path_url backup_req.GET['collection_id'] = db.collection_id for key in 'exclude', 'include': if key in backup_req.GET: del backup_req.GET[key] backup_req.body = req.body backup_req.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(backup_req) #print 'sending backup req', backup_req, resp if resp.status_code >= 300: ## FIXME: what then?! print 'WARNING: bad response from %s: %s' % (backup_req.url, resp)
def take_over(self, req): """Attached to ``POST /take-over`` Takes over databases from another server, that presumably has gone offline without notice. This goes through all of the local databases, and sees if this node was either using the bad node as a backup, or is a backup for the bad node. In either case it finds the new node that should be either master or handling the bad node, and sends the local database to that server. Takes a JSON body with keys: `other`: a list of all nodes `name`: the name of *this* node `bad`: the bad node being removed `backups`: the number of backups """ self.assert_is_internal(req) status = Response(content_type='text/plain') data = req.json nodes = data['other'] self_name = data['name'] bad_node = data['bad'] assert self_name != bad_node backups = data['backups'] ring = HashRing(nodes) for domain, username, bucket in self.storage.all_dbs(): assert bucket.startswith('/') path = '/' + domain + '/' + username + bucket iterator = iter(ring.iterate_nodes(path)) active_nodes = [iterator.next() for i in xrange(backups + 1)] replacement_node = iterator.next() # Not all the backups should try to restore the database, so instead # just the "first" does it restore = False if active_nodes[0] == bad_node and active_nodes[1:] and active_nodes[1] == self_name: status.write('Master node %s for %s removed\n' % (bad_node, path)) restore = True elif bad_node in active_nodes and active_nodes[0] == self_name: status.write('Backup node %s for %s removed\n' % (bad_node, path)) restore = True if not restore: continue db = self.storage.for_user(domain, username, bucket) send = Request.blank(replacement_node + urllib.quote(path) + '?paste', method='POST', body=''.join(db.encode_db())) send.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(send) assert resp.status_code == 201, str(resp) #status.write(' nodes: %r - %r / %r\n' % (active_nodes, bad_node, self_name)) status.write(' success, added to %s (from %s)\n' % (replacement_node, self_name)) return status
def __call__(self, req): """Forwards a request to the node""" url = urlparse.urljoin(req.application_url, self.url) parsed = urlparse.urlsplit(url) req.scheme = parsed.scheme req.host = parsed.netloc req.server_name = parsed.hostname req.server_port = parsed.port or '80' req.script_name = urllib.unquote(parsed.path) resp = forward(req) resp.headers['X-Node-Name'] = self.url return resp
def apply_backup(self, req, db): """Responds to ``POST /db-name?backup-from-pos=N`` This is the request that is sent when the master node wants to backup a POST request to this backup node. This is handled similar to a POST request, but the data is kept unconditionally. When a backup arrives but is ahead of local records, this node will try to catch up with a `?copy` request. This has the additional GET parameters of: `backup-from-pos`: what the last id was on the master node; if it's ahead of what we have then we need to catch up. `source`: the master node. """ self.assert_is_internal(req) backup_pos = int(req.GET['backup-from-pos']) source = req.GET['source'] collection_id = req.GET['collection_id'] if collection_id != db.collection_id: if db.empty: db.set_collection_id(collection_id) else: dir, timer = db.dir, db.timer db.clear() db = Storage(dir, timer) items = req.json datas = [ (backup_pos + index + 1, json.dumps(item)) for index, item in enumerate(items)] try: db.db.extend(datas, expect_last_counter=backup_pos, with_counters=True) except ExpectationFailed: # The canonical server is ahead of us, we must catch up! has_queue = db.has_queue if has_queue: # We're in the middle of transferring, all is well db.queue.extend(datas, with_counters=True) ## FIXME: we should really try to extend the else: # We need to catch up catchup_req = Request.blank(source) catchup_req.GET['copy'] = '' catchup_req.GET['until'] = backup_pos catchup_req.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(catchup_req) assert resp.status_code == 200, str(resp) fp = StringIO(resp.body) db.decode_db(fp, append_queue=True) return Response(status=201)
def remove_self(self, req): """Responds to ``POST /remove-self`` This is a request for this node to gracefully remove itself. It will attempt to back up its data to the other nodes that should take over. This takes a request with the JSON data: `name`: the name of this node `other`: a list of all nodes (including this) `backups`: the number of backups to make It responds with a text description of what it did. """ self.assert_is_internal(req) status = Response(content_type='text/plain') self.storage.disable() data = req.json self_name = data['name'] status.write('Disabling node %s\n' % self_name) ring = HashRing(data['other']) for domain, username, bucket in self.storage.all_dbs(): assert bucket.startswith('/') path = '/' + domain + '/' + username + bucket db = self.storage.for_user(domain, username, bucket) if db.is_deprecated: db.clear() continue iterator = iter(ring.iterate_nodes(path)) active_nodes = [iterator.next() for i in xrange(data['backups'] + 1)] new_node = iterator.next() assert self_name in active_nodes, '%r not in %r' % (self_name, active_nodes) status.write('Sending %s to node %s\n' % (path, new_node)) url = urlparse.urljoin(req.application_url, '/' + new_node) send = Request.blank(url + urllib.quote(path) + '?paste', method='POST', body=''.join(db.encode_db())) send.environ['cutout.root'] = req.environ.get('cutout.root') resp = forward(send) assert resp.status_code == 201, str(resp) status.write(' success, deleting\n') db.clear() self.storage.clear() return status
def added(self, other_nodes, backups=0, root=None): """Called when the node was added""" req = Request.blank( self.url + '/node-added', json={'other': other_nodes, 'new': self.url, 'backups': backups}) print forward(req, root=root).body.strip()