def remove(self, other_nodes, backups=0, root=None, force=False):
"""Called when the node was removed"""
if force:
for other in other_nodes:
if other == self.url:
continue
node = SubNode(other)
node.take_over(self.url, other_nodes, backups=backups, root=root)
else:
req = Request.blank(
self.url + '/remove-self', json={'other': other_nodes, 'name': self.url, 'backups': backups})
print forward(req, root=root).body.strip()
def node_added(self, req):
"""Responds to ``POST /node-added``
This is called to ask this node to take over from any other
nodes, as appropriate.
Takes a request with the JSON body:
`other`: list of all nodes.
`name`: the name of this node.
Responds with a text description of what it did.
"""
self.assert_is_internal(req)
status = Response(content_type='text/plain')
data = req.json
dbs = []
for other_node in data['other']:
req_data = data.copy()
req_data['name'] = other_node
url = urlparse.urljoin(req.application_url, '/' + other_node)
status.write('Deprecating from %s\n' % url)
query = Request.blank(url + '/query-deprecate', json=req_data, method='POST')
query.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(query)
assert resp.status_code == 200, str(resp)
resp_data = resp.json
for db_data in resp_data['deprecated']:
status.write(' deprecated: %(path)s\n' % db_data)
dbs.append((other_node, db_data))
for other_node, db_data in dbs:
status.write('Copying database %s from %s\n' % (db_data['path'], other_node))
url = urlparse.urljoin(req.application_url, '/' + other_node)
copier = Request.blank(url + urllib.quote(db_data['path']) + '?copy')
copier.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(copier)
assert resp.status_code == 200, str(resp)
## FIXME: the terribleness!
fp = StringIO(resp.body)
db = self.storage.for_user(db_data['domain'], db_data['username'], db_data['bucket'])
db.decode_db(fp)
status.write(' copied %i bytes\n' % resp.content_length)
deleter = Request.blank(url + db_data['path'] + '?delete')
deleter.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(deleter)
assert resp.status_code < 300, str(resp)
status.write(' deleted\n')
status.write('done.\n')
return status
def take_over(self, bad_node, other_nodes, backups=0, root=None):
"""Called when the node should take over from `bad_node`"""
req = Request.blank(
self.url + '/take-over', json={'other': other_nodes, 'bad': bad_node, 'name': self.url, 'backups': backups})
body = forward(req, root=root).body.strip()
if body:
print body
def post_backup(self, req, db, backup, last_pos):
"""Handles backups from a POST request.
Forwards the request, as come from the given database, and
with the known last position of this database. `backup` is
the node to back up to.
"""
url = urlparse.urljoin(req.application_url, '/' + backup)
url += urllib.quote(req.path_info)
if req.query_string:
## FIXME: not sure if 'since' should propagate, or maybe it doesn't matter?
url += '?' + req.query_string
backup_req = Request.blank(url, method='POST')
backup_req.GET['backup-from-pos'] = str(last_pos)
backup_req.GET['source'] = req.path_url
backup_req.GET['collection_id'] = db.collection_id
for key in 'exclude', 'include':
if key in backup_req.GET:
del backup_req.GET[key]
backup_req.body = req.body
backup_req.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(backup_req)
#print 'sending backup req', backup_req, resp
if resp.status_code >= 300:
## FIXME: what then?!
print 'WARNING: bad response from %s: %s' % (backup_req.url, resp)
def take_over(self, req):
"""Attached to ``POST /take-over``
Takes over databases from another server, that presumably has
gone offline without notice.
This goes through all of the local databases, and sees if this
node was either using the bad node as a backup, or is a backup
for the bad node. In either case it finds the new node that
should be either master or handling the bad node, and sends
the local database to that server.
Takes a JSON body with keys:
`other`: a list of all nodes
`name`: the name of *this* node
`bad`: the bad node being removed
`backups`: the number of backups
"""
self.assert_is_internal(req)
status = Response(content_type='text/plain')
data = req.json
nodes = data['other']
self_name = data['name']
bad_node = data['bad']
assert self_name != bad_node
backups = data['backups']
ring = HashRing(nodes)
for domain, username, bucket in self.storage.all_dbs():
assert bucket.startswith('/')
path = '/' + domain + '/' + username + bucket
iterator = iter(ring.iterate_nodes(path))
active_nodes = [iterator.next() for i in xrange(backups + 1)]
replacement_node = iterator.next()
# Not all the backups should try to restore the database, so instead
# just the "first" does it
restore = False
if active_nodes[0] == bad_node and active_nodes[1:] and active_nodes[1] == self_name:
status.write('Master node %s for %s removed\n' % (bad_node, path))
restore = True
elif bad_node in active_nodes and active_nodes[0] == self_name:
status.write('Backup node %s for %s removed\n' % (bad_node, path))
restore = True
if not restore:
continue
db = self.storage.for_user(domain, username, bucket)
send = Request.blank(replacement_node + urllib.quote(path) + '?paste',
method='POST', body=''.join(db.encode_db()))
send.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(send)
assert resp.status_code == 201, str(resp)
#status.write(' nodes: %r - %r / %r\n' % (active_nodes, bad_node, self_name))
status.write(' success, added to %s (from %s)\n' % (replacement_node, self_name))
return status
def __call__(self, req):
"""Forwards a request to the node"""
url = urlparse.urljoin(req.application_url, self.url)
parsed = urlparse.urlsplit(url)
req.scheme = parsed.scheme
req.host = parsed.netloc
req.server_name = parsed.hostname
req.server_port = parsed.port or '80'
req.script_name = urllib.unquote(parsed.path)
resp = forward(req)
resp.headers['X-Node-Name'] = self.url
return resp
def apply_backup(self, req, db):
"""Responds to ``POST /db-name?backup-from-pos=N``
This is the request that is sent when the master node wants to
backup a POST request to this backup node. This is handled
similar to a POST request, but the data is kept
unconditionally. When a backup arrives but is ahead of local
records, this node will try to catch up with a `?copy` request.
This has the additional GET parameters of:
`backup-from-pos`: what the last id was on the master node; if
it's ahead of what we have then we need to catch up.
`source`: the master node.
"""
self.assert_is_internal(req)
backup_pos = int(req.GET['backup-from-pos'])
source = req.GET['source']
collection_id = req.GET['collection_id']
if collection_id != db.collection_id:
if db.empty:
db.set_collection_id(collection_id)
else:
dir, timer = db.dir, db.timer
db.clear()
db = Storage(dir, timer)
items = req.json
datas = [
(backup_pos + index + 1, json.dumps(item))
for index, item in enumerate(items)]
try:
db.db.extend(datas, expect_last_counter=backup_pos, with_counters=True)
except ExpectationFailed:
# The canonical server is ahead of us, we must catch up!
has_queue = db.has_queue
if has_queue:
# We're in the middle of transferring, all is well
db.queue.extend(datas, with_counters=True)
## FIXME: we should really try to extend the
else:
# We need to catch up
catchup_req = Request.blank(source)
catchup_req.GET['copy'] = ''
catchup_req.GET['until'] = backup_pos
catchup_req.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(catchup_req)
assert resp.status_code == 200, str(resp)
fp = StringIO(resp.body)
db.decode_db(fp, append_queue=True)
return Response(status=201)
def remove_self(self, req):
"""Responds to ``POST /remove-self``
This is a request for this node to gracefully remove itself.
It will attempt to back up its data to the other nodes that
should take over.
This takes a request with the JSON data:
`name`: the name of this node
`other`: a list of all nodes (including this)
`backups`: the number of backups to make
It responds with a text description of what it did.
"""
self.assert_is_internal(req)
status = Response(content_type='text/plain')
self.storage.disable()
data = req.json
self_name = data['name']
status.write('Disabling node %s\n' % self_name)
ring = HashRing(data['other'])
for domain, username, bucket in self.storage.all_dbs():
assert bucket.startswith('/')
path = '/' + domain + '/' + username + bucket
db = self.storage.for_user(domain, username, bucket)
if db.is_deprecated:
db.clear()
continue
iterator = iter(ring.iterate_nodes(path))
active_nodes = [iterator.next() for i in xrange(data['backups'] + 1)]
new_node = iterator.next()
assert self_name in active_nodes, '%r not in %r' % (self_name, active_nodes)
status.write('Sending %s to node %s\n' % (path, new_node))
url = urlparse.urljoin(req.application_url, '/' + new_node)
send = Request.blank(url + urllib.quote(path) + '?paste',
method='POST', body=''.join(db.encode_db()))
send.environ['cutout.root'] = req.environ.get('cutout.root')
resp = forward(send)
assert resp.status_code == 201, str(resp)
status.write(' success, deleting\n')
db.clear()
self.storage.clear()
return status
def added(self, other_nodes, backups=0, root=None):
"""Called when the node was added"""
req = Request.blank(
self.url + '/node-added', json={'other': other_nodes, 'new': self.url, 'backups': backups})
print forward(req, root=root).body.strip()
