def HandleRebalanceCopy(self):
reb = rdf_data_server.DataServerRebalance(self.post_data)
index = 0
if not self.MASTER:
index = self.DATA_SERVER.Index()
rebalance.CopyFiles(reb, index)
self._EmptyResponse(constants.RESPONSE_OK)
def FetchRebalanceInformation(self):
"""Asks data servers for number of changes for rebalancing."""
body = self.rebalance.SerializeToString()
size = len(body)
headers = {"Content-Length": size}
for pool in self.rebalance_pool:
try:
res = pool.urlopen("POST", "/rebalance/statistics", headers=headers,
body=body)
if res.status != constants.RESPONSE_OK:
self.CancelRebalancing()
return False
reb = rdf_data_server.DataServerRebalance()
reb.ParseFromString(res.data)
ls = list(reb.moving)
if ls:
logging.warning("Moving %d", ls[0])
self.rebalance.moving.Append(ls[0])
else:
self.CancelRebalancing()
return False
except urllib3.exceptions.MaxRetryError:
self.CancelRebalancing()
return False
return True
def HandleRebalancePhase1(self):
"""Call master to perform phase 1 of the rebalancing operation."""
if not self.MASTER:
self._EmptyResponse(constants.RESPONSE_NOT_MASTER_SERVER)
return
if self.MASTER.IsRebalancing():
self._EmptyResponse(constants.RESPONSE_MASTER_IS_REBALANCING)
return
new_mapping = rdf_data_server.DataServerMapping.FromSerializedString(
self.post_data)
rebalance_id = str(uuid.uuid4())
reb = rdf_data_server.DataServerRebalance(id=rebalance_id,
mapping=new_mapping)
if not self.MASTER.SetRebalancing(reb):
logging.warning("Could not contact servers for rebalancing")
self._EmptyResponse(constants.RESPONSE_DATA_SERVERS_UNREACHABLE)
return
if not self.MASTER.FetchRebalanceInformation():
logging.warning(
"Could not contact servers for rebalancing statistics")
self._EmptyResponse(constants.RESPONSE_DATA_SERVERS_UNREACHABLE)
return
self.rebalance_id = rebalance_id
body = reb.SerializeToString()
self._Response(constants.RESPONSE_OK, body)
def _Recover(self, transid):
"""Completes a rebalancing transaction that was unsuccessful."""
print "Contacting master about transaction %s..." % transid,
pool = None
try:
pool = urllib3.connectionpool.HTTPConnectionPool(self.addr,
port=self.port)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
print "OK."
try:
body = transid
headers = {"Content-Length": len(body)}
res = pool.urlopen("POST",
"/rebalance/recover",
headers=headers,
body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
if res.status == constants.RESPONSE_TRANSACTION_NOT_FOUND:
print "Transaction %s was not found" % transid
return
if res.status != constants.RESPONSE_OK:
print "Potential data master error. Giving up..."
return
rebalance = rdf_data_server.DataServerRebalance(res.data)
print "Got transaction object %s" % rebalance.id
answer = raw_input("Proceed with the recover process? (y/n) ")
if answer != "y":
return
body = rebalance.SerializeToString()
headers = {"Content-Length": len(body)}
try:
res = pool.urlopen("POST",
"/rebalance/commit",
headers=headers,
body=body)
except urllib3.exceptions.MaxRetryError:
print "Could not commit re-sharding transaction with id %s" % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
if res.status != constants.RESPONSE_OK:
print "Could not commit transaction %s" % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
self.mapping = rdf_data_server.DataServerMapping(res.data)
print "Rebalance with id %s fully performed." % rebalance.id
def HandleRebalanceStatistics(self):
"""Call data server to count how much data needs to move in rebalancing."""
reb = rdf_data_server.DataServerRebalance(self.post_data)
mapping = reb.mapping
index = 0
if not self.MASTER:
index = self.DATA_SERVER.Index()
moving = rebalance.ComputeRebalanceSize(mapping, index)
reb.moving.Append(moving)
body = reb.SerializeToString()
self._Response(constants.RESPONSE_OK, body)
def GetCommitInformation(transid):
"""Read transaction information from stored file."""
loc = data_store.DB.Location()
if not os.path.exists(loc):
return False
if not os.path.isdir(loc):
return False
tempdir = _GetTransactionDirectory(loc, transid)
tempfile = utils.JoinPath(tempdir, constants.TRANSACTION_FILENAME)
if not os.path.exists(tempfile):
return None
if not os.path.isfile(tempfile):
return None
with open(tempfile, "rb") as fp:
return rdf_data_server.DataServerRebalance(fp.read())
def HandleRebalancePhase2(self):
"""Call master to perform phase 2 of rebalancing."""
if not self.MASTER:
self._EmptyResponse(constants.RESPONSE_NOT_MASTER_SERVER)
return
reb = rdf_data_server.DataServerRebalance(self.post_data)
current = self.MASTER.IsRebalancing()
if not current or current.id != reb.id:
# Not the same ID.
self._EmptyResponse(constants.RESPONSE_WRONG_TRANSACTION)
return
if not self.MASTER.CopyRebalanceFiles():
self._EmptyResponse(constants.RESPONSE_FILES_NOT_COPIED)
return
self._EmptyResponse(constants.RESPONSE_OK)
def HandleRebalanceCommit(self):
"""Call master to commit rebalance transaction."""
if not self.MASTER:
self._EmptyResponse(constants.RESPONSE_NOT_MASTER_SERVER)
return
reb = rdf_data_server.DataServerRebalance(self.post_data)
current = self.MASTER.IsRebalancing()
if not current or current.id != reb.id:
# Not the same ID.
self._EmptyResponse(constants.RESPONSE_WRONG_TRANSACTION)
return
new_mapping = self.MASTER.RebalanceCommit()
if not new_mapping:
self._EmptyResponse(constants.RESPONSE_NOT_COMMITED)
return
self._Response(constants.RESPONSE_OK, self.MAPPING.SerializeToString())
def HandleRebalancePerform(self):
"""Call data server to perform rebalance transaction."""
reb = rdf_data_server.DataServerRebalance(self.post_data)
if not rebalance.MoveFiles(reb, self.MASTER):
logging.critical("Failed to perform transaction %s", reb.id)
self._EmptyResponse(constants.RESPONSE_FILES_NOT_MOVED)
return
# Update range of servers.
# But only for regular data servers since the master is responsible for
# starting the operation.
if self.DATA_SERVER:
for i, serv in enumerate(list(reb.mapping.servers)):
self.MAPPING.servers[i].interval.start = serv.interval.start
self.MAPPING.servers[i].interval.end = serv.interval.end
self.DATA_SERVER.SetMapping(self.MAPPING)
# Send back server state.
stat = self.GetStatistics()
body = stat.SerializeToString()
self._Response(constants.RESPONSE_OK, body)
def _DoRebalance(self, new_mapping):
"""Performs a new rebalancing operation with the master server."""
print "Contacting master server to start re-sharding...",
# Send mapping information to master.
pool = None
try:
pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
body = new_mapping.SerializeToString()
headers = {"Content-Length": len(body)}
res = None
try:
res = pool.urlopen(
"POST", "/rebalance/phase1", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to talk with master..."
pool.close()
return
if res.status != constants.RESPONSE_OK:
print "Re-sharding cannot be done!"
return
rebalance = rdf_data_server.DataServerRebalance(res.data)
print "OK"
print
print "The following servers will need to move data:"
for i, move in enumerate(list(rebalance.moving)):
print "Server %d moves %dKB" % (i, move / 1024)
answer = raw_input("Proceed with re-sharding? (y/n) ")
if answer != "y":
return
body = rebalance.SerializeToString()
headers = {"Content-Length": len(body)}
try:
res = pool.urlopen(
"POST", "/rebalance/phase2", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact server for re-sharding."
print "Make sure the data servers are up and try again."
return
if res.status != constants.RESPONSE_OK:
print "Could not start copying files for re-sharding"
print "Make sure the data servers are up and try again."
return
try:
res = pool.urlopen(
"POST", "/rebalance/commit", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print("Could not commit the re-sharding transaction with id "
"%s") % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
if res.status != constants.RESPONSE_OK:
print "Could not commit the transaction %s" % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
self.mapping = rdf_data_server.DataServerMapping(res.data)
print "Rebalance with id %s fully performed." % rebalance.id