def HandleRebalanceCopy(self): reb = rdf_data_server.DataServerRebalance(self.post_data) index = 0 if not self.MASTER: index = self.DATA_SERVER.Index() rebalance.CopyFiles(reb, index) self._EmptyResponse(constants.RESPONSE_OK)
def FetchRebalanceInformation(self): """Asks data servers for number of changes for rebalancing.""" body = self.rebalance.SerializeToString() size = len(body) headers = {"Content-Length": size} for pool in self.rebalance_pool: try: res = pool.urlopen("POST", "/rebalance/statistics", headers=headers, body=body) if res.status != constants.RESPONSE_OK: self.CancelRebalancing() return False reb = rdf_data_server.DataServerRebalance() reb.ParseFromString(res.data) ls = list(reb.moving) if ls: logging.warning("Moving %d", ls[0]) self.rebalance.moving.Append(ls[0]) else: self.CancelRebalancing() return False except urllib3.exceptions.MaxRetryError: self.CancelRebalancing() return False return True
def HandleRebalancePhase1(self): """Call master to perform phase 1 of the rebalancing operation.""" if not self.MASTER: self._EmptyResponse(constants.RESPONSE_NOT_MASTER_SERVER) return if self.MASTER.IsRebalancing(): self._EmptyResponse(constants.RESPONSE_MASTER_IS_REBALANCING) return new_mapping = rdf_data_server.DataServerMapping.FromSerializedString( self.post_data) rebalance_id = str(uuid.uuid4()) reb = rdf_data_server.DataServerRebalance(id=rebalance_id, mapping=new_mapping) if not self.MASTER.SetRebalancing(reb): logging.warning("Could not contact servers for rebalancing") self._EmptyResponse(constants.RESPONSE_DATA_SERVERS_UNREACHABLE) return if not self.MASTER.FetchRebalanceInformation(): logging.warning( "Could not contact servers for rebalancing statistics") self._EmptyResponse(constants.RESPONSE_DATA_SERVERS_UNREACHABLE) return self.rebalance_id = rebalance_id body = reb.SerializeToString() self._Response(constants.RESPONSE_OK, body)
def _Recover(self, transid): """Completes a rebalancing transaction that was unsuccessful.""" print "Contacting master about transaction %s..." % transid, pool = None try: pool = urllib3.connectionpool.HTTPConnectionPool(self.addr, port=self.port) except urllib3.exceptions.MaxRetryError: print "Unable to contact master..." return print "OK." try: body = transid headers = {"Content-Length": len(body)} res = pool.urlopen("POST", "/rebalance/recover", headers=headers, body=body) except urllib3.exceptions.MaxRetryError: print "Unable to contact master..." return if res.status == constants.RESPONSE_TRANSACTION_NOT_FOUND: print "Transaction %s was not found" % transid return if res.status != constants.RESPONSE_OK: print "Potential data master error. Giving up..." return rebalance = rdf_data_server.DataServerRebalance(res.data) print "Got transaction object %s" % rebalance.id answer = raw_input("Proceed with the recover process? (y/n) ") if answer != "y": return body = rebalance.SerializeToString() headers = {"Content-Length": len(body)} try: res = pool.urlopen("POST", "/rebalance/commit", headers=headers, body=body) except urllib3.exceptions.MaxRetryError: print "Could not commit re-sharding transaction with id %s" % rebalance.id print "Make sure the data servers are up and then run:" print "'recover %s' in order to re-run transaction" % rebalance.id return if res.status != constants.RESPONSE_OK: print "Could not commit transaction %s" % rebalance.id print "Make sure the data servers are up and then run:" print "'recover %s' in order to re-run transaction" % rebalance.id return self.mapping = rdf_data_server.DataServerMapping(res.data) print "Rebalance with id %s fully performed." % rebalance.id
def HandleRebalanceStatistics(self): """Call data server to count how much data needs to move in rebalancing.""" reb = rdf_data_server.DataServerRebalance(self.post_data) mapping = reb.mapping index = 0 if not self.MASTER: index = self.DATA_SERVER.Index() moving = rebalance.ComputeRebalanceSize(mapping, index) reb.moving.Append(moving) body = reb.SerializeToString() self._Response(constants.RESPONSE_OK, body)
def GetCommitInformation(transid): """Read transaction information from stored file.""" loc = data_store.DB.Location() if not os.path.exists(loc): return False if not os.path.isdir(loc): return False tempdir = _GetTransactionDirectory(loc, transid) tempfile = utils.JoinPath(tempdir, constants.TRANSACTION_FILENAME) if not os.path.exists(tempfile): return None if not os.path.isfile(tempfile): return None with open(tempfile, "rb") as fp: return rdf_data_server.DataServerRebalance(fp.read())
def HandleRebalancePhase2(self): """Call master to perform phase 2 of rebalancing.""" if not self.MASTER: self._EmptyResponse(constants.RESPONSE_NOT_MASTER_SERVER) return reb = rdf_data_server.DataServerRebalance(self.post_data) current = self.MASTER.IsRebalancing() if not current or current.id != reb.id: # Not the same ID. self._EmptyResponse(constants.RESPONSE_WRONG_TRANSACTION) return if not self.MASTER.CopyRebalanceFiles(): self._EmptyResponse(constants.RESPONSE_FILES_NOT_COPIED) return self._EmptyResponse(constants.RESPONSE_OK)
def HandleRebalanceCommit(self): """Call master to commit rebalance transaction.""" if not self.MASTER: self._EmptyResponse(constants.RESPONSE_NOT_MASTER_SERVER) return reb = rdf_data_server.DataServerRebalance(self.post_data) current = self.MASTER.IsRebalancing() if not current or current.id != reb.id: # Not the same ID. self._EmptyResponse(constants.RESPONSE_WRONG_TRANSACTION) return new_mapping = self.MASTER.RebalanceCommit() if not new_mapping: self._EmptyResponse(constants.RESPONSE_NOT_COMMITED) return self._Response(constants.RESPONSE_OK, self.MAPPING.SerializeToString())
def HandleRebalancePerform(self): """Call data server to perform rebalance transaction.""" reb = rdf_data_server.DataServerRebalance(self.post_data) if not rebalance.MoveFiles(reb, self.MASTER): logging.critical("Failed to perform transaction %s", reb.id) self._EmptyResponse(constants.RESPONSE_FILES_NOT_MOVED) return # Update range of servers. # But only for regular data servers since the master is responsible for # starting the operation. if self.DATA_SERVER: for i, serv in enumerate(list(reb.mapping.servers)): self.MAPPING.servers[i].interval.start = serv.interval.start self.MAPPING.servers[i].interval.end = serv.interval.end self.DATA_SERVER.SetMapping(self.MAPPING) # Send back server state. stat = self.GetStatistics() body = stat.SerializeToString() self._Response(constants.RESPONSE_OK, body)
def _DoRebalance(self, new_mapping): """Performs a new rebalancing operation with the master server.""" print "Contacting master server to start re-sharding...", # Send mapping information to master. pool = None try: pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port) except urllib3.exceptions.MaxRetryError: print "Unable to contact master..." return body = new_mapping.SerializeToString() headers = {"Content-Length": len(body)} res = None try: res = pool.urlopen( "POST", "/rebalance/phase1", headers=headers, body=body) except urllib3.exceptions.MaxRetryError: print "Unable to talk with master..." pool.close() return if res.status != constants.RESPONSE_OK: print "Re-sharding cannot be done!" return rebalance = rdf_data_server.DataServerRebalance(res.data) print "OK" print print "The following servers will need to move data:" for i, move in enumerate(list(rebalance.moving)): print "Server %d moves %dKB" % (i, move / 1024) answer = raw_input("Proceed with re-sharding? (y/n) ") if answer != "y": return body = rebalance.SerializeToString() headers = {"Content-Length": len(body)} try: res = pool.urlopen( "POST", "/rebalance/phase2", headers=headers, body=body) except urllib3.exceptions.MaxRetryError: print "Unable to contact server for re-sharding." print "Make sure the data servers are up and try again." return if res.status != constants.RESPONSE_OK: print "Could not start copying files for re-sharding" print "Make sure the data servers are up and try again." return try: res = pool.urlopen( "POST", "/rebalance/commit", headers=headers, body=body) except urllib3.exceptions.MaxRetryError: print("Could not commit the re-sharding transaction with id " "%s") % rebalance.id print "Make sure the data servers are up and then run:" print "'recover %s' in order to re-run transaction" % rebalance.id return if res.status != constants.RESPONSE_OK: print "Could not commit the transaction %s" % rebalance.id print "Make sure the data servers are up and then run:" print "'recover %s' in order to re-run transaction" % rebalance.id return self.mapping = rdf_data_server.DataServerMapping(res.data) print "Rebalance with id %s fully performed." % rebalance.id