def _Sync(self):
"""Forces the master to sync with the other data servers."""
pool = None
try:
pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
body = ""
headers = {"Content-Length": len(body)}
res = pool.urlopen(
"POST", "/servers/sync-all", headers=headers, body=body)
if res.status == constants.RESPONSE_INCOMPLETE_SYNC:
print "Master has tried to contact all the data servers, but failed."
return False
if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
print "Master server says that some data servers are not running."
print "Giving up..."
return False
if res.status != constants.RESPONSE_OK:
print "Unable to sync servers."
return False
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return False
print "Sync done."
# Update mapping.
self.mapping = rdf_data_server.DataServerMapping(res.data)
return True
def SyncMapping(self, skip=None):
"""Syncs mapping with other servers."""
pools = []
try:
# Update my state.
self._PeriodicThread()
for serv in self.servers[1:]:
if skip and serv in skip:
continue
pool = connectionpool.HTTPConnectionPool(serv.Address(),
port=serv.Port())
pools.append((serv, pool))
body = self.mapping.SerializeToString()
headers = {"Content-Length": len(body)}
for serv, pool in pools:
res = pool.urlopen("POST",
"/servers/sync",
headers=headers,
body=body)
if res.status != constants.RESPONSE_OK:
logging.warning("Could not sync with server %s:%d",
serv.Address(), serv.Port())
return False
state = rdf_data_server.DataServerState()
state.ParseFromString(res.data)
serv.UpdateState(state)
except urllib3.exceptions.MaxRetryError:
return False
finally:
for _, pool in pools:
pool.close()
return True
def _Recover(self, transid):
"""Completes a rebalancing transaction that was unsuccessful."""
print "Contacting master about transaction %s..." % transid,
pool = None
try:
pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
print "OK."
try:
body = transid
headers = {"Content-Length": len(body)}
res = pool.urlopen(
"POST", "/rebalance/recover", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
if res.status == constants.RESPONSE_TRANSACTION_NOT_FOUND:
print "Transaction %s was not found" % transid
return
if res.status != constants.RESPONSE_OK:
print "Potential data master error. Giving up..."
return
rebalance = rdf_data_server.DataServerRebalance(res.data)
print "Got transaction object %s" % rebalance.id
answer = raw_input("Proceed with the recover process? (y/n) ")
if answer != "y":
return
body = rebalance.SerializeToString()
headers = {"Content-Length": len(body)}
try:
res = pool.urlopen(
"POST", "/rebalance/commit", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Could not commit re-sharding transaction with id %s" % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
if res.status != constants.RESPONSE_OK:
print "Could not commit transaction %s" % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
self.mapping = rdf_data_server.DataServerMapping(res.data)
print "Rebalance with id %s fully performed." % rebalance.id
def SetRebalancing(self, reb):
"""Sets a new rebalance operation and starts communication with servers."""
self.rebalance = reb
self.rebalance_pool = []
try:
for serv in self.servers:
pool = connectionpool.HTTPConnectionPool(serv.Address(),
port=serv.Port())
self.rebalance_pool.append(pool)
except urllib3.exceptions.MaxRetryError:
self.CancelRebalancing()
return False
return True
def __init__(self):
servers = config_lib.CONFIG["Dataserver.server_list"]
if not servers:
raise errors.DataServerError("List of data servers not available.")
master_location = servers[0]
loc = urlparse.urlparse(master_location, scheme="http")
self.addr = loc.hostname
self.port = int(loc.port)
self.pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
self.history_path = os.path.expanduser("~/.grr-data-store-manager")
if os.path.exists(self.history_path):
readline.read_history_file(self.history_path)
self.periodic_thread = None
self.mapping = None
self.mapping_time = 0
def __init__(self, my_port, handler_cls):
servers = config_lib.CONFIG["Dataserver.server_list"]
if not servers:
raise errors.DataServerError("List of data servers not available.")
master_location = servers[0]
loc = urlparse.urlparse(master_location, scheme="http")
self.index = None
self.master_addr = loc.hostname
self.master_port = loc.port
self.my_port = my_port
self.pool = connectionpool.HTTPConnectionPool(
self.master_addr, port=int(self.master_port), maxsize=1)
self.registered = False
self.periodic_fail = 0
self.handler_cls = handler_cls
def _RecCopyFiles(rebalance, server_id, dspath, subpath, pool_cache,
removed_list):
"""Recursively send files for moving to the required data server."""
fulldir = utils.JoinPath(dspath, subpath)
mapping = rebalance.mapping
for comp in os.listdir(fulldir):
if comp == constants.REBALANCE_DIRECTORY:
continue
path = utils.JoinPath(fulldir, comp)
name, unused_extension = os.path.splitext(comp)
if name in COPY_EXCEPTIONS:
continue
if os.path.isdir(path):
result = _RecCopyFiles(rebalance, server_id, dspath,
utils.JoinPath(subpath, comp), pool_cache,
removed_list)
if not result:
return False
continue
if not os.path.isfile(path):
continue
key = common.MakeDestinationKey(subpath, name)
where = sutils.MapKeyToServer(mapping, key)
if where != server_id:
server = mapping.servers[where]
addr = server.address
port = server.port
key = (addr, port)
try:
pool = pool_cache[key]
except KeyError:
pool = connectionpool.HTTPConnectionPool(addr, port=port)
pool_cache[key] = pool
logging.info("Need to move %s from %d to %d", key, server_id,
where)
if not _SendFileToServer(pool, path, subpath, comp, rebalance):
return False
removed_list.append(path)
else:
logging.info("File %s stays here", path)
return True
def _RemServer(self, addr, port):
"""Remove server from group."""
# Find server.
server, _ = self._FindServer(addr, port)
if not server:
print "Server not found."
return
if server.interval.start != server.interval.end:
print "Server has some data in it!"
print "Giving up..."
return
pool = None
try:
pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
body = self._PackNewServer(addr, port)
headers = {"Content-Length": len(body)}
try:
res = pool.urlopen(
"POST", "/servers/rem/check", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
if res.status == constants.RESPONSE_DATA_SERVER_NOT_FOUND:
print "Master server says the data server does not exist."
return
if res.status == constants.RESPONSE_RANGE_NOT_EMPTY:
print "Master server says the data server has still some data."
print "Giving up..."
return
if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
print "Master server says some data servers are not running."
print "Giving up..."
return
if res.status != constants.RESPONSE_OK:
print "Master server error. Is the server running?"
return
print "Master server allows us to remove server %s:%d" % (addr, port)
answer = raw_input("Do you really want to remove server //%s:%d? (y/n) " %
(addr, port))
if answer != "y":
return
try:
res = pool.urlopen("POST", "/servers/rem", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
print "Master server says that some data servers are not running."
print "Giving up..."
return
if res.status == constants.RESPONSE_OK:
# Update mapping.
self.mapping = rdf_data_server.DataServerMapping(res.data)
self._CompleteRemServerHelpComplete(addr, port)
return
if res.status == constants.RESPONSE_INCOMPLETE_SYNC:
# We were unable to sync, so we try again:
if self._Sync():
self._CompleteRemServerHelpComplete(addr, port)
return
else:
# If we cannot sync in the second attempt, we give up.
print("The master server has removed the new server, but the other "
"servers may not know about it.")
print "Please run 'sync' to fix the problem, followed by:"
self._CompleteRemServerHelp(addr, port)
return
if res.status != constants.RESPONSE_OK:
print "Master has returned an unknown error..."
return
def _AddServer(self, addr, port):
"""Starts the process of adding a new server."""
if port <= 0:
print "Wrong port: %d" % port
return
pool = None
try:
pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
body = self._PackNewServer(addr, port)
headers = {"Content-Length": len(body)}
try:
res = pool.urlopen(
"POST", "/servers/add/check", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
if res.status == constants.RESPONSE_EQUAL_DATA_SERVER:
print "Master server says there is already a similar server."
print "Giving up..."
return
if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
print "Master server says that some data servers are not running."
print "Giving up..."
return
if res.status != constants.RESPONSE_OK:
print "Master server error. Is the server running?"
return
print "Master server allows us to add server %s:%d" % (addr, port)
answer = raw_input("Do you really want to add server //%s:%d? (y/n) " %
(addr, port))
if answer != "y":
return
try:
res = pool.urlopen("POST", "/servers/add", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
print "Master server says that some data servers are not running."
print "Giving up..."
return
if res.status == constants.RESPONSE_INCOMPLETE_SYNC:
print("The master server has set up the new server, but the other "
"servers may not know about it.")
print "Please run 'sync' to fix the problem."
print "Afterwards, you have to rebalance server data with the following:"
self._CompleteAddServerHelp(addr, port)
return
if res.status != constants.RESPONSE_OK:
print "Failed to contact master server."
return
print "============================================="
print "Operation completed."
print "To rebalance server data you have to do the following:"
self._CompleteAddServerHelp(addr, port)
# Update mapping.
self.mapping = rdf_data_server.DataServerMapping(res.data)
def _DoRebalance(self, new_mapping):
"""Performs a new rebalancing operation with the master server."""
print "Contacting master server to start re-sharding...",
# Send mapping information to master.
pool = None
try:
pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact master..."
return
body = new_mapping.SerializeToString()
headers = {"Content-Length": len(body)}
res = None
try:
res = pool.urlopen(
"POST", "/rebalance/phase1", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to talk with master..."
pool.close()
return
if res.status != constants.RESPONSE_OK:
print "Re-sharding cannot be done!"
return
rebalance = rdf_data_server.DataServerRebalance(res.data)
print "OK"
print
print "The following servers will need to move data:"
for i, move in enumerate(list(rebalance.moving)):
print "Server %d moves %dKB" % (i, move / 1024)
answer = raw_input("Proceed with re-sharding? (y/n) ")
if answer != "y":
return
body = rebalance.SerializeToString()
headers = {"Content-Length": len(body)}
try:
res = pool.urlopen(
"POST", "/rebalance/phase2", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print "Unable to contact server for re-sharding."
print "Make sure the data servers are up and try again."
return
if res.status != constants.RESPONSE_OK:
print "Could not start copying files for re-sharding"
print "Make sure the data servers are up and try again."
return
try:
res = pool.urlopen(
"POST", "/rebalance/commit", headers=headers, body=body)
except urllib3.exceptions.MaxRetryError:
print("Could not commit the re-sharding transaction with id "
"%s") % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
if res.status != constants.RESPONSE_OK:
print "Could not commit the transaction %s" % rebalance.id
print "Make sure the data servers are up and then run:"
print "'recover %s' in order to re-run transaction" % rebalance.id
return
self.mapping = rdf_data_server.DataServerMapping(res.data)
print "Rebalance with id %s fully performed." % rebalance.id