def test_node_stop(node_id, ip, port):
request = Req()
node = ServiceInfo(node_id, ip, port)
con = request.connector(IoType.EVENTIO, node)
resp = request.node_stop(node_id, con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
else:
print "Failed"
con.close()
def test_get_comp_list(my_id, gl_id, gl_ip, gl_port):
request = Req()
node = ServiceInfo(gl_id, gl_ip, gl_port)
con = request.connector(IoType.EVENTIO, node)
resp = request.get_comp_list(my_id, con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
else:
print "Failed"
con.close()
def test_node_deletion(node_id, ip, port):
request = Req()
node = ServiceInfo(node_id, ip, port)
dest = ServiceInfo("Rajnikant", "192.168.123.13", 87654)
con = request.connector(IoType.SOCKETIO, node)
resp = request.node_deletion(dest, con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
else:
print "Failed"
con.close()
def test_local_node_status(dest_id, dest_ip, dest_port):
request = Req()
node = ServiceInfo(dest_id, dest_ip, dest_port)
con = request.connector(IoType.EVENTIO, node)
resp = request.local_node_status(dest_id, con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
#print "status_code: %s, Msg: %s, enum: %s" % (resp.status.get_status_code(), resp.status.get_status_message(), resp.message.get_node_status())
else:
print "Failed: status_code: %s, msg: %s" % (
resp.status.get_status_code(), resp.status.get_status_message())
con.close()
def comp_transfer_info(my_id, gl_id, gl_ip, gl_port):
request = Req()
node = ServiceInfo(gl_id, gl_ip, gl_port)
con = request.connector(IoType.EVENTIO, node)
resp = request.comp_transfer_info(my_id, get_trnsfr_copm_list(), con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
#print resp.message.map()
#print resp.message.version()
else:
print "Failed"
con.close()
def test_get_gl_map(my_id, gl_id, gl_ip, gl_port):
request = Req()
node = ServiceInfo(gl_id, gl_ip, gl_port)
con = request.connector(IoType.EVENTIO, node)
resp = request.get_global_map(my_id, con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
#print resp.message.map()
#print resp.message.version()
else:
print "Failed"
con.close()
def test_node_addition(my_id, gl_id, gl_ip, gl_port):
request = Req()
node = ServiceInfo(gl_id, gl_ip, gl_port)
con = request.connector(IoType.EVENTIO, node)
nodes = [
ServiceInfo("", "10.0.0.45", 2222),
ServiceInfo("", "10.0.0.45", 2222),
ServiceInfo("", "10.0.0.45", 2222)
]
resp = request.node_addition(nodes, con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
#print resp.message.node_ack_list()
else:
print "Failed"
con.close()
def test_get_osd_status(node_id, ip, port):
request = Req()
node = ServiceInfo(node_id, ip, port)
service_id = "cli"
con = request.connector(IoType.SOCKETIO, node)
resp = request.get_osd_status(service_id, con)
if resp.status.get_status_code() == Resp.SUCCESS:
print "Success"
#mp = resp.message
#print mp
#for i in mp:
# print i, mp[i]
else:
print "Failed"
print resp.status.get_status_code()
print resp.status.get_status_message()
con.close()
def __init__(self, logger):
self.logger = logger
self.conn_obj = None
self.__acc_updater_map_version = 0.0
self.__acc_map_version = 0.0
self.__cont_map_version = 0.0
self.__global_map_version = 0.0
self.__global_map = {}
self.__acc_updater_map = []
self.__acc_map = []
self.__cont_map = []
self.__ownership_list = []
self.__get_comp_counter_queue = Queue.Queue()
self.__container_sweeper_list = []
self.__condition = threading.Condition()
self.__transfer_comp_list = []
#self.__getcmpmap = threading.Event()
self.__complete_all_event = threading.Event()
self.__transfer_cmp_event = threading.Event()
self.__accept_cmp_event = threading.Event()
self.request_handler = Req(self.logger)
def do_TRANSFER_COMPONENTS(self):
"""
Handle TRANSFER COMPONENTS HTTP request
"""
self.logger = SimpleLogger(conf=None).get_logger_object()
self.conf = SimpleLogger(conf=None).get_conf()
self.msg = GlobalVariables(self.logger)
self._request_handler = Req()
transfer_component_timeout = int(
self.conf.get('\
transfer_component_timeout', 600))
self.ll_port = int(self.conf.get('llport', 61014))
self.service_id = self.msg.get_service_id()
self.deleted_comp_list = []
transfer_component_map = {
} #dictionary containing{'(dest_node_obj)':'[comp_list]'}
#eg: {('169.254.1.12', '61009', 'HN0101_61014_account-updater'):'['1', '2']'}
self.final_transfer_status_list = [
] #final component status list which will be send to GL
#[(1, True),(2, False),(3, True)]
self.final_status = False #final response to GL
self.check_transfer_component_map = {
} # to check if component transfer completed or failed:
#{dest_node_obj1:True, dest_node_obj2:"Failed", dest_node_obj3:False}
all_transfer_event_received = []
self.protocol_version = "HTTP/1.1"
self.logger.info("Account-updater received Transfer component request")
try:
content_length = int(self.headers['Content-Length'])
#sending acknowledgement of TRANSFER_COMPONENT to GL
self.send_response(HTTP_CONTINUE, "Continue\r\n\r\n")
data = self.rfile.read(content_length)
message = TransferCompMessage()
message.de_serialize(data)
transfer_component_map = message.service_comp_map()
self.logger.info("Deserialized transfer component map:%s" \
%transfer_component_map)
except Exception, ex:
self.logger.error("Exception: %s" % ex)
self.send_header('Message-Type', \
typeEnums.TRANSFER_COMPONENT_RESPONSE)
self.send_response(HTTP_INTERNAL_SERVER_ERROR)
self.end_headers()
return
class GlobalVariables:
__metaclass__ = SingletonType
def __init__(self, logger):
self.logger = logger
self.conn_obj = None
self.__acc_updater_map_version = 0.0
self.__acc_map_version = 0.0
self.__cont_map_version = 0.0
self.__global_map_version = 0.0
self.__global_map = {}
self.__acc_updater_map = []
self.__acc_map = []
self.__cont_map = []
self.__ownership_list = []
self.__get_comp_counter_queue = Queue.Queue()
self.__container_sweeper_list = []
self.__condition = threading.Condition()
self.__transfer_comp_list = []
#self.__getcmpmap = threading.Event()
self.__complete_all_event = threading.Event()
self.__transfer_cmp_event = threading.Event()
self.__accept_cmp_event = threading.Event()
self.request_handler = Req(self.logger)
def put_into_Queue(self):
self.__get_comp_counter_queue.put(True)
def get_from_Queue(self):
try:
comp_counter = self.__get_comp_counter_queue.get(block=True, \
timeout=300)
return comp_counter
except Queue.Empty:
self.logger.error("Queue is empty")
raise Queue.Empty("Nothing in queue")
def load_ownership(self):
self.logger.info("Loading my ownership")
retry = 3
counter = 0
self.ownership_status = False
while counter < retry:
counter += 1
try:
self.gl_info = self.request_handler.get_gl_info()
self.conn_obj = self.request_handler.connector(\
IoType.SOCKETIO, self.gl_info)
if self.conn_obj != None:
result = self.request_handler.get_comp_list(\
self.__service_id, self.conn_obj)
if result.status.get_status_code() == Resp.SUCCESS:
self.__ownership_list = result.message
self.logger.debug("Ownership list:%s" %
self.__ownership_list)
self.ownership_status = True
self.conn_obj.close()
return
else:
self.logger.warning("get_comp_list() error code:%s msg:%s"\
% (result.status.get_status_code(), \
result.status.get_status_message()))
self.conn_obj.close()
except Exception as ex:
self.logger.exception("Exception occured during load_ownership"
":%s" % ex)
if self.conn_obj:
self.conn_obj.close()
self.logger.warning("Failed to get comp list, Retry: %s" % counter)
time.sleep(10)
def get_my_ownership(self):
self.logger.info("Sending request for retrieving my ownership")
self.load_ownership()
if not self.ownership_status:
self.logger.error("Get my ownership list failed")
os._exit(130)
while not self.__ownership_list:
self.logger.debug(
"Ownership list is empty, retry after 20 seconds ")
time.sleep(20)
self.load_ownership()
def set_ownershipList(self, ownership_list):
self.__ownership_list = ownership_list
def get_ownershipList(self):
return self.__ownership_list
def load_gl_map(self):
self.logger.info("Sending request for retrieving global map")
retry = 3
counter = 0
while counter < retry:
counter += 1
try:
self.gl_info = self.request_handler.get_gl_info()
self.conn_obj = self.request_handler.connector(
IoType.SOCKETIO, self.gl_info)
if self.conn_obj != None:
result = self.request_handler.get_global_map(
self.__service_id, self.conn_obj)
if result.status.get_status_code() == Resp.SUCCESS:
self.__global_map = result.message.map()
self.__global_map_version = result.message.version()
self.logger.info("Global map version: %s" %
(self.__global_map_version))
self.__acc_updater_map = self.__global_map[
'accountUpdater-server'][0]
self.__acc_updater_map_version = self.__global_map[
'accountUpdater-server'][1]
self.__acc_map = self.__global_map['account-server'][0]
self.__acc_map_version = self.__global_map[
'account-server'][1]
self.__cont_map = self.__global_map[
'container-server'][0]
self.__cont_map_version = self.__global_map[
'container-server'][1]
self.conn_obj.close()
return True
else:
self.logger.warning("load_gl_map() error code: %s msg: %s" \
% (result.status.get_status_code(), \
result.status.get_status_message()))
self.conn_obj.close()
except Exception as ex:
self.logger.exception("Exception occured during get_global_map"
":%s" % ex)
if self.conn_obj:
self.conn_obj.close()
self.logger.warning("Failed to get global map, Retry: %s" %
counter)
self.logger.error("Failed to load global map")
return False
def get_account_map(self):
while not self.__acc_map:
self.load_gl_map()
time.sleep(10)
return self.__acc_map
def get_container_map(self):
while not self.__cont_map:
self.load_gl_map()
time.sleep(10)
return self.__cont_map
def get_acc_updater_map(self):
while not self.__acc_updater_map:
self.load_gl_map()
time.sleep(10)
return self.__acc_updater_map
def get_acc_updater_map_version(self):
return self.__acc_updater_map_version
def get_global_map_version(self):
return self.__global_map_version
def set_service_id(self, service_id):
self.__service_id = service_id
def get_service_id(self):
return self.__service_id
def create_sweeper_list(self, comp_tuple):
self.__container_sweeper_list.append(comp_tuple)
def get_sweeper_list(self):
return self.__container_sweeper_list
def pop_from_sweeper_list(self, temp_list):
self.__container_sweeper_list = temp_list
def get_conditionVariable(self):
return self.__condition
def set_transferCompList(self, comp_list):
self.__transfer_comp_list = comp_list
def get_transferCompList(self):
return self.__transfer_comp_list
"""
def get_componentGetEventObject(self):
return self.__getcmpmap
"""
def get_complete_all_event(self):
return self.__complete_all_event
def get_accept_cmp_event(self):
return self.__accept_cmp_event
def get_transfer_cmp_event(self):
return self.__transfer_cmp_event
def __init__(self, file_location, logger=None):
self.logger = logger
self.__ring_obj = RingFileReadHandler(file_location)
self._fs_list = []
self._account_level_dir_list = []
self._container_level_dir_list = []
self._object_level_dir_list = []
self._global_map = {}
self._global_map_version = 0.0
self._account_service_map = []
self._account_service_map_version = None
self._container_service_map = []
self._container_service_map_version = None
self._object_service_map = []
self._object_service_map_version = None
self._account_updater_map = []
self._account_updater_map_version = None
self._update_map_request = False
self.gl_map_status = False
self._result_gl_obj = None
self.conn_obj = None
for i in range(fs_list_count):
self._fs_list.append(self.__ring_obj.get_fs_list()[i])
for i in range(account_dir_count):
self._account_level_dir_list.append(self.__ring_obj.\
get_account_level_dir_list()[i])
for i in range(container_dir_count):
self._container_level_dir_list.append(self.__ring_obj.\
get_container_level_dir_list()[i])
for i in range(object_dir_count):
self._object_level_dir_list.append(self.__ring_obj.\
get_object_level_dir_list()[i])
#GL map maintainer
self.request_handler = Req(self.logger)
self._service_id = get_own_service_id()
self._result_gl_obj = self.load_global_map()
if self.gl_map_status:
self._global_map = self._result_gl_obj.message.map()
self._global_map_version = self._result_gl_obj.message.version()
self._account_service_map = self._global_map['account-server'][0]
self._account_service_map_version = \
self._global_map['account-server'][1]
self._container_service_map = \
self._global_map['container-server'][0]
self._container_service_map_version = \
self._global_map['container-server'][1]
self._object_service_map = self._global_map['object-server'][0]
self._object_service_map_version = \
self._global_map['object-server'][1]
self._account_updater_map = \
self._global_map['accountUpdater-server'][0]
self._account_updater_map_version = \
self._global_map['accountUpdater-server'][1]
self.logger.debug("Global map : %s, Global map version : %s " \
%(self._global_map, self._global_map_version))
if self.conn_obj:
self.conn_obj.close()
class RingData(object):
__metaclass__ = SingletonType
def __init__(self, file_location, logger=None):
self.logger = logger
self.__ring_obj = RingFileReadHandler(file_location)
self._fs_list = []
self._account_level_dir_list = []
self._container_level_dir_list = []
self._object_level_dir_list = []
self._global_map = {}
self._global_map_version = 0.0
self._account_service_map = []
self._account_service_map_version = None
self._container_service_map = []
self._container_service_map_version = None
self._object_service_map = []
self._object_service_map_version = None
self._account_updater_map = []
self._account_updater_map_version = None
self._update_map_request = False
self.gl_map_status = False
self._result_gl_obj = None
self.conn_obj = None
for i in range(fs_list_count):
self._fs_list.append(self.__ring_obj.get_fs_list()[i])
for i in range(account_dir_count):
self._account_level_dir_list.append(self.__ring_obj.\
get_account_level_dir_list()[i])
for i in range(container_dir_count):
self._container_level_dir_list.append(self.__ring_obj.\
get_container_level_dir_list()[i])
for i in range(object_dir_count):
self._object_level_dir_list.append(self.__ring_obj.\
get_object_level_dir_list()[i])
#GL map maintainer
self.request_handler = Req(self.logger)
self._service_id = get_own_service_id()
self._result_gl_obj = self.load_global_map()
if self.gl_map_status:
self._global_map = self._result_gl_obj.message.map()
self._global_map_version = self._result_gl_obj.message.version()
self._account_service_map = self._global_map['account-server'][0]
self._account_service_map_version = \
self._global_map['account-server'][1]
self._container_service_map = \
self._global_map['container-server'][0]
self._container_service_map_version = \
self._global_map['container-server'][1]
self._object_service_map = self._global_map['object-server'][0]
self._object_service_map_version = \
self._global_map['object-server'][1]
self._account_updater_map = \
self._global_map['accountUpdater-server'][0]
self._account_updater_map_version = \
self._global_map['accountUpdater-server'][1]
self.logger.debug("Global map : %s, Global map version : %s " \
%(self._global_map, self._global_map_version))
if self.conn_obj:
self.conn_obj.close()
def load_global_map(self):
retry = 3
counter = 0
self.logger.debug("Sending request for retrieving global map")
self.gl_map_status = False
while counter < retry:
counter += 1
try:
self.gl_info_obj = self.request_handler.get_gl_info()
self.conn_obj = self.request_handler.connector(IoType.EVENTIO, \
self.gl_info_obj)
if self.conn_obj != None:
self._result_gl_obj = self.request_handler.get_global_map(\
self._service_id, self.conn_obj)
if self._result_gl_obj.status.get_status_code() == \
Resp.SUCCESS:
self.gl_map_status = True
return self._result_gl_obj
else:
self.logger.warning("get_global_map() error code:%s msg:%s"\
% (self._result_gl_obj.status.get_status_code(), \
self._result_gl_obj.status.get_status_message()))
self.conn_obj.close()
except Exception as ex:
self.logger.exception("Exception occured during get_global_map"
":%s" % ex)
if self.conn_obj:
self.conn_obj.close()
self.logger.debug("Failed to get global map, Retry: %s" % counter)
self.logger.error("Failed to load global map")
return self._result_gl_obj
def update_all_maps_in_memory(self):
self._global_map = self.updated_map_obj.message.map()
self._global_map_version = \
self.updated_map_obj.message.version()
self._account_service_map = \
self._global_map['account-server'][0]
self._account_service_map_version = \
self._global_map['account-server'][1]
self._container_service_map = \
self._global_map['container-server'][0]
self._container_service_map_version = \
self._global_map['container-server'][1]
self._object_service_map = \
self._global_map['object-server'][0]
self._object_service_map_version = \
self._global_map['object-server'][1]
self._account_updater_map = \
self._global_map['accountUpdater-server'][0]
self._account_updater_map_version = \
self._global_map['accountUpdater-server'][1]
self.logger.info("Updated Global map : %s, Updated Global"\
"map version : %s " %(self._global_map, \
self._global_map_version))
def update_global_map(self):
"""
update global map
"""
if not self._update_map_request:
self.logger.debug("Sending request for global map update " \
"old_map_version:%s" %self._global_map_version)
self._update_map_request = True
self.updated_map_obj = self.load_global_map()
if self.gl_map_status:
map_diff = float_comp(self._global_map_version, \
self.updated_map_obj.message.version())
if map_diff < 0:
self.update_all_maps_in_memory()
else:
self.logger.info("Global map is not updated yet old map" \
" version :%s current map version:%s"
%(self._global_map_version,
self.updated_map_obj.message.version()))
if self.conn_obj:
self.conn_obj.close()
self._update_map_request = False
else:
self.logger.info(
"Global map update request is already in progress")
sleep(10) #need to change
def update_object_map(self):
if not self._update_map_request:
self.logger.debug("Sending request for global map update " \
"old_map_version:%s" %self._global_map_version)
self._update_map_request = True
self.updated_map_obj = self.load_global_map()
if self.gl_map_status:
map_diff = float_comp(self._global_map_version, \
self.updated_map_obj.message.version())
object_service_map_diff = float_comp(\
self._object_service_map_version,
self.updated_map_obj.message.map()['object-server'][1])
if map_diff < 0:
self.update_all_maps_in_memory()
elif object_service_map_diff < 0:
self.logger.debug("Object Service map version updated")
self.update_all_maps_in_memory()
else:
self.logger.info("Global map is not updated yet old map" \
" version :%s current map version:%s"
%(self._global_map_version,
self.updated_map_obj.message.version()))
if self.conn_obj:
self.conn_obj.close()
self._update_map_request = False
else:
self.logger.info(
"Global map update request is already in progress")
sleep(10) #need to change
def get_account_map(self):
"""
Returns account map list
"""
return self._account_service_map
def get_account_map_version(self):
"""
Returns account map version
"""
return self._account_service_map_version
def get_container_map(self):
"""
Return container map list
"""
return self._container_service_map
def get_container_map_version(self):
"""
Returns container map version
"""
return self._container_service_map_version
def get_object_map(self):
"""
Returns object map list
"""
return self._object_service_map
def get_object_map_version(self):
"""
Returns object map version
"""
return self._object_service_map_version
def get_account_updater_map(self):
"""
Returns account-updater map list
"""
return self._account_updater_map
def get_account_updater_map_version(self):
"""
Returns account-updater map version
"""
return self._account_updater_map_version
def get_global_map(self):
"""
Returns global map
"""
return self._global_map
def get_global_map_version(self):
"""
Returns global map version
"""
return self._global_map_version
def get_fs_list(self):
"""
Returns file system list
"""
return self._fs_list
def get_container_level_dir_list(self):
"""
Returns container level directory list
"""
return self._container_level_dir_list
def get_account_level_dir_list(self):
"""
Returns account level directory list
"""
return self._account_level_dir_list
def get_object_level_dir_list(self):
"""
Returns object level directory list
"""
return self._object_level_dir_list
def get_acc_dir_by_hash(self, acc_hash):
"""
Returns account level directory for object storage
:param acc_hash: acc hash
:returns: account level directory
"""
directories = self._account_level_dir_list
shift_param = len(directories)
dir_no = Calculation.evaluate(acc_hash, shift_param)
try:
dir_name = directories[dir_no - 1]
except (IndexError, Exception) as err:
self.logger.exception("Exception occured :%s" % err)
raise err
return dir_name
def get_cont_dir_by_hash(self, cont_hash):
"""
Returns container level directory for object storage
:param cont_hash: container hash
:returns: container level directory
"""
directories = self._container_level_dir_list
shift_param = len(directories)
dir_no = Calculation.evaluate(cont_hash, shift_param)
try:
dir_name = directories[dir_no - 1]
except (IndexError, Exception) as err:
self.logger.exception("Exception occured :%s" % err)
raise err
return dir_name
@classmethod
def load(cls, file_location):
raise NotImplementedError
description="Recovery process option parser")
parser.add_argument("server_id", help="Service id from Global leader.")
args = parser.parse_args()
__service_id = args.server_id
#Change Process name of Script.
script_prefix = "RECOVERY_"
proc_name = script_prefix + __service_id
procname.setprocname(proc_name)
recovery_status = RecoveryStatus()
conf = {
'log_requests': 'false',
'log_failure': 'true',
'log_level': 'debug',
'log_facility': 'LOG_LOCAL6',
'log_name': 'object_recovery'
}
#create logger obj
logger_obj = get_logger(conf, log_route='object-recovery')
logger_obj.info("Service received is : %s of type %s " % \
(__service_id, type(__service_id)))
#create communication object
communicator_obj = Req(logger_obj)
recover = RecoveryObject(conf, logger_obj, communicator_obj,
recovery_status, __service_id)
recover.recover_data()
logger_obj.info("Object Recovery finished")
#Global variables for intermediated response handling
list_of_tuple = queue(maxsize=1000)
finish_recovery_flag = False
final_status = True
final_recovery_status_list = list()
list_to_gl = []
clean_journal_flag = True
#logger initialize
logger_obj = get_logger(config_dict, log_route='recovery')
logger_obj.info("Parsed aguments:%s, configfile: %s, proc %s" % \
(service_id_, config_dict, proc_name))
#Get Global map and retry 2 times.
logger_obj.info("Start global map request")
Request_handler_ = Req(logger_obj, timeout=0)
gl_info_obj = Request_handler_.get_gl_info()
retry_count = 0
while retry_count < 3:
retry_count += 1
communicator_object_ = Request_handler_.connector(
IoType.EVENTIO, gl_info_obj)
if communicator_object_:
logger_obj.debug("CONN object success")
break
if not communicator_object_:
logger_obj.debug("No Connection object Found : Exiting")
sys.exit(130)