class ContainerStatReader:
"""
Read the container stat info from container file.
"""
def __init__(self, container_name, container_path, account_name, conf, \
logger):
"""
Constructor for RecordReader.
:param container_name: container name
:param container_path: container path
"""
self.__container_name = container_name
self.__container_path = container_path
self.__account_name = account_name
self.stat_info = {}
self.conf = conf
self.logger = logger
self.msg = GlobalVariables(self.logger)
self.retry_count = 3
#self.logger.debug("ContainerStatReader constructed")
self.connection_creator = ConnectionCreator(self.conf, self.logger)
self.__is_completed = False
def __repr__(self):
return "container : %s account: %s" %(self.__container_name, self.__account_name)
def is_completed(self):
return self.__is_completed
def read_container_stat_info(self, account_map):
"""
Get the stat info of container using H-File interface.
"""
try:
conn = None
self.logger.debug("Enter in read_container_stat_info for container")
conn = self.connection_creator.connect_container('HEAD', \
self.__account_name, self.__container_name, self.__container_path)
resp = self.connection_creator.get_http_response_instance(conn)
while resp.status != 204 and resp.status != 404 and self.retry_count != 0:
self.retry_count -= 1
if resp.status == 301 or resp.status == 307 resp.status == 503:
self.msg.load_gl_map()
conn = self.connection_creator.connect_container('HEAD', \
self.__account_name, self.__container_name, self.__container_path)
resp = self.connection_creator.get_http_response_instance(conn)
if resp.status == 204:
headers = dict(resp.getheaders())
self.stat_info[self.__container_name] = \
{'container' : headers['x-container'], \
'put_timestamp' : headers['x-put-timestamp'] , \
'object_count' : headers['x-container-object-count'], \
'bytes_used' : headers['x-container-bytes-used'], \
'delete_timestamp' : '0',
'deleted' : False}
account_map[self.__account_name, self.__container_name] = \
"success"
self.__is_completed = True
self.logger.info("container stats for %s: %s" \
%(self.__container_name, self.stat_info[self.__container_name]))
elif resp.status == 404:
self.logger.debug("Container info file %s not found" \
% self.__container_path)
self.stat_info[self.__container_name] = \
{'container' : '', \
'put_timestamp' : '0', \
'object_count' : 0, \
'bytes_used' : 0, \
'delete_timestamp' : '0',
'deleted' : True}
account_map[self.__account_name, self.__container_name] = \
"success"
self.__is_completed = True
else:
self.logger.warning("Could not read stats form container: %s" \
% self.__container_name)
except Exception as ex:
self.logger.error("While getting container stats for:%s, Error: %s" \
% (self.__container_name, ex))
class AccountUpdater(Daemon):
"""
Update container information in account listings.
"""
def __init__(self, conf, logger=None):
"""
constructor for account updater
:param conf: configuration of account-updater
"""
self.logger = logger or SimpleLogger(conf).get_logger_object()
Daemon.__init__(self, conf, self.logger)
libraryUtils.OSDLoggerImpl(
"account-updater-monitoring").initialize_logger()
create_recovery_file('account-updater-server')
self.conf = conf
self.__interval = int(conf.get('interval', 1800))
self.__ll_port = int(conf.get('llport', 61014))
self.__account_updater_port = int(\
conf.get('account_updater_port', 61009))
self.__service_id = gethostname() + "_" + str(self.__ll_port) + \
"_account-updater-server"
self.__param = self.__get_param(conf)
self.msg = GlobalVariables(self.logger)
self.msg.set_service_id(self.__service_id)
self.walker_map = WalkerMap()
self.reader_map = ReaderMap()
# Start sending health to local leader
self.logger.info("Loading health monitoring library")
self.health_instance = healthMonitoring(self.__get_node_ip\
(gethostname()), self.__account_updater_port, \
self.__ll_port, self.__service_id)
self.logger.info("Loaded health monitoring library")
remove_recovery_file('account-updater-server')
# load global map
if not self.msg.load_gl_map():
sys.exit(130)
self.logger.info("Account updater started")
def __get_node_ip(self, hostname):
"""
Get internal node ip on which service is running
"""
try:
command = "grep -w " + hostname + " /etc/hosts | awk {'print $1'}"
child = subprocess.Popen(command, stdout = subprocess.PIPE, \
stderr = subprocess.PIPE, shell = True)
std_out, std_err = child.communicate()
return std_out.strip()
except Exception as err:
self.logger.error("Error occurred while getting ip of node:%s" %
err)
return ""
def __get_param(self, conf):
"""
Getting parameters through config file
:param conf: configuration file of account-updater
"""
return {
'filesystems': conf.get('filesystems', '/export'),
'stat_file_location': conf.get('stat_file_location', \
'/export/OSP_01/updater'),
'file_expire_time': conf.get('file_expire_time', 900),
'interval': conf.get('interval', 600),
'conn_timeout': float(conf.get('conn_timeout', 10)),
'node_timeout': float(conf.get('node_timeout', 10)),
'osd_dir': conf.get('osd_dir', '/export/.osd_meta_config'),
'reader_max_count': int(conf.get('reader_max_count', 10)),
'max_update_files': int(conf.get('max_update_files', 10)),
'max_files_stored_in_cache': int(conf.get('max_files_stored_in_cache', 100)),
'updaterfile_expire_delta_time': int(conf.get('updaterfile_expire_delta_time', 1020))
}
def run_forever(self, *args, **kwargs):
"""
Run the updator continuously.
"""
try:
self.logger.debug('Begin account update')
# get account-updater server ownership
self.get_ownership_obj = threading.Thread(
target=self.msg.get_my_ownership)
self.get_ownership_obj.setDaemon(True)
self.get_ownership_obj.start()
self.walker_obj = Walker(self.walker_map, self.__param,
self.logger)
self.walker_obj.setDaemon(True)
self.walker_obj.start()
self.logger.info("Walker Started")
self.reader_obj = Reader(self.walker_map, self.reader_map, \
self.__param, self.logger)
self.reader_obj.setDaemon(True)
self.reader_obj.start()
self.logger.info("Reader Started")
self.account_sweeper = AccountSweep(self.__param, self.logger)
self.account_sweeper.setDaemon(True)
self.account_sweeper.start()
self.logger.info("Account Sweeper Started")
self.updater_obj = Updater(self.walker_map, self.reader_map, \
self.__param, self.logger)
self.updater_obj.setDaemon(True)
self.updater_obj.start()
self.logger.info("Updater Started")
self.container_sweeper = ContainerSweeper(self.walker_map, \
self.reader_map, self.__param, self.logger)
self.container_sweeper.setDaemon(True)
self.container_sweeper.start()
self.logger.info("Container Sweeper Started")
account_updater_server = ThreadedAccountUpdaterServer(\
(self.__get_node_ip(gethostname()), \
self.__account_updater_port), HttpListener)
account_updater_server.serve_forever()
except Exception as ex:
self.logger.error("Exception occured: %s" % ex)
class TestGlobalVariables(unittest.TestCase):
@mock.patch("osd.accountUpdaterService.monitor.Req", mockReq)
@mock.patch("osd.accountUpdaterService.monitor.Resp", mockResp)
@mock.patch("osd.accountUpdaterService.monitor.ServiceInfo",
mockServiceInfo)
def setUp(self):
self.global_var = GlobalVariables(logger)
self.global_var.set_service_id("service_id")
def test_load_gl_map(self):
#self.assertTrue(self.global_var.load_gl_map())
with mock.patch(
'osd.accountUpdaterService.unitTests.mockReq.connector',
return_value=None):
self.assertFalse(self.global_var.load_gl_map())
with mock.patch(
'osd.accountUpdaterService.unitTests.mockStatus.get_status_code',
return_value="Resp.FAILURE"):
self.assertFalse(self.global_var.load_gl_map())
def test_load_ownership(self):
with mock.patch(
'osd.accountUpdaterService.unitTests.mockReq.connector',
return_value=None):
self.global_var.load_ownership()
self.assertEquals(self.global_var.get_ownershipList(), [])
with mock.patch(
'osd.accountUpdaterService.unitTests.mockStatus.get_status_code',
return_value="Resp.FAILURE"):
self.global_var.load_ownership()
self.assertEquals(self.global_var.get_ownershipList(), [])
self.global_var.load_ownership()
self.assertEquals(self.global_var.get_ownershipList(), range(1, 513))
def test_get_account_map(self):
self.assertEquals(
[obj.get_id() for obj in self.global_var.get_account_map()],
acc_id)
self.assertEquals(
[obj.get_ip() for obj in self.global_var.get_account_map()],
acc_ip)
self.assertEquals(
[obj.get_port() for obj in self.global_var.get_account_map()],
acc_port)
def test_get_container_map(self):
self.assertEquals(
[obj.get_id() for obj in self.global_var.get_container_map()],
cont_id)
self.assertEquals(
[obj.get_ip() for obj in self.global_var.get_container_map()],
cont_ip)
self.assertEquals(
[obj.get_port() for obj in self.global_var.get_container_map()],
cont_port)
def test_get_acc_updater_map(self):
self.assertEquals(
[obj.get_id() for obj in self.global_var.get_acc_updater_map()],
au_id)
self.assertEquals(
[obj.get_ip() for obj in self.global_var.get_acc_updater_map()],
au_ip)
self.assertEquals(
[obj.get_port() for obj in self.global_var.get_acc_updater_map()],
au_port)
def test_get_acc_updater_map_version(self):
self.assertEquals(self.global_var.get_acc_updater_map_version(), "5.0")
def test_get_global_map_version(self):
self.assertEquals(self.global_var.get_global_map_version(), "5.0")
def test_get_service_id(self):
self.assertEquals(self.global_var.get_service_id(), "service_id")
class AccountServiceCommunicator:
"""
Communicate with the account service.
"""
def __init__(self, conf, logger):
"""
Constructor of AccountServiceCommunicator.
"""
self.logger = logger
#self.logger.debug("AccountServiceCommunicator constructed")
self.__conn_creator = ConnectionCreator(conf, self.logger)
self.msg = GlobalVariables(self.logger)
def __send_http_request_to_account_service(self, account_name, \
stat_info):
"""
Send the request to the account service.
:param conn_timeout: connection time to account service
:param node_timeout: response time to account service
:return resp: response from account service
"""
info = {
'recovery_flag': False,
'method_name': 'PUT_CONTAINER_RECORD',
'entity_name': '',
'body_size': len(str(stat_info)),
'account_name': account_name,
'flag': False
}
return self.__conn_creator.get_http_connection_instance(info, \
str(stat_info))
def update_container_stat_info(self, account_name = None, \
stat_info = None):
"""
Update the container information to account service.
"""
updated = False
retry_count = 3
self.logger.debug(
'Sending http request to account service for updation.')
conn = None
try:
conn = self.__send_http_request_to_account_service(
account_name, stat_info)
resp = self.__conn_creator.get_http_response_instance(conn)
self.logger.info("Response from account service: status: %s," \
" message: %s" % (resp.status, resp.read()))
while resp.status != 202 and resp.status != 404 and retry_count != 0:
retry_count -= 1
if resp.status == 301 or resp.status == 307:
self.msg.load_gl_map()
conn = self.__send_http_request_to_account_service(
account_name, stat_info)
resp = self.__conn_creator.get_http_response_instance(conn)
self.logger.info("Response from account service: status: %s," \
" message: %s" % (resp.status, resp.read()))
if resp.status == 202 or resp.status == 404:
updated = True
except Exception as ex:
self.logger.error("Error while updating container stat info: %s" %
ex)
finally:
if conn:
conn.close()
return updated