def __process_dir(self, tmp_dir):
'''
Process each tmp dir for each files
:param tmp_dir: location of tmp dir to be processed
'''
data_file = []
meta_file = []
try:
files = os.listdir(tmp_dir)
for afile in files:
if afile.endswith('.data'):
data_file.append(afile)
elif afile.endswith('.meta'):
meta_file.append(afile)
else:
pass
except OSError:
self._logger.error(__(
'ERROR No such directory %(tmp_dir)s'),
{'tmp_dir': tmp_dir})
except Exception as err:
self._logger.error(__(
'ERROR While recovering directory %(tmp_dir)s'
' close failure: %(exc)s : %(stack)s'),
{'exc': err, 'stack': ''.join(traceback.format_stack()),
'tmp_dir': tmp_dir})
raise
return data_file, meta_file
def recover_delete_request(self, data_file_path, meta_file_path, path, host):
"""
Does triggered recovery for object PUT operations
:param data_file_path: data file path
:param meta_file_path: metadata file path
:param path: object path
:param host: string containing container service details
"""
if os.path.exists(data_file_path) and os.path.exists( \
meta_file_path):
release_status = self.__communicator_obj. \
release_lock(path, host=host)
if release_status:
self._logger.info(__("Release lock successful for "
"file: %s" % path))
else:
self._logger.error("Could not release lock for "
"file: %s" % path)
else:
if os.path.exists(meta_file_path):
with open(meta_file_path, 'rb') as meta_fd:
orig_metadata = pickle.loads(meta_fd.read())
metadata = HeaderKeyDict({'name': orig_metadata['name'],
'x-size': orig_metadata['Content-Length'],
'X-Timestamp': orig_metadata['X-Timestamp']})
# TODO:
# Now there is no way to identify whether the update
# request is overwrite case or not. Hence sending request
# for new entry.
status = self.__communicator_obj. \
container_update(metadata, \
method='DELETE', host=host, unknown=True)
if not status:
self._logger.error(__("Could not update container "
"for file: %s" % path))
else:
self._logger.info(__("Container update successful for "
"file: %s" % path))
os.remove(meta_file_path)
release_status = self.__communicator_obj. \
release_lock(path, host=host)
if release_status:
self._logger.info(__("Release lock successful for "
"file: %s" % path))
else:
self._logger.error("Could not release lock for "
"file: %s" % path)
else:
release_status = self.__communicator_obj. \
release_lock(path, host=host)
if release_status:
self._logger.info(__("Release lock successful for "
"file: %s" % path))
else:
self._logger.error("Could not release lock for "
"file: %s" % path)
def release_lock(self, obj_hash, metadata=None, host=''):
'''
Send request to container service for releasing transaction lock.
:param obj_hash: object hash
:returns: True, if lock is released
False, otherwise.
'''
# TODO: In case of recovery of incomplete files, we do not have
# account, container, obj name. So evaluating container service
# ip, port to which request will be submitted to release lock is
# not possible at this time. So this is a just a work around to
# get container service details.
# In future, some other solution can be implemented to better cope
# with such situation.
directory, filesystem = '', ''
if not metadata:
try:
try:
self._logger.info("obj hash : %s " % obj_hash)
#hash(/acc/cont)
cont_hash = obj_hash.split('/')[1]
#hash(/acc)
acc_hash = obj_hash.split('/')[0]
except Exception as err:
self._logger.error("Exception raised")
service_details, filesystem, directory = \
self.__container_ring.get_service_list(acc_hash, cont_hash)
self._logger.info("ip : port in release lock : %s" % service_details)
#service_list = ([{'ip': u'169.254.1.52', 'port': 61007}], filesystem, directory)
if not host:
host = '%s:%s' % (service_details[0]['ip'], service_details[0]['port'])
self._logger.debug(" Details -> filesystem: %s directory: %s" % \
(filesystem, directory))
except Exception as err:
self._logger.error(__("Error raised in release lock %s" \
% err))
else:
host, filesystem, directory = self.__get_service_details(metadata, host=host)
try:
header = HeaderKeyDict({'x-object-path': obj_hash, 'x-global-map-version': '-1'})
resp = http_connection(host, filesystem, directory, \
header, 'FREE_LOCKS')
except (Exception, Timeout):
self._logger.exception(__(
'ERROR Release lock failed '
'%(ip)s for %(obj_name)s'),
{'ip': host,
'obj_name': obj_hash})
return False
resp.read()
if not is_success(resp.status):
return False
return True
def startup_recovery(self, __get_node_ip=None, __port=None, __ll_port=None, __serv_id=None, recovery_flag=False):
'''
Start the start up recovery procedure.
'''
self._logger.info("Starting start up recovery process for "
"object server: %s" % self.__object_service_id)
try:
create_recovery_file('object-server')
except Exception as err:
self._logger.error('Failed to create recovery file')
return False
#jai:-To recover data directory wise
#we would need to comment out this section
#try:
# self._move_data()
#except (OSError, Exception):
# return False
tmp_dir_list_to_recover = self._get_tmp_dir_list()
try:
for tmp_dir in tmp_dir_list_to_recover:
try:
self.__directory_iterator_obj.recover(tmp_dir)
except (OSError, Exception) as err:
self._logger.error(__(
'ERROR Could not complete startup recovery process '
'for tmp directory: %(tmp_dir)s'
' close failure: %(exc)s : %(stack)s'),
{'exc': err, 'stack': ''.join(traceback.format_stack()),
'tmp_dir': tmp_dir})
return False
except Exception as err:
self._logger.error(__(
'ERROR Could not complete startup recovery process for '
'object server: %(object_service_id)s'
' close failure: %(exc)s : %(stack)s'),
{'exc': err, 'stack': ''.join(traceback.format_stack()),
'object_service_id': self.__object_service_id})
return False
finally:
try:
# Start sending healthMonitoring to local leader
if not recovery_flag:
self.health = healthMonitoring(__get_node_ip, __port, __ll_port, __serv_id)
self._logger.debug("Health monitoring instance executed")
status = None
remove_recovery_file('object-server')
except Exception as err:
self._logger.error('Failed to remove recovery file %s'% err)
status = True
if status:
return False
return True
def recover(self, tmp_dir):
'''
Calls recovery for each tmp file.
:param tmp_dir: tmp directory
'''
status = None
self._logger.info("Starting recovery for tmp dir: %s" % tmp_dir)
try:
data_file_list, meta_file_list = self.__process_dir(tmp_dir)
except Exception:
raise
if not data_file_list and not meta_file_list:
self._logger.info(__(
'No tmp files found in %(tmp_dir)s'),
{'tmp_dir': tmp_dir})
status = True
else:
try:
status = self._recover_each_tmp_dir(tmp_dir, data_file_list, \
meta_file_list)
except Exception as err:
self._logger.exception((
'ERROR failure in startup recovery: %(tmp_dir)s'
' close failure: %(exc)s : %(stack)s'),
{'exc': err, 'stack': ''.join(traceback.format_stack()),
'tmp_dir' : tmp_dir})
raise
if status:
self._logger.info("Recovery completed for tmp_dir: %s" % tmp_dir)
else:
self._logger.info("Could not complete recovery "
"for tmp_dir: %s" % tmp_dir)
def _move_data(self):
'''
Move data from old node to current node id
'''
tmp_dir_list = self._get_tmp_dir_list_to_move()
try:
for tmp_dir in tmp_dir_list:
old_node_id = tmp_dir.split('/')[-1].split('_')[0]
current_node_id = self.__object_service_id.split('_')[0]
new_tmp_dir = tmp_dir.replace(old_node_id, current_node_id)
if os.path.exists(new_tmp_dir):
if os.listdir(tmp_dir):
for root, _, files in os.walk(tmp_dir):
for afile in files:
shutil.move(os.path.join(root, afile), new_tmp_dir)
shutil.rmtree(tmp_dir)
else:
os.rename(tmp_dir, new_tmp_dir)
except (OSError, Exception) as err:
self._logger.error(__(
'ERROR While moving data from %(old)s to %(new)s '
' close failure: %(exc)s : %(stack)s'),
{'exc': err, 'stack': ''.join(traceback.format_stack()),
'old': old_node_id, 'new': current_node_id})
raise
def recover_put_request(self, data_file_path, meta_file_path, path, host):
"""
Does triggered recovery for object PUT operations
:param data_file_path: data file path
:param meta_file_path: metadata file path
:param path: object path
:param host: string containing container service details
"""
if os.path.exists(data_file_path) and os.path.exists( \
meta_file_path):
with open(meta_file_path, 'rb') as meta_fd:
orig_metadata = pickle.loads(meta_fd.read())
metadata = dict(
[(key, val) for key, val in orig_metadata.iteritems()
if key.lower() in DATAFILE_SYSTEM_META])
metadata.update({'name': orig_metadata['name']})
metadata.update({'X-Timestamp': orig_metadata['X-Timestamp']})
# TODO:
# Now there is no way to identify whether the update
# request is overwrite case or not. Hence sending request
# for new entry.
status = self.__communicator_obj. \
container_update(metadata, host=host, unknown=True)
if not status:
self._logger.error(__("Could not update container "
"for file: %s" % path))
else:
self._logger.info(__("Container update successful for "
"file: %s" % path))
release_status = self.__communicator_obj. \
release_lock(path, host=host)
if release_status:
self._logger.info(__("Release lock successful for "
"file: %s" % metadata['name']))
else:
self._logger.error("Could not release lock for "
"file: %s" % metadata['name'])
else:
release_status = self.__communicator_obj. \
release_lock(path, host=host)
if release_status:
self._logger.info(__("Release lock successful for "
"file: %s" % path))
else:
self._logger.error("Could not release lock for "
"file: %s" % path)
def container_update(self, metadata, duplicate=False,
method='PUT', host=None, unknown=False):
'''
Send request to container service for info file update
:param metadata: object metadata
:param duplicate: If set to true, then container update request
will contain 'x-duplicate-update: True' header
else it not contain any additional header.
:param method: method name
:param host: host IP and port
:returns: True, If update successful
False, otherwise.
'''
if not metadata:
header = HeaderKeyDict({'x-timestamp': \
normalize_timestamp(time.time())})
file_system, directory = '', ''
else:
host, directory, file_system = self.__get_service_details(metadata, host=host)
try:
header = HeaderKeyDict({
'x-global-map-version': '-1',
'x-size': metadata['Content-Length'],
'x-content-type': metadata['Content-Type'],
'x-timestamp': metadata['X-Timestamp'],
'x-etag': metadata['ETag']})
except KeyError:
return False
header.update({'x-object-path': metadata['name']})
if duplicate:
header.update({'x-duplicate-update': True})
if unknown:
header.update({'x-duplicate-unknown': True})
try:
resp = http_connection(host, file_system, directory, \
header, method)
except (Exception, Timeout):
self._logger.exception(__(
'ERROR container update failed '
'%(host)s/%(fs)s '),
{'host': host,
'fs': file_system})
return False
resp.read()
if not is_success(resp.status):
return False
return True
def triggered_recovery(self, path, method, host):
'''
Start the triggered recovery procedure.
:param path: object path to be recovered
:returns: True, when file is recoverd successfully
False, otherwise.
'''
try:
self._logger.info('Starting triggered recovery for file: %s'
% path)
self.__triggered_recovery_handler_obj.recover(path, method, host)
except Exception as err:
self._logger.error(__(
'ERROR Could not complete triggered recovery '
'process for file: %(file)s'
'close failure: %(exc)s : %(stack)s'),
{'exc': err, 'stack': ''.join(traceback.format_stack()),
'file': path})
self._logger.info('Successfully completed triggered recovery '
'for file: %s' % path)
def recover_complete_file(self, tmp_dir, file_name, only_meta=False):
'''
Writes file to actual location and call container update request method
and then finally release lock request.
:param tmp_dir: tmp directory
:param file_name: file name to be recovered
:param only_meta: flag to identify only meta file case
'''
update_status, release_status = None, None
metadata = None
duplicate = False
try:
meta_file = file_name + '.meta'
with open(os.path.join(tmp_dir, meta_file), 'rb') as meta_fd:
orig_metadata = pickle.loads(meta_fd.read())
metadata = dict(
[(key, val) for key, val in orig_metadata.iteritems()
if key.lower() in DATAFILE_SYSTEM_META])
metadata.update({'name': orig_metadata['name']})
metadata.update({'X-Timestamp': orig_metadata['X-Timestamp']})
except (Exception, IOError) as err:
self._logger.error(__(
'ERROR While reading %(meta)s file'
' close failure: %(exc)s : %(stack)s'),
{'exc': err, 'stack': ''.join(traceback.format_stack()),
'meta' : os.path.join(tmp_dir, file_name + '.meta')})
raise
obj_hash = file_name.replace('_', '/')
data_target_path, meta_target_path = self.__get_target_path(metadata,
file_name)
data_file = os.path.join(tmp_dir, file_name) + '.data'
meta_file = os.path.join(tmp_dir, file_name) + '.meta'
if os.path.exists(data_target_path) and os.path.exists(meta_target_path):
duplicate = True
try:
if not only_meta:
mkdirs(os.path.dirname(data_target_path))
os.rename(data_file, data_target_path)
mkdirs(os.path.dirname(meta_target_path))
os.rename(meta_file, meta_target_path)
except OSError:
self._logger.error("Failure during file renaming")
raise
self._logger.info("Sending request to container service "
"for file: %s" % data_file)
update_status = self.__communicator_obj. \
container_update(metadata, duplicate)
if not update_status:
self._logger.error("Could not update container")
else:
self._logger.info(__("Container update successful for "
"file: %s" % data_file))
self._logger.info(__("Sending request for releasing "
"lock: %s" % data_file))
release_status = self.__communicator_obj. \
release_lock(obj_hash, metadata)
if release_status:
self._logger.info(__("Release lock successful for "
"file: %s" % metadata['name']))
else:
self._logger.error("Could not release lock for "
"file: %s" % metadata['name'])