def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
"""
Fetch the submanifest, parse it, and return it.
Raise exception on failures.
"""
sub_req = make_subrequest(
req.environ,
path='/'.join(['', version, acc, con, obj]),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'SLO MultipartGET'),
swift_source='SLO')
sub_resp = sub_req.get_response(self.slo.app)
if not is_success(sub_resp.status_int):
raise ListingIterError(
'ERROR: while fetching %s, GET of submanifest %s '
'failed with status %d' %
(req.path, sub_req.path, sub_resp.status_int))
try:
with closing_if_possible(sub_resp.app_iter):
return json.loads(''.join(sub_resp.app_iter))
except ValueError as err:
raise ListingIterError(
'ERROR: while fetching %s, JSON-decoding of submanifest %s '
'failed with %s' % (req.path, sub_req.path, err))
def direct_delete_container(node,
part,
account,
container,
conn_timeout=5,
response_timeout=15,
headers=None):
if headers is None:
headers = {}
path = '/%s/%s' % (account, container)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'DELETE',
path,
headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Container server %s:%s direct DELETE %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
return resp.status
def release_lock(self, host, file_system, directory, data):
"""
Release lock from container
param host: Host information of container
param file_system: File system name in which object resides
param directory: Directory name in which object resides
param data: Request header dictionary
return response: HTTP response
"""
method_type = 'FREE_LOCK'
try:
response = http_connection(host, file_system, directory, data, \
method_type)
response.read()
if not is_success(response.status):
self._logger.error(('ERROR method %(method)s failed '
'response status: %(status)d '), {
'method': method_type,
'status': response.status
})
return response
except Exception as err:
self._logger.error(('ERROR release_lock failed '
'%(host)s/%(fs)s/%(dir)s '), {
'host': host,
'fs': file_system,
'dir': directory
})
self._logger.exception(err)
raise err
def release_lock(self, obj_hash, metadata=None, host=''):
'''
Send request to container service for releasing transaction lock.
:param obj_hash: object hash
:returns: True, if lock is released
False, otherwise.
'''
# TODO: In case of recovery of incomplete files, we do not have
# account, container, obj name. So evaluating container service
# ip, port to which request will be submitted to release lock is
# not possible at this time. So this is a just a work around to
# get container service details.
# In future, some other solution can be implemented to better cope
# with such situation.
directory, filesystem = '', ''
if not metadata:
try:
try:
self._logger.info("obj hash : %s " % obj_hash)
#hash(/acc/cont)
cont_hash = obj_hash.split('/')[1]
#hash(/acc)
acc_hash = obj_hash.split('/')[0]
except Exception as err:
self._logger.error("Exception raised")
service_details, filesystem, directory = \
self.__container_ring.get_service_list(acc_hash, cont_hash)
self._logger.info("ip : port in release lock : %s" % service_details)
#service_list = ([{'ip': u'169.254.1.52', 'port': 61007}], filesystem, directory)
if not host:
host = '%s:%s' % (service_details[0]['ip'], service_details[0]['port'])
self._logger.debug(" Details -> filesystem: %s directory: %s" % \
(filesystem, directory))
except Exception as err:
self._logger.error(__("Error raised in release lock %s" \
% err))
else:
host, filesystem, directory = self.__get_service_details(metadata, host=host)
try:
header = HeaderKeyDict({'x-object-path': obj_hash, 'x-global-map-version': '-1'})
resp = http_connection(host, filesystem, directory, \
header, 'FREE_LOCKS')
except (Exception, Timeout):
self._logger.exception(__(
'ERROR Release lock failed '
'%(ip)s for %(obj_name)s'),
{'ip': host,
'obj_name': obj_hash})
return False
resp.read()
if not is_success(resp.status):
return False
return True
def _get_direct_account_container(path,
stype,
node,
part,
account,
marker=None,
limit=None,
prefix=None,
delimiter=None,
conn_timeout=5,
response_timeout=15):
"""Base class for get direct account and container.
Do not use directly use the get_direct_account or
get_direct_container instead.
"""
qs = 'format=json'
if marker:
qs += '&marker=%s' % quote(marker)
if limit:
qs += '&limit=%d' % limit
if prefix:
qs += '&prefix=%s' % quote(prefix)
if delimiter:
qs += '&delimiter=%s' % quote(delimiter)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'GET',
path,
query_string=qs,
headers=gen_headers())
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
resp.read()
raise ClientException(
'%s server %s:%s direct GET %s gave stats %s' %
(stype, node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == HTTP_NO_CONTENT:
resp.read()
return resp_headers, []
return resp_headers, json.loads(resp.read()), resp.status
def direct_put_container(node,
part,
account,
container,
conn_timeout=5,
response_timeout=15,
headers=None):
"""
Request container put directly from the container server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:param headers: additional headers to include in the request
:returns: a dict containing the response's headers (all header names will
be lowercase)
"""
path = '/%s/%s' % (account, container)
if headers is None:
headers = {}
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'PUT',
path,
headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Container server %s:%s direct PUT %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def container_update(self, metadata, duplicate=False,
method='PUT', host=None, unknown=False):
'''
Send request to container service for info file update
:param metadata: object metadata
:param duplicate: If set to true, then container update request
will contain 'x-duplicate-update: True' header
else it not contain any additional header.
:param method: method name
:param host: host IP and port
:returns: True, If update successful
False, otherwise.
'''
if not metadata:
header = HeaderKeyDict({'x-timestamp': \
normalize_timestamp(time.time())})
file_system, directory = '', ''
else:
host, directory, file_system = self.__get_service_details(metadata, host=host)
try:
header = HeaderKeyDict({
'x-global-map-version': '-1',
'x-size': metadata['Content-Length'],
'x-content-type': metadata['Content-Type'],
'x-timestamp': metadata['X-Timestamp'],
'x-etag': metadata['ETag']})
except KeyError:
return False
header.update({'x-object-path': metadata['name']})
if duplicate:
header.update({'x-duplicate-update': True})
if unknown:
header.update({'x-duplicate-unknown': True})
try:
resp = http_connection(host, file_system, directory, \
header, method)
except (Exception, Timeout):
self._logger.exception(__(
'ERROR container update failed '
'%(host)s/%(fs)s '),
{'host': host,
'fs': file_system})
return False
resp.read()
if not is_success(resp.status):
return False
return True
def direct_head_object(node,
part,
account,
container,
obj,
conn_timeout=5,
response_timeout=15):
"""
Request object information directly from the object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a dict containing the response's headers (all header names will
be lowercase)
"""
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'HEAD',
path,
headers=gen_headers())
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct HEAD %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, resp.status
def direct_delete_object(node,
part,
account,
container,
obj,
conn_timeout=5,
response_timeout=15,
headers=None):
"""
Delete object directly from the object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: response from server
"""
if headers is None:
headers = {}
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'DELETE',
path,
headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct DELETE %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
return resp.status
def direct_delete_account_metadata(node,
part,
account,
headers,
conn_timeout=5,
response_timeout=15):
"""
Request account metadata delete directly from the account server.
:param node: node dictionary from the ring
:param part: partition the account is on
:param account: account name
:param headers: headers to delete metadata
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a dict containing the response's headers (all header names will
be lowercase)
"""
path = '/%s' % (account)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'POST',
path,
headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Account server %s:%s direct DELETE METADATA %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, resp.status
def direct_post_object(node,
part,
account,
container,
name,
headers,
conn_timeout=5,
response_timeout=15):
"""
Direct update to object metadata on object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param name: object name
:param headers: headers to store as metadata
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:raises ClientException: HTTP POST request failed
"""
path = '/%s/%s/%s' % (account, container, name)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'POST',
path,
headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct POST %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
return resp.status
def _need_to_refetch_manifest(self, req):
"""
Just because a response shows that an object is a SLO manifest does not
mean that response's body contains the entire SLO manifest. If it
doesn't, we need to make a second request to actually get the whole
thing.
Note: this assumes that X-Static-Large-Object has already been found.
"""
if req.method == 'HEAD':
return True
response_status = int(self._response_status[:3])
# These are based on etag, and the SLO's etag is almost certainly not
# the manifest object's etag. Still, it's highly likely that the
# submitted If-None-Match won't match the manifest object's etag, so
# we can avoid re-fetching the manifest if we got a successful
# response.
if ((req.if_match or req.if_none_match)
and not is_success(response_status)):
return True
if req.range and response_status in (206, 416):
content_range = ''
for header, value in self._response_headers:
if header.lower() == 'content-range':
content_range = value
break
# e.g. Content-Range: bytes 0-14289/14290
match = re.match('bytes (\d+)-(\d+)/(\d+)$', content_range)
if not match:
# Malformed or missing, so we don't know what we got.
return True
first_byte, last_byte, length = [int(x) for x in match.groups()]
# If and only if we actually got back the full manifest body, then
# we can avoid re-fetching the object.
got_everything = (first_byte == 0 and last_byte == length - 1)
return not got_everything
return False
def conn_for_obj(inner_dict, conn_time, node_timeout, logger, object_map):
"""
Make connection to object service for expired IDs
Returns: None
"""
try:
service_id = inner_dict['x-server-id']
full_path = inner_dict['x-object-path']
# get ip, port of object service
logger.debug("object_map: %s" % object_map)
data = {}
data['ip'] = object_map[service_id][0]
data['port'] = object_map[service_id][1]
# update dict for container service id
#TODO:(jaivish):just look whether this needs to passed as default parameter,
#else no need to change
cont_service_id = get_container_id()
inner_dict['x-server-id'] = cont_service_id
#TODO
# make connection
with ConnectionTimeout(conn_time):
logger.debug('Connection to object service started')
conn = http_connect(data['ip'], data['port'], '/', '/', \
'RECOVER', full_path, inner_dict)
logger.debug('Connection to object service completed: path:%s , dict: %s' % \
(full_path, inner_dict))
with Timeout(node_timeout):
response = conn.getresponse()
response.read()
if not is_success(response.status):
logger.error((
'ERROR connection to object service failed for '
'service-id: %(service_id)s'
'HTTP response status: %(status)d '),
{'service_id': service_id,
'status': response.status})
except (Exception, Timeout) as err:
logger.error('Exception occurred in sending'
'request to object service')
logger.exception(err)
def process_bulk_delete_transctions(obj_list, conn_time, node_timeout, obj_version, ll_port, logger):
"""
Method to call BULK_DELETE interface of object service
"""
try:
service_id = socket.gethostname() + '_' + str(ll_port) +'_object-server'
#obj_ring = ObjectRing(RING_DIR, logger)
#data = obj_ring.get_service_details(service_id)[0]
service_obj = get_service_obj('object', logger)
cont_service_id = get_container_id()
headers = {'x-server-id':cont_service_id, \
'X-GLOBAL-MAP-VERSION': obj_version}
with ConnectionTimeout(conn_time):
logger.debug('Connection to object service started')
# add the obj_list in the baody of request
conn = httplib.HTTPConnection(service_obj.get_ip(), \
service_obj.get_port())
if conn:
conn.request("BULK_DELETE", '', str(obj_list),headers)
else:
logger.debug("Can not create connection to object server ")
logger.debug('Connection to object service completed')
with Timeout(node_timeout):
response = conn.getresponse()
response.read()
if not is_success(response.status):
logger.error((
'ERROR connection to object service failed for '
'service-id: %(service_id)s'
'HTTP response status: %(status)d '),
{'service_id': service_id,
'status': response.status})
except (Exception, Timeout) as err:
logger.error('Exception occurred in sending'
'request to object service')
logger.exception(err)
def __iter__(self):
start_time = time.time()
have_yielded_data = False
if self.response and self.response.content_length:
bytes_left = int(self.response.content_length)
else:
bytes_left = None
try:
for seg_path, seg_etag, seg_size, first_byte, last_byte \
in self.listing_iter:
if time.time() - start_time > self.max_get_time:
raise SegmentError('ERROR: While processing manifest %s, '
'max LO GET time of %ds exceeded' %
(self.name, self.max_get_time))
# Make sure that the segment is a plain old object, not some
# flavor of large object, so that we can check its MD5.
path = seg_path + '?multipart-manifest=get'
seg_req = make_subrequest(
self.req.environ,
path=path,
method='GET',
headers={
'x-auth-token': self.req.headers.get('x-auth-token')
},
agent=('%(orig)s ' + self.ua_suffix),
swift_source=self.swift_source)
if first_byte is not None or last_byte is not None:
seg_req.headers['Range'] = "bytes=%s-%s" % (
# The 0 is to avoid having a range like "bytes=-10",
# which actually means the *last* 10 bytes.
'0' if first_byte is None else first_byte,
'' if last_byte is None else last_byte)
seg_resp = seg_req.get_response(self.app)
if not is_success(seg_resp.status_int):
close_if_possible(seg_resp.app_iter)
raise SegmentError(
'ERROR: While processing manifest %s, '
'got %d while retrieving %s' %
(self.name, seg_resp.status_int, seg_path))
elif ((seg_etag and (seg_resp.etag != seg_etag))
or (seg_size and (seg_resp.content_length != seg_size)
and not seg_req.range)):
# The content-length check is for security reasons. Seems
# possible that an attacker could upload a >1mb object and
# then replace it with a much smaller object with same
# etag. Then create a big nested SLO that calls that
# object many times which would hammer our obj servers. If
# this is a range request, don't check content-length
# because it won't match.
close_if_possible(seg_resp.app_iter)
raise SegmentError(
'Object segment no longer valid: '
'%(path)s etag: %(r_etag)s != %(s_etag)s or '
'%(r_size)s != %(s_size)s.' % {
'path': seg_req.path,
'r_etag': seg_resp.etag,
'r_size': seg_resp.content_length,
's_etag': seg_etag,
's_size': seg_size
})
seg_hash = hashlib.md5()
for chunk in seg_resp.app_iter:
seg_hash.update(chunk)
have_yielded_data = True
if bytes_left is None:
yield chunk
elif bytes_left >= len(chunk):
yield chunk
bytes_left -= len(chunk)
else:
yield chunk[:bytes_left]
bytes_left -= len(chunk)
close_if_possible(seg_resp.app_iter)
raise SegmentError(
'Too many bytes for %(name)s; truncating in '
'%(seg)s with %(left)d bytes left' % {
'name': self.name,
'seg': seg_req.path,
'left': bytes_left
})
close_if_possible(seg_resp.app_iter)
if seg_resp.etag and seg_hash.hexdigest() != seg_resp.etag \
and first_byte is None and last_byte is None:
raise SegmentError(
"Bad MD5 checksum in %(name)s for %(seg)s: headers had"
" %(etag)s, but object MD5 was actually %(actual)s" % {
'seg': seg_req.path,
'etag': seg_resp.etag,
'name': self.name,
'actual': seg_hash.hexdigest()
})
if bytes_left:
raise SegmentError(
'Not enough bytes for %s; closing connection' % self.name)
except ListingIterError as err:
# I have to save this error because yielding the ' ' below clears
# the exception from the current stack frame.
excinfo = sys.exc_info()
self.logger.exception('ERROR: While processing manifest %s, %s',
self.name, err)
# Normally, exceptions before any data has been yielded will
# cause Eventlet to send a 5xx response. In this particular
# case of ListingIterError we don't want that and we'd rather
# just send the normal 2xx response and then hang up early
# since 5xx codes are often used to judge Service Level
# Agreements and this ListingIterError indicates the user has
# created an invalid condition.
if not have_yielded_data:
yield ' '
raise excinfo
except SegmentError as err:
self.logger.exception(err)
# This doesn't actually change the response status (we're too
# late for that), but this does make it to the logs.
if self.response:
self.response.status = HTTP_SERVICE_UNAVAILABLE
raise
def send_accept_component_request(self, method, target_service_obj, \
comp_list):
"""
Creating http connection, sending ACCEPT_COMPONENT_TRANSFER
http request with component transfer list to target node
and sending intermediate response to GL.
:param method: ACCEPT_COMPONENT_TRANSFER
:param target_service_obj: target node object containing (ip, port, id)
:param comp_list: transfer component list, need to send to target node
"""
headers = {}
create_connection = False
conn_timeout = float(self.conf.get('conn_timeout', 10))
node_timeout = int(self.conf.get('node_timeout', 10))
self.logger.debug("Entering connect_target_node method")
filesystem = 'export'
directory = 'OSP_01'
path = '/recovery_process/'
#Parameters are fixed, could be changes according to requirement.
headers['Expect'] = '100-continue'
headers['X-Timestamp'] = time.time()
headers['Content-Type'] = 'text/plain'
try:
#header_content_key = (self.node['ip'], self.node['port'])
Content_length = len(str(comp_list))
headers['Content-Length'] = Content_length
self.logger.info("Header Sent:%s, ip:%s, port:%s, filesystem:%s,"
" directory:%s, path:%s method:%s" %(headers, \
target_service_obj.get_ip(), target_service_obj.get_port(), \
filesystem, directory, path, method))
#Creating http connection to target node
with ConnectionTimeout(conn_timeout):
conn = http_connect(
target_service_obj.get_ip(), target_service_obj.get_port(),\
filesystem, directory, method, path, headers)
with Timeout(node_timeout):
resp = conn.getexpect()
if resp.status == HTTP_CONTINUE:
conn.resp = None
self.logger.info("HTTP continue %s" % resp.status)
create_connection = True
elif is_success(resp.status):
conn.resp = resp
self.logger.info("Successfull status:%s" % resp.status)
create_connection = True
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.logger.error('ERROR Insufficient Storage' \
'ip:%s, port:%s' %(target_service_obj.get_ip(), \
target_service_obj.get_port()))
create_connection = False
self.check_transfer_component_map[
target_service_obj] = "Failed"
except (Exception, Timeout) as err:
self.logger.exception(
"Exception occured: %s during http connect id :%s, \
Expected: 100-continue" % (err, target_service_obj.get_id()))
create_connection = False
self.check_transfer_component_map[target_service_obj] = "Failed"
# sending component list to target node over http connection
if create_connection:
conn.reader = str(comp_list)
self.logger.info("Sending component List: %s" % \
conn.reader)
try:
with ChunkWriteTimeout(node_timeout):
conn.send(conn.reader)
conn.send_data = True
except (Exception, ChunkWriteTimeout) as err:
self.logger.error('Exception occured : %s at id : %s \
info: send file failed' %
(target_service_obj.get_id(), err))
conn.send_data = False
self.check_transfer_component_map[target_service_obj] = \
"Failed"
self.logger.info("Sending component list:%s completed ip:%s port:%s"
%(comp_list, target_service_obj.get_ip(), \
target_service_obj.get_port()))
def get_conn_response(conn, comp_list, node_timeout):
"""
Getting connection response
"""
try:
with Timeout(node_timeout):
if conn.resp:
self.logger.debug("conn.resp returned")
return conn.resp
else:
self.logger.debug("conn.getexpect()")
return conn.getexpect()
except (Exception, Timeout):
self.check_transfer_component_map[target_service_obj] = \
"Failed"
self.logger.exception('get_conn_response: Trying to get \
final status for id:%s' %
(target_service_obj.get_id()))
retry_count = 3
counter = 0
if conn.send_data:
response = get_conn_response(conn, comp_list, node_timeout)
if response and is_success(response.status):
# send intermediate status to GL
self.logger.info("Sending intermediate state to GL for \
target service : %s" % target_service_obj.get_id())
transferred_comp_list = []
for comp in comp_list:
transferred_comp_list.append(
(comp, target_service_obj))
while counter < retry_count:
counter += 1
gl_info_obj = self._request_handler.get_gl_info()
conn_obj = self._request_handler.connector(\
IoType.EVENTIO, gl_info_obj)
if conn_obj != None:
ret = self._request_handler.comp_transfer_info(\
self.service_id, transferred_comp_list, \
conn_obj)
if ret.status.get_status_code() == Resp.SUCCESS:
self.logger.info("Sent intermediate response " \
":%s to GL" %transferred_comp_list)
self.update_final_transfer_status(comp_list)
self.check_transfer_component_map[\
target_service_obj] = True
break
else:
self.logger.warning("Sending intermediate" \
"response to GL failed, Retrying")
conn_obj.close()
if ret.status.get_status_code() != Resp.SUCCESS:
self.check_transfer_component_map[target_service_obj] =\
"Failed"
else:
self.logger.error('get_response failed id:%s' \
%(target_service_obj.get_id()))
self.check_transfer_component_map[target_service_obj] = \
"Failed"
def _get_put_responses(self, conns, nodes, send_intermediate_resp):
post_quorum_timeout = float(self.conf.get('POST_QUORUM_TIMEOUT', 0.5))
global final_recovery_status_list
global final_status
statuses = []
reasons = []
bodies = []
self.logger.info("running get_put_response ..")
def get_conn_response(conn):
node_timeout = int(self.conf.get('NODE_TIMEOUT', 600))
try:
with Timeout(node_timeout):
if conn.resp:
self.logger.info("conn.resp returned")
return (conn.resp, conn)
else:
self.logger.info("conn.getresponse()")
return (conn.getresponse(), conn)
except (Exception, Timeout):
self.logger.exception(_("connection: %(conn)s \
get_put_response: %(status)s" ), \
{'conn': conn.node, \
'status': _('Trying to get final status ')})
#Get response on Async.
pile = GreenAsyncPile(len(conns))
#pile = GreenPile(len(conns))
for conn in conns:
self.logger.info("Spawning get_conn_response for:%s" % conn.node)
pile.spawn(get_conn_response, conn)
component_list_sent = []
for (response, conn) in pile:
#Return responses from here to Gl.
global list_to_gl
if response:
self.logger.info("response.status:%s" % type(response.status))
#Send Intermediate response to Global Leader
#for service_id, components in self.Map_from_gl.iteritems():
for components in \
self.service_component_map['%s:%s' % (conn.node['ip'], \
conn.node['port'])]:
#Need to change 200 in int or str depends.
if response.status == 200:
self.logger.debug("Component no: %s" % components)
obj_id = ''
for key in self.Map_from_gl.keys():
if (key.get_ip() == str(conn.node['ip'])) and (str(
key.get_port()) == str(conn.node['port'])):
obj_id = key
list_to_gl.append((components, obj_id))
#self.logger.info("Blocked before list to gl appended: %s" % obj_id)
eventlet.sleep()
#context-switch
else:
if (components, obj_id) in list_to_gl:
list_to_gl.remove((components, obj_id))
component_list_sent.append('%s:%s'%(conn.node['ip'], \
conn.node['port']))
eventlet.sleep(3)
#context-switch
self.logger.info("Component List: %s, list_to_gl: %s"% \
(list(set(component_list_sent)), list_to_gl))
statuses.append((response.status, \
"%s:%s"%(conn.node['ip'], conn.node['port'])))
reasons.append(response.reason)
bodies.append(response.read())
if response.status >= HTTP_INTERNAL_SERVER_ERROR:
self.logger.error("Error:%s returned from ip:%s port:%s" %\
(response.status, conn.node['ip'], \
conn.node['port']))
elif is_success(response.status):
self.logger.info("success status returned")
#pile.waitall(post_quorum_timeout)
if send_intermediate_resp:
self.logger.info("Filling Data in Queue")
try:
#list_of_tuple.put(list_to_gl, block=False)
list_of_tuple.put_nowait(list(set(list_to_gl)))
except Exception as err:
self.logger.info("Queue Full exception: %s" % err)
#list_of_tuple.put(list_to_gl)
self.logger.info("Queue Put successfull")
while len(statuses) < self.pile_size:
statuses.append(HTTP_SERVICE_UNAVAILABLE)
reasons.append('')
bodies.append('')
#send Final Response to GL.
global clean_journal_flag
for i in self.service_component_map:
for j in self.service_component_map[i]:
if i in component_list_sent:
final_recovery_status_list.append((j, True))
else:
final_recovery_status_list.append((j, False))
final_status = False
clean_journal_flag = False # Do not clean journal if anything failed
self.logger.info("final recovery status: %s" %
final_recovery_status_list)
self.logger.info("Returned status:%s, reason:%s, bodies:%s" % \
(statuses, reasons, bodies))
return statuses, reasons, bodies, component_list_sent
def _connect_put_node(self,
node,
headers,
logger_thread_locals,
method="ACCEPT_COMPONENT_CONT_DATA"):
"""Method for a connect"""
self.logger.thread_locals = logger_thread_locals
try:
client_timeout = int(self.conf.get('CLIENT_TIMEOUT', 500))
conn_timeout = float(self.conf.get('CONNECTION_TIMEOUT', 30))
node_timeout = int(self.conf.get('NODE_TIMEOUT', 600))
except Exception as err:
self.logger.info("error :%s" % err)
self.logger.info("Entering connect_put method")
#Some values are needed for connections.
filesystem = 'export'
directory = 'OSP'
path = '/recovery_process/'
#Parameters are fixed, could be changes according to requirement.
#Making headers
headers['Expect'] = '100-continue'
headers['X-Timestamp'] = time.time()
headers['Content-Type'] = 'text/plain'
headers['X-Object-Gl-Version-Id'] = self.__object_gl_version
try:
header_content_key = "%s:%s" % (node['ip'], node['port'])
#Content-length is assumed to be in integer.
Content_length = \
len(pickle.dumps(self.dictionary_new.get(header_content_key)))
headers['Node-ip'] = header_content_key
headers['Content-Length'] = Content_length
#headers['X-GLOBAL-MAP-VERSION'] = -1 (for now send actual map version)
#Receive map version from GL.(service_id,logger)
try:
global_map_version = get_global_map_version(
service_id_, logger_obj)
if global_map_version:
logger_obj.info("Map version received: %s" %
global_map_version)
else:
logger_obj.debug("Map version not received so exit")
sys.exit(130)
except Exception as err:
logger_obj.debug("Map version exception raised, Exit :%s" %
err)
sys.exit(130)
headers['X-GLOBAL-MAP-VERSION'] = global_map_version
headers['X-Recovery-Request'] = True
if self.service_component_map.has_key(header_content_key):
headers[
'X-COMPONENT-NUMBER-LIST'] = self.service_component_map[
header_content_key]
else:
headers['X-COMPONENT-NUMBER-LIST'] = []
self.logger.info("Header Sent: %s" % headers)
start_time = time.time()
with ConnectionTimeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], filesystem,
directory, method, path, headers)
with Timeout(node_timeout):
resp = conn.getexpect()
if resp.status == HTTP_CONTINUE:
conn.resp = None
conn.node = node
self.logger.info("HTTP continue %s" % resp.status)
return conn
elif is_success(resp.status):
conn.resp = resp
conn.node = node
self.logger.info("Successfull status:%s" % resp.status)
return conn
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.logger.error(
_('%(msg)s %(ip)s:%(port)s'), {
'msg': _('ERROR Insufficient Storage'),
'ip': node['ip'],
'port': node['port']
})
except (Exception, Timeout) as err:
self.logger.exception(
_('ERROR with %(type)s server %(ip)s:%(port)s/ re: '
'%(info)s'), {
'type': "Container",
'ip': node['ip'],
'port': node['port'],
'info': "Expect: 100-continue on "
})
def direct_get_object(node,
part,
account,
container,
obj,
conn_timeout=5,
response_timeout=15,
resp_chunk_size=None,
headers=None):
"""
Get object directly from the object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:param resp_chunk_size: if defined, chunk size of data to read.
:param headers: dict to be passed into HTTPConnection headers
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
"""
if headers is None:
headers = {}
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'GET',
path,
headers=gen_headers(headers))
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
resp.read()
raise ClientException(
'Object server %s:%s direct GET %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
if resp_chunk_size:
def _object_body():
buf = resp.read(resp_chunk_size)
while buf:
yield buf
buf = resp.read(resp_chunk_size)
object_body = _object_body()
else:
object_body = resp.read()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body, resp.status
def direct_put_object(node,
part,
account,
container,
name,
contents,
content_length=None,
etag=None,
content_type=None,
headers=None,
conn_timeout=5,
response_timeout=15,
chunk_size=65535):
"""
Put object directly from the object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param name: object name
:param contents: an iterable or string to read object data from
:param content_length: value to send as content-length header
:param etag: etag of contents
:param content_type: value to send as content-type header
:param headers: additional headers to include in the request
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:param chunk_size: if defined, chunk size of data to send.
:returns: etag from the server response
"""
path = '/%s/%s/%s' % (account, container, name)
if headers is None:
headers = {}
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
headers['Content-Length'] = str(content_length)
else:
for n, v in headers.iteritems():
if n.lower() == 'content-length':
content_length = int(v)
if content_type is not None:
headers['Content-Type'] = content_type
else:
headers['Content-Type'] = 'application/octet-stream'
if not contents:
headers['Content-Length'] = '0'
if isinstance(contents, basestring):
contents = [contents]
#Incase the caller want to insert an object with specific age
add_ts = 'X-Timestamp' not in headers
if content_length is None:
headers['Transfer-Encoding'] = 'chunked'
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'PUT',
path,
headers=gen_headers(headers, add_ts))
contents_f = FileLikeIter(contents)
if content_length is None:
chunk = contents_f.read(chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents_f.read(chunk_size)
conn.send('0\r\n\r\n')
else:
left = content_length
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents_f.read(size)
if not chunk:
break
conn.send(chunk)
left -= len(chunk)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct PUT %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'],
http_port=node['port'],
http_device=node['device'],
http_status=resp.status,
http_reason=resp.reason)
return resp.getheader('etag').strip('"'), resp.status
