def direct_delete_container(node, part, account, container, conn_timeout=5, response_timeout=15, headers=None): if headers is None: headers = {} path = '/%s/%s' % (account, container) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'DELETE', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Container server %s:%s direct DELETE %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) return resp.status
def _get_direct_account_container(path, stype, node, part, account, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15): """Base class for get direct account and container. Do not use directly use the get_direct_account or get_direct_container instead. """ qs = 'format=json' if marker: qs += '&marker=%s' % quote(marker) if limit: qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) if delimiter: qs += '&delimiter=%s' % quote(delimiter) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, query_string=qs, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise ClientException( '%s server %s:%s direct GET %s gave stats %s' % (stype, node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value if resp.status == HTTP_NO_CONTENT: resp.read() return resp_headers, [] return resp_headers, json.loads(resp.read()), resp.status
def _make_request_for_bulk_delete(self, nodes, filesystem, directory, \ method, path, headers, query, body_data, logger_thread_locals): self.app.logger.thread_locals = logger_thread_locals self.app.logger.debug("in _make_request_for_bulk_delete___") for node in nodes: try: start_node_timing = time.time() headers['Content-Length'] = len(body_data) with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], filesystem, directory, method, path, headers=headers, query_string=query) if conn: try: bytes_transferred = 0 data_size = len(body_data) while bytes_transferred < data_size: chunk = body_data[:REQ_BODY_CHUNK_SIZE] self.app.logger.debug("Sending chunk: %s" \ % chunk) conn.send(chunk) bytes_transferred += len(chunk) body_data = body_data[REQ_BODY_CHUNK_SIZE:] self.app.logger.debug("Total sent bytes: %s" \ % bytes_transferred) except Exception, ex: self.app.logger.error(\ "Error while sending body: %s" % ex) conn.node = node self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): resp = conn.getresponse() if not is_informational(resp.status) and \ not is_server_error(resp.status): return resp.status, resp.reason, resp.getheaders(), \ resp.read() elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.app.logger.error(_('%(msg)s %(ip)s:%(port)s'), {'msg': _('ERROR Insufficient Storage'), \ 'ip': node['ip'], 'port': node['port']}) except (Exception, Timeout): self.app.exception_occurred( node, self.server_type, _('Trying to %(method)s %(path)s') % { 'method': method, 'path': path })
def direct_put_container(node, part, account, container, conn_timeout=5, response_timeout=15, headers=None): """ Request container put directly from the container server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param headers: additional headers to include in the request :returns: a dict containing the response's headers (all header names will be lowercase) """ path = '/%s/%s' % (account, container) if headers is None: headers = {} with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Container server %s:%s direct PUT %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers
def direct_delete_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15, headers=None): """ Delete object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: response from server """ if headers is None: headers = {} path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'DELETE', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct DELETE %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) return resp.status
def direct_head_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15): """ Request object information directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a dict containing the response's headers (all header names will be lowercase) """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'HEAD', path, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct HEAD %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers, resp.status
def __connect_target_node(self, target_service, comp_list, \ logger_thread_locals, method): self.logger.thread_locals = logger_thread_locals #client_timeout = int(self.conf.get('CLIENT_TIMEOUT', 500)) conn_timeout = float(self.conf.get('CONNECTION_TIMEOUT', 30)) node_timeout = int(self.conf.get('NODE_TIMEOUT', 600)) #Some values are needed for connections. filesystem = 'export' directory = 'OSP_01' path = '/recovery_process/' try: # create headers headers = dict() headers['Expect'] = '100-continue' headers['X-Timestamp'] = time.time() headers['Content-Type'] = 'text/plain' headers['X-GLOBAL-MAP-VERSION'] = self._latest_gl_version headers['Content-Length'] = len(pickle.dumps(comp_list)) with ConnectionTimeout(conn_timeout): conn = http_connect(target_service.get_ip(), target_service.get_port(), filesystem, directory, method, path, headers) with Timeout(node_timeout): resp = conn.getexpect() if resp.status == HTTP_CONTINUE: conn.resp = None conn.target_service = target_service self.logger.info("HTTP continue %s" % resp.status) return conn else: self.logger.error("Http connection status: %s" % resp.status) return None except (Exception, Timeout) as err: self.logger.exception( _('ERROR with %(type)s server %(ip)s:%(port)s/ re: ' '%(info)s'), { 'type': "Account", 'ip': target_service.get_ip(), 'port': target_service.get_port(), #check 'info': "Expect: 100-continue on " })
def direct_post_object(node, part, account, container, name, headers, conn_timeout=5, response_timeout=15): """ Direct update to object metadata on object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param headers: headers to store as metadata :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :raises ClientException: HTTP POST request failed """ path = '/%s/%s/%s' % (account, container, name) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'POST', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct POST %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) return resp.status
def direct_delete_account_metadata(node, part, account, headers, conn_timeout=5, response_timeout=15): """ Request account metadata delete directly from the account server. :param node: node dictionary from the ring :param part: partition the account is on :param account: account name :param headers: headers to delete metadata :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a dict containing the response's headers (all header names will be lowercase) """ path = '/%s' % (account) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'POST', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Account server %s:%s direct DELETE METADATA %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers, resp.status
def http_connection(host, file_system, directory, headers, method): """ Method for sending http request to container param host: Host information of container param file_system: File system name in which object resides param directory: Directory name in which object resides param headers: Request header dictionary param method: Method name to be called return response: HTTP response """ try: with ConnectionTimeout(CONN_TIMEOUT): ip_add, port = host.rsplit(':', 1) full_path = headers['x-object-path'] conn = http_connect(ip_add, port, file_system, \ directory, method, full_path, headers) with Timeout(NODE_TIMEOUT): response = conn.getresponse() return response except (Exception, Timeout): raise
def conn_for_obj(inner_dict, conn_time, node_timeout, logger, object_map): """ Make connection to object service for expired IDs Returns: None """ try: service_id = inner_dict['x-server-id'] full_path = inner_dict['x-object-path'] # get ip, port of object service logger.debug("object_map: %s" % object_map) data = {} data['ip'] = object_map[service_id][0] data['port'] = object_map[service_id][1] # update dict for container service id #TODO:(jaivish):just look whether this needs to passed as default parameter, #else no need to change cont_service_id = get_container_id() inner_dict['x-server-id'] = cont_service_id #TODO # make connection with ConnectionTimeout(conn_time): logger.debug('Connection to object service started') conn = http_connect(data['ip'], data['port'], '/', '/', \ 'RECOVER', full_path, inner_dict) logger.debug('Connection to object service completed: path:%s , dict: %s' % \ (full_path, inner_dict)) with Timeout(node_timeout): response = conn.getresponse() response.read() if not is_success(response.status): logger.error(( 'ERROR connection to object service failed for ' 'service-id: %(service_id)s' 'HTTP response status: %(status)d '), {'service_id': service_id, 'status': response.status}) except (Exception, Timeout) as err: logger.error('Exception occurred in sending' 'request to object service') logger.exception(err)
def send_accept_component_request(self, method, target_service_obj, \ comp_list): """ Creating http connection, sending ACCEPT_COMPONENT_TRANSFER http request with component transfer list to target node and sending intermediate response to GL. :param method: ACCEPT_COMPONENT_TRANSFER :param target_service_obj: target node object containing (ip, port, id) :param comp_list: transfer component list, need to send to target node """ headers = {} create_connection = False conn_timeout = float(self.conf.get('conn_timeout', 10)) node_timeout = int(self.conf.get('node_timeout', 10)) self.logger.debug("Entering connect_target_node method") filesystem = 'export' directory = 'OSP_01' path = '/recovery_process/' #Parameters are fixed, could be changes according to requirement. headers['Expect'] = '100-continue' headers['X-Timestamp'] = time.time() headers['Content-Type'] = 'text/plain' try: #header_content_key = (self.node['ip'], self.node['port']) Content_length = len(str(comp_list)) headers['Content-Length'] = Content_length self.logger.info("Header Sent:%s, ip:%s, port:%s, filesystem:%s," " directory:%s, path:%s method:%s" %(headers, \ target_service_obj.get_ip(), target_service_obj.get_port(), \ filesystem, directory, path, method)) #Creating http connection to target node with ConnectionTimeout(conn_timeout): conn = http_connect( target_service_obj.get_ip(), target_service_obj.get_port(),\ filesystem, directory, method, path, headers) with Timeout(node_timeout): resp = conn.getexpect() if resp.status == HTTP_CONTINUE: conn.resp = None self.logger.info("HTTP continue %s" % resp.status) create_connection = True elif is_success(resp.status): conn.resp = resp self.logger.info("Successfull status:%s" % resp.status) create_connection = True elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error('ERROR Insufficient Storage' \ 'ip:%s, port:%s' %(target_service_obj.get_ip(), \ target_service_obj.get_port())) create_connection = False self.check_transfer_component_map[ target_service_obj] = "Failed" except (Exception, Timeout) as err: self.logger.exception( "Exception occured: %s during http connect id :%s, \ Expected: 100-continue" % (err, target_service_obj.get_id())) create_connection = False self.check_transfer_component_map[target_service_obj] = "Failed" # sending component list to target node over http connection if create_connection: conn.reader = str(comp_list) self.logger.info("Sending component List: %s" % \ conn.reader) try: with ChunkWriteTimeout(node_timeout): conn.send(conn.reader) conn.send_data = True except (Exception, ChunkWriteTimeout) as err: self.logger.error('Exception occured : %s at id : %s \ info: send file failed' % (target_service_obj.get_id(), err)) conn.send_data = False self.check_transfer_component_map[target_service_obj] = \ "Failed" self.logger.info("Sending component list:%s completed ip:%s port:%s" %(comp_list, target_service_obj.get_ip(), \ target_service_obj.get_port())) def get_conn_response(conn, comp_list, node_timeout): """ Getting connection response """ try: with Timeout(node_timeout): if conn.resp: self.logger.debug("conn.resp returned") return conn.resp else: self.logger.debug("conn.getexpect()") return conn.getexpect() except (Exception, Timeout): self.check_transfer_component_map[target_service_obj] = \ "Failed" self.logger.exception('get_conn_response: Trying to get \ final status for id:%s' % (target_service_obj.get_id())) retry_count = 3 counter = 0 if conn.send_data: response = get_conn_response(conn, comp_list, node_timeout) if response and is_success(response.status): # send intermediate status to GL self.logger.info("Sending intermediate state to GL for \ target service : %s" % target_service_obj.get_id()) transferred_comp_list = [] for comp in comp_list: transferred_comp_list.append( (comp, target_service_obj)) while counter < retry_count: counter += 1 gl_info_obj = self._request_handler.get_gl_info() conn_obj = self._request_handler.connector(\ IoType.EVENTIO, gl_info_obj) if conn_obj != None: ret = self._request_handler.comp_transfer_info(\ self.service_id, transferred_comp_list, \ conn_obj) if ret.status.get_status_code() == Resp.SUCCESS: self.logger.info("Sent intermediate response " \ ":%s to GL" %transferred_comp_list) self.update_final_transfer_status(comp_list) self.check_transfer_component_map[\ target_service_obj] = True break else: self.logger.warning("Sending intermediate" \ "response to GL failed, Retrying") conn_obj.close() if ret.status.get_status_code() != Resp.SUCCESS: self.check_transfer_component_map[target_service_obj] =\ "Failed" else: self.logger.error('get_response failed id:%s' \ %(target_service_obj.get_id())) self.check_transfer_component_map[target_service_obj] = \ "Failed"
def _connect_put_node(self, node, headers, logger_thread_locals, method="ACCEPT_COMPONENT_CONT_DATA"): """Method for a connect""" self.logger.thread_locals = logger_thread_locals try: client_timeout = int(self.conf.get('CLIENT_TIMEOUT', 500)) conn_timeout = float(self.conf.get('CONNECTION_TIMEOUT', 30)) node_timeout = int(self.conf.get('NODE_TIMEOUT', 600)) except Exception as err: self.logger.info("error :%s" % err) self.logger.info("Entering connect_put method") #Some values are needed for connections. filesystem = 'export' directory = 'OSP' path = '/recovery_process/' #Parameters are fixed, could be changes according to requirement. #Making headers headers['Expect'] = '100-continue' headers['X-Timestamp'] = time.time() headers['Content-Type'] = 'text/plain' headers['X-Object-Gl-Version-Id'] = self.__object_gl_version try: header_content_key = "%s:%s" % (node['ip'], node['port']) #Content-length is assumed to be in integer. Content_length = \ len(pickle.dumps(self.dictionary_new.get(header_content_key))) headers['Node-ip'] = header_content_key headers['Content-Length'] = Content_length #headers['X-GLOBAL-MAP-VERSION'] = -1 (for now send actual map version) #Receive map version from GL.(service_id,logger) try: global_map_version = get_global_map_version( service_id_, logger_obj) if global_map_version: logger_obj.info("Map version received: %s" % global_map_version) else: logger_obj.debug("Map version not received so exit") sys.exit(130) except Exception as err: logger_obj.debug("Map version exception raised, Exit :%s" % err) sys.exit(130) headers['X-GLOBAL-MAP-VERSION'] = global_map_version headers['X-Recovery-Request'] = True if self.service_component_map.has_key(header_content_key): headers[ 'X-COMPONENT-NUMBER-LIST'] = self.service_component_map[ header_content_key] else: headers['X-COMPONENT-NUMBER-LIST'] = [] self.logger.info("Header Sent: %s" % headers) start_time = time.time() with ConnectionTimeout(conn_timeout): conn = http_connect(node['ip'], node['port'], filesystem, directory, method, path, headers) with Timeout(node_timeout): resp = conn.getexpect() if resp.status == HTTP_CONTINUE: conn.resp = None conn.node = node self.logger.info("HTTP continue %s" % resp.status) return conn elif is_success(resp.status): conn.resp = resp conn.node = node self.logger.info("Successfull status:%s" % resp.status) return conn elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error( _('%(msg)s %(ip)s:%(port)s'), { 'msg': _('ERROR Insufficient Storage'), 'ip': node['ip'], 'port': node['port'] }) except (Exception, Timeout) as err: self.logger.exception( _('ERROR with %(type)s server %(ip)s:%(port)s/ re: ' '%(info)s'), { 'type': "Container", 'ip': node['ip'], 'port': node['port'], 'info': "Expect: 100-continue on " })
def direct_put_object(node, part, account, container, name, contents, content_length=None, etag=None, content_type=None, headers=None, conn_timeout=5, response_timeout=15, chunk_size=65535): """ Put object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param contents: an iterable or string to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param content_type: value to send as content-type header :param headers: additional headers to include in the request :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param chunk_size: if defined, chunk size of data to send. :returns: etag from the server response """ path = '/%s/%s/%s' % (account, container, name) if headers is None: headers = {} if etag: headers['ETag'] = etag.strip('"') if content_length is not None: headers['Content-Length'] = str(content_length) else: for n, v in headers.iteritems(): if n.lower() == 'content-length': content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type else: headers['Content-Type'] = 'application/octet-stream' if not contents: headers['Content-Length'] = '0' if isinstance(contents, basestring): contents = [contents] #Incase the caller want to insert an object with specific age add_ts = 'X-Timestamp' not in headers if content_length is None: headers['Transfer-Encoding'] = 'chunked' with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers=gen_headers(headers, add_ts)) contents_f = FileLikeIter(contents) if content_length is None: chunk = contents_f.read(chunk_size) while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents_f.read(chunk_size) conn.send('0\r\n\r\n') else: left = content_length while left > 0: size = chunk_size if size > left: size = left chunk = contents_f.read(size) if not chunk: break conn.send(chunk) left -= len(chunk) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct PUT %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) return resp.getheader('etag').strip('"'), resp.status
def direct_get_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15, resp_chunk_size=None, headers=None): """ Get object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param resp_chunk_size: if defined, chunk size of data to read. :param headers: dict to be passed into HTTPConnection headers :returns: a tuple of (response headers, the object's contents) The response headers will be a dict and all header names will be lowercase. """ if headers is None: headers = {} path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, headers=gen_headers(headers)) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise ClientException( 'Object server %s:%s direct GET %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) if resp_chunk_size: def _object_body(): buf = resp.read(resp_chunk_size) while buf: yield buf buf = resp.read(resp_chunk_size) object_body = _object_body() else: object_body = resp.read() resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers, object_body, resp.status