def request_to_ks(self, req, servers, port): """ Routing multiple keystone servers. """ succ_resps = [] fail_resps = [] auth_tokens = self._split_auth_token(req, servers) bodies = self._split_body(req, servers) for site, token, body in zip(servers, auth_tokens, bodies): for node in site: parsed = urlparse(self._combinate_ks_url(node, port, req)) connector = HTTPSConnection if parsed.scheme == 'https' else HTTPConnection try: with ConnectionTimeout(self.conn_timeout): (host, port) = parsed.netloc.split(':') headers = req.headers if req.headers.has_key('Host'): headers['Host'] = host + ':' + str(port) if token: headers['X-Auth-Token'] = token if req.headers.has_key('Content-Length'): del headers['Content-Length'] conn = connector(host, port) conn.request(req.method, parsed.path, body, headers) with Timeout(self.timeout): resp = conn.getresponse() if resp.status >= 200 and resp.status <= 300: succ_resps.append(resp) break else: fail_resps.append(resp) except ValueError, err: fail_resps.append(HTTPPreconditionFailed(request=req)) except (Exception, TimeoutError), err: fail_resps.append(HTTPServiceUnavailable(request=req))
def object_update(self, node, part, op, obj, headers): """ Perform the object update to the container :param node: node dictionary from the container ring :param part: partition that holds the container :param op: operation performed (ex: 'POST' or 'DELETE') :param obj: object name being updated :param headers: headers to send with the update """ headers_out = headers.copy() headers_out['user-agent'] = 'obj-updater %s' % os.getpid() try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, op, obj, headers_out) with Timeout(self.node_timeout): resp = conn.getresponse() resp.read() return resp.status except (Exception, Timeout): self.logger.exception( _('ERROR with remote server ' '%(ip)s:%(port)s/%(device)s'), node) return HTTP_INTERNAL_SERVER_ERROR
def _repl_to_node(self, node, broker, partition, info): """ Replicate a database to a node. :param node: node dictionary from the ring to be replicated to :param broker: DB broker for the DB to be replication :param partition: partition on the node to replicate to :param info: DB info as a dictionary of {'max_row', 'hash', 'id', 'created_at', 'put_timestamp', 'delete_timestamp', 'metadata'} :returns: True if successful, False otherwise """ with ConnectionTimeout(self.conn_timeout): http = self._http_connect(node, partition, broker.db_file) if not http: self.logger.error( _('ERROR Unable to connect to remote server: %s'), node) return False sync_args = self._gather_sync_args(info) with Timeout(self.node_timeout): response = http.replicate('sync', *sync_args) if not response: return False return self._handle_sync_response(node, response, info, broker, http)
def _make_request(self, nodes, part, method, path, headers, query, logger_thread_locals): self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, method, path, headers=headers, query_string=query) conn.node = node with Timeout(self.app.node_timeout): resp = conn.getresponse() if not is_informational(resp.status) and \ not is_server_error(resp.status): return resp.status, resp.reason, resp.read() elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) except (Exception, Timeout): self.exception_occurred( node, self.server_type, _('Trying to %(method)s %(path)s') % { 'method': method, 'path': path })
def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy_index): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in :param policy_index: the associated storage policy index """ headers_out['user-agent'] = 'object-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)' ), { 'ip': ip, 'port': port, 'dev': contdevice }) data = { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out } timestamp = headers_out['x-timestamp'] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp, policy_index)
def _get_response(self, node, part, path, headers, policy): """ Helper method for reconstruction that GETs a single EC fragment archive :param node: the node to GET from :param part: the partition :param path: full path of the desired EC archive :param headers: the headers to send :param policy: an instance of :class:`~swift.common.storage_policy.BaseStoragePolicy` :returns: response """ resp = None try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, headers=headers) with Timeout(self.node_timeout): resp = conn.getresponse() if resp.status not in [HTTP_OK, HTTP_NOT_FOUND]: self.logger.warning( _("Invalid response %(resp)s from %(full_path)s"), {'resp': resp.status, 'full_path': self._full_path(node, part, path, policy)}) resp = None elif resp.status == HTTP_NOT_FOUND: resp = None except (Exception, Timeout): self.logger.exception( _("Trying to GET %(full_path)s"), { 'full_path': self._full_path(node, part, path, policy)}) return resp
def _connect_put_node(self, nodes, part, path, headers, logger_thread_locals): """Method for a file PUT connect""" self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers) with Timeout(self.app.node_timeout): resp = conn.getexpect() if resp.status == HTTP_CONTINUE: conn.resp = None conn.node = node return conn elif is_success(resp.status): conn.resp = resp conn.node = node return conn elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) except: self.exception_occurred(node, _('Object'), _('Expect: 100-continue on %s') % path)
def _get_response(self, node, part, path, headers, full_path): """ Helper method for reconstruction that GETs a single EC fragment archive :param node: the node to GET from :param part: the partition :param path: path of the desired EC archive relative to partition dir :param headers: the headers to send :param full_path: full path to desired EC archive :returns: response """ resp = None try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, headers=headers) with Timeout(self.node_timeout): resp = conn.getresponse() resp.full_path = full_path if resp.status not in [HTTP_OK, HTTP_NOT_FOUND]: self.logger.warning( _("Invalid response %(resp)s from %(full_path)s"), {'resp': resp.status, 'full_path': full_path}) resp = None elif resp.status == HTTP_NOT_FOUND: resp = None except (Exception, Timeout): self.logger.exception( _("Trying to GET %(full_path)s"), { 'full_path': full_path}) return resp
def _connect_put_node(self, nodes, part, path, headers, logger_thread_locals): """Method for a file PUT connect""" self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: start_time = time.time() with ConnectionTimeout(self.app.conn_timeout): conn = http_connect( node['ip'], node['port'], node['device'], part, 'PUT', path, headers) self.app.set_node_timing(node, time.time() - start_time) with Timeout(self.app.node_timeout): resp = conn.getexpect() if resp.status == HTTP_CONTINUE: conn.resp = None conn.node = node return conn elif is_success(resp.status): conn.resp = resp conn.node = node return conn elif headers['If-None-Match'] is not None and \ resp.status == HTTP_PRECONDITION_FAILED: conn.resp = resp conn.node = node return conn elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.app.error_limit(node, _('ERROR Insufficient Storage')) except (Exception, Timeout): self.app.exception_occurred( node, _('Object'), _('Expect: 100-continue on %s') % path)
def object_update(self, node, part, op, obj, headers_out): """ Perform the object update to the container :param node: node dictionary from the container ring :param part: partition that holds the container :param op: operation performed (ex: 'PUT' or 'DELETE') :param obj: object name being updated :param headers_out: headers to send with the update """ try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, op, obj, headers_out) with Timeout(self.node_timeout): resp = conn.getresponse() resp.read() success = is_success(resp.status) if not success: self.logger.debug( _('Error code %(status)d is returned from remote ' 'server %(ip)s: %(port)s / %(device)s'), { 'status': resp.status, 'ip': node['ip'], 'port': node['port'], 'device': node['device'] }) return (success, node['id']) except (Exception, Timeout): self.logger.exception( _('ERROR with remote server ' '%(ip)s:%(port)s/%(device)s'), node) return HTTP_INTERNAL_SERVER_ERROR, node['id']
def _connect_put_node(self, host, port, method, path, headers, query_string, ssl=False): try: with ConnectionTimeout(self.conn_timeout): conn = http_connect_raw(host, port, method, path, headers=headers, query_string=query_string, ssl=ssl) if headers.has_key('content-length') and int( headers['content-length']) == 0: return conn with Timeout(self.node_timeout): resp = conn.getexpect() if resp.status == 100: return conn elif resp.status == 507: self.logger.error('507 Insufficient Storage in %s:%s%s' % (host, port, path)) raise Exception except: self.logger.error('Expect: 100-continue on %s:%s%s' % (host, port, path)) return None
def account_update(self, req, account, container, broker): """ Update the account server with latest container info. :param req: webob.Request object :param account: account name :param container: container name :param borker: container DB broker object :returns: if the account request returns a 404 error code, HTTPNotFound response object, otherwise None. """ account_host = req.headers.get('X-Account-Host') account_partition = req.headers.get('X-Account-Partition') account_device = req.headers.get('X-Account-Device') if all([account_host, account_partition, account_device]): account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() account_headers = { 'x-put-timestamp': info['put_timestamp'], 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], 'x-trans-id': req.headers.get('x-trans-id', '-') } if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(account_ip, account_port, account_device, account_partition, 'PUT', new_path, account_headers) with Timeout(self.node_timeout): account_response = conn.getresponse() account_response.read() if account_response.status == 404: return HTTPNotFound(request=req) elif account_response.status < 200 or \ account_response.status > 299: self.logger.error( _('ERROR Account update failed ' 'with %(ip)s:%(port)s/%(device)s (will retry ' 'later): Response %(status)s %(reason)s'), { 'ip': account_ip, 'port': account_port, 'device': account_device, 'status': account_response.status, 'reason': account_response.reason }) except (Exception, Timeout): self.logger.exception( _('ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later)'), { 'ip': account_ip, 'port': account_port, 'device': account_device }) return None
def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy, logger_thread_locals=None): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'object-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed: %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s'), { 'ip': ip, 'port': port, 'dev': contdevice })
def _request_to_keystone(self, method, url, path, headers, body): """ """ parsed = urlparse(url) connector = HTTPSConnection if parsed.scheme == 'https' else HTTPConnection host, port = self._split_netloc(parsed) with ConnectionTimeout(300): conn = connector(host, port) conn.request(method, path, body, headers) with Timeout(300): return conn.getresponse()
def _make_request(self, nodes, part, method, path, headers, query, logger_thread_locals): """ Iterates over the given node iterator, sending an HTTP request to one node at a time. The first non-informational, non-server-error response is returned. If no non-informational, non-server-error response is received from any of the nodes, returns None. :param nodes: an iterator of the backend server and handoff servers :param part: the partition number :param method: the method to send to the backend :param path: the path to send to the backend (full path ends up being /<$device>/<$part>/<$path>) :param headers: a list of dicts, where each dict represents one backend request that should be made. :param query: query string to send to the backend. :param logger_thread_locals: The thread local values to be set on the self.app.logger to retain transaction logging information. :returns: a swob.Response object, or None if no responses were received """ self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: start_node_timing = time.time() with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, method, path, headers=headers, query_string=query) conn.node = node self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): resp = conn.getresponse() if not is_informational(resp.status) and \ not is_server_error(resp.status): return resp.status, resp.reason, resp.getheaders(), \ resp.read() elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.app.error_limit(node, _('ERROR Insufficient Storage')) except (Exception, Timeout): self.app.exception_occurred( node, self.server_type, _('Trying to %(method)s %(path)s') % { 'method': method, 'path': path })
def _repl_to_node(self, node, broker, partition, info): """ Replicate a database to a node. :param node: node dictionary from the ring to be replicated to :param broker: DB broker for the DB to be replication :param partition: partition on the node to replicate to :param info: DB info as a dictionary of {'max_row', 'hash', 'id', 'created_at', 'put_timestamp', 'delete_timestamp', 'metadata'} :returns: True if successful, False otherwise """ with ConnectionTimeout(self.conn_timeout): http = self._http_connect(node, partition, broker.db_file) if not http: self.logger.error( _('ERROR Unable to connect to remote server: %s'), node) return False with Timeout(self.node_timeout): response = http.replicate('sync', info['max_row'], info['hash'], info['id'], info['created_at'], info['put_timestamp'], info['delete_timestamp'], info['metadata']) if not response: return False elif response.status == HTTP_NOT_FOUND: # completely missing, rsync self.stats['rsync'] += 1 self.logger.increment('rsyncs') return self._rsync_db(broker, node, http, info['id']) elif response.status == HTTP_INSUFFICIENT_STORAGE: raise DriveNotMounted() elif response.status >= 200 and response.status < 300: rinfo = simplejson.loads(response.data) local_sync = broker.get_sync(rinfo['id'], incoming=False) if self._in_sync(rinfo, info, broker, local_sync): return True # if the difference in rowids between the two differs by # more than 50%, rsync then do a remote merge. if rinfo['max_row'] / float(info['max_row']) < 0.5: self.stats['remote_merge'] += 1 self.logger.increment('remote_merges') return self._rsync_db(broker, node, http, info['id'], replicate_method='rsync_then_merge', replicate_timeout=(info['count'] / 2000)) # else send diffs over to the remote server return self._usync_db(max(rinfo['point'], local_sync), broker, http, rinfo['id'], info['id'])
def sendData(self, metaList, data_type, server_ip, server_port): ip = server_ip port = server_port updatedData = json.dumps(metaList) headers = {'user-agent': data_type} with ConnectionTimeout(self.conn_timeout): try: conn = HTTPConnection('%s:%s' % (ip, port)) conn.request('PUT', '/', headers=headers, body=updatedData) resp = conn.getresponse() return resp except (Exception, Timeout): return HTTP_INTERNAL_SERVER_ERROR
def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error(_( 'ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, Timeout): self.logger.exception(_( 'ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)'), {'ip': ip, 'port': port, 'dev': contdevice}) async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) ohash = hash_path(account, container, obj) self.logger.increment('async_pendings') self.threadpools[objdevice].run_in_thread( write_pickle, {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out}, os.path.join(async_dir, ohash[-3:], ohash + '-' + normalize_timestamp(headers_out['x-timestamp'])), os.path.join(self.devices, objdevice, 'tmp'))
def _make_request(self, nodes, part, method, path, headers, query, logger_thread_locals): """ Sends an HTTP request to a single node and aggregates the result. It attempts the primary node, then iterates over the handoff nodes as needed. :param nodes: an iterator of the backend server and handoff servers :param part: the partition number :param method: the method to send to the backend :param path: the path to send to the backend :param headers: a list of dicts, where each dict represents one backend request that should be made. :param query: query string to send to the backend. :param logger_thread_locals: The thread local values to be set on the self.app.logger to retain transaction logging information. :returns: a swob.Response object """ self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: start_node_timing = time.time() with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, method, path, headers=headers, query_string=query) conn.node = node self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): resp = conn.getresponse() if not is_informational(resp.status) and \ not is_server_error(resp.status): return resp.status, resp.reason, resp.getheaders(), \ resp.read() elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node, _('ERROR Insufficient Storage')) except (Exception, Timeout): self.exception_occurred( node, self.server_type, _('Trying to %(method)s %(path)s') % { 'method': method, 'path': path })
def container_report(self, node, part, container, put_timestamp, delete_timestamp, count, bytes, storage_policy_index): """ Report container info to an account server. :param node: node dictionary from the account ring :param part: partition the account is on :param container: container name :param put_timestamp: put timestamp :param delete_timestamp: delete timestamp :param count: object count in the container :param bytes: bytes used in the container :param storage_policy_index: the policy index for the container """ with ConnectionTimeout(self.conn_timeout): try: headers = { 'X-Put-Timestamp': put_timestamp, 'X-Delete-Timestamp': delete_timestamp, 'X-Object-Count': count, 'X-Bytes-Used': bytes, 'X-Account-Override-Deleted': 'yes', 'X-Backend-Storage-Policy-Index': storage_policy_index, 'user-agent': self.user_agent } conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', container, headers=headers) except (Exception, Timeout): self.logger.exception( _('ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later): '), node) return HTTP_INTERNAL_SERVER_ERROR with Timeout(self.node_timeout): try: resp = conn.getresponse() resp.read() return resp.status except (Exception, Timeout): if self.logger.getEffectiveLevel() <= logging.DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) return HTTP_INTERNAL_SERVER_ERROR finally: conn.close()
def container_update(self, op, account, container, obj, headers_in, headers_out, objdevice): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param headers_in: dictionary of headers from the original request :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ host = headers_in.get('X-Container-Host', None) partition = headers_in.get('X-Container-Partition', None) contdevice = headers_in.get('X-Container-Device', None) if not all([host, partition, contdevice]): return full_path = '/%s/%s/%s' % (account, container, obj) try: with ConnectionTimeout(self.conn_timeout): ip, port = host.split(':') conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if 200 <= response.status < 300: return else: self.logger.error(_('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, TimeoutError): self.logger.exception(_('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)'), {'ip': ip, 'port': port, 'dev': contdevice}) async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) ohash = hash_path(account, container, obj) write_pickle( {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out}, os.path.join(async_dir, ohash[-3:], ohash + '-' + normalize_timestamp(headers_out['x-timestamp'])), os.path.join(self.devices, objdevice, 'tmp'))
def object_update(self, node, part, op, obj, headers_out): """ Perform the object update to the container :param node: node dictionary from the container ring :param part: partition that holds the container :param op: operation performed (ex: 'PUT' or 'DELETE') :param obj: object name being updated :param headers_out: headers to send with the update :return: a tuple of (``success``, ``node_id``, ``redirect``) where ``success`` is True if the update succeeded, ``node_id`` is the_id of the node updated and ``redirect`` is either None or a tuple of (a path, a timestamp string). """ redirect = None try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, op, obj, headers_out) with Timeout(self.node_timeout): resp = conn.getresponse() resp.read() if resp.status == HTTP_MOVED_PERMANENTLY: try: redirect = get_redirect_data(resp) except ValueError as err: self.logger.error( 'Container update failed for %r; problem with ' 'redirect location: %s' % (obj, err)) success = is_success(resp.status) if not success: self.logger.debug( _('Error code %(status)d is returned from remote ' 'server %(ip)s: %(port)s / %(device)s'), { 'status': resp.status, 'ip': node['ip'], 'port': node['port'], 'device': node['device'] }) return success, node['id'], redirect except (Exception, Timeout): self.logger.exception( _('ERROR with remote server ' '%(ip)s:%(port)s/%(device)s'), node) return HTTP_INTERNAL_SERVER_ERROR, node['id'], redirect
def _connect_put_node(self, nodes, part, path, headers, logger_thread_locals): """ Make a connection for a replicated object. Connects to the first working node that it finds in node_iter and sends over the request headers. Returns an HTTPConnection object to handle the rest of the streaming. """ self.app.logger.thread_locals = logger_thread_locals for node in nodes: print("===In _connect_put_node===", node) try: start_time = time.time() with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers) self.app.set_node_timing(node, time.time() - start_time) with Timeout(self.app.node_timeout): resp = conn.getexpect() if resp.status == HTTP_CONTINUE: conn.resp = None conn.node = node return conn elif is_success(resp.status) or resp.status == HTTP_CONFLICT: conn.resp = resp conn.node = node return conn elif headers['If-None-Match'] is not None and \ resp.status == HTTP_PRECONDITION_FAILED: conn.resp = resp conn.node = node return conn elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.app.error_limit(node, _('ERROR Insufficient Storage')) elif is_server_error(resp.status): self.app.error_occurred( node, _('ERROR %(status)d Expect: 100-continue ' 'From Object Server') % {'status': resp.status}) except (Exception, Timeout): self.app.exception_occurred( node, _('Object'), _('Expect: 100-continue on %s') % path)
def container_report(self, node, part, container, put_timestamp, delete_timestamp, count, bytes): """ Report container info to an account server. :param node: node dictionary from the account ring :param part: partition the account is on :param container: container name :param put_timestamp: put timestamp :param delete_timestamp: delete timestamp :param count: object count in the container :param bytes: bytes used in the container """ with ConnectionTimeout(self.conn_timeout): try: conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', container, headers={ 'X-Put-Timestamp': put_timestamp, 'X-Delete-Timestamp': delete_timestamp, 'X-Object-Count': count, 'X-Bytes-Used': bytes, 'X-Account-Override-Deleted': 'yes' }) except (Exception, TimeoutError): self.logger.exception( _('ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later): '), node) return 500 with Timeout(self.node_timeout): try: resp = conn.getresponse() resp.read() return resp.status except (Exception, TimeoutError): if self.logger.getEffectiveLevel() <= logging.DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) return 500
def _get_source_and_node(self): self.statuses = [] self.reasons = [] self.bodies = [] self.source_headers = [] sources = [] for node in self.app.iter_nodes(self.ring, self.partition): if node in self.used_nodes: continue start_node_timing = time.time() try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], self.partition, self.req_method, self.path, headers=self.backend_headers, query_string=self.req_query_string) self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.app.exception_occurred( node, self.server_type, _('Trying to %(method)s %(path)s') % { 'method': self.req_method, 'path': self.req_path }) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): self.statuses.append(HTTP_NOT_FOUND) self.reasons.append('') self.bodies.append('') self.source_headers.append('') close_swift_conn(possible_source) else: if self.used_source_etag: src_headers = dict( (k.lower(), v) for k, v in possible_source.getheaders()) if src_headers.get('etag', '').strip('"') != \ self.used_source_etag: self.statuses.append(HTTP_NOT_FOUND) self.reasons.append('') self.bodies.append('') self.source_headers.append('') continue self.statuses.append(possible_source.status) self.reasons.append(possible_source.reason) self.bodies.append('') self.source_headers.append('') sources.append((possible_source, node)) if not self.newest: # one good source is enough break else: self.statuses.append(possible_source.status) self.reasons.append(possible_source.reason) self.bodies.append(possible_source.read()) self.source_headers.append(possible_source.getheaders()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.app.error_limit(node, _('ERROR Insufficient Storage')) elif is_server_error(possible_source.status): self.app.error_occurred( node, _('ERROR %(status)d %(body)s ' 'From %(type)s Server') % { 'status': possible_source.status, 'body': self.bodies[-1][:1024], 'type': self.server_type }) if sources: sources.sort(key=lambda s: source_key(s[0])) source, node = sources.pop() for src, _junk in sources: close_swift_conn(src) self.used_nodes.append(node) src_headers = dict( (k.lower(), v) for k, v in possible_source.getheaders()) self.used_source_etag = src_headers.get('etag', '').strip('"') return source, node return None, None
def GETorHEAD_base(self, req, server_type, ring, partition, path): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type :param ring: the ring to obtain nodes from :param partition: partition :param path: path for the request :returns: swob.Response object """ statuses = [] reasons = [] bodies = [] sources = [] newest = config_true_value(req.headers.get('x-newest', 'f')) for node in self.iter_nodes(ring, partition): start_node_timing = time.time() try: with ConnectionTimeout(self.app.conn_timeout): headers = dict(req.headers) headers['Connection'] = 'close' conn = http_connect(node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, query_string=req.query_string) self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _('Trying to %(method)s %(path)s') % { 'method': req.method, 'path': req.path }) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(HTTP_NOT_FOUND) reasons.append('') bodies.append('') self.close_swift_conn(possible_source) else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append('') sources.append((possible_source, node)) if not newest: # one good source is enough break else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node, _('ERROR Insufficient Storage')) elif is_server_error(possible_source.status): self.error_occurred( node, _('ERROR %(status)d %(body)s ' 'From %(type)s Server') % { 'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type }) if sources: sources.sort(key=lambda s: source_key(s[0])) source, node = sources.pop() for src, _junk in sources: self.close_swift_conn(src) res = Response(request=req, conditional_response=True) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') res.accept_ranges = 'bytes' res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res return self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method))
def account_update(self, req, account, container, broker): """ Update the account server(s) with latest container info. :param req: swob.Request object :param account: account name :param container: container name :param broker: container DB broker object :returns: if all the account requests return a 404 error code, HTTPNotFound response object, if the account cannot be updated due to a malformed header, an HTTPBadRequest response object, otherwise None. """ account_hosts = [ h.strip() for h in req.headers.get('X-Account-Host', '').split(',') ] account_devices = [ d.strip() for d in req.headers.get('X-Account-Device', '').split(',') ] account_partition = req.headers.get('X-Account-Partition', '') if len(account_hosts) != len(account_devices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _('ERROR Account update failed: different ' 'numbers of hosts and devices in request: ' '"%(hosts)s" vs "%(devices)s"') % { 'hosts': req.headers.get('X-Account-Host', ''), 'devices': req.headers.get('X-Account-Device', '') }) return HTTPBadRequest(req=req) if account_partition: # zip is lazy on py3, but we need a list, so force evaluation. # On py2 it's an extra list copy, but the list is so small # (one element per replica in account ring, usually 3) that it # doesn't matter. updates = list(zip(account_hosts, account_devices)) else: updates = [] account_404s = 0 for account_host, account_device in updates: account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() account_headers = HeaderKeyDict({ 'x-put-timestamp': info['put_timestamp'], 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], 'x-trans-id': req.headers.get('x-trans-id', '-'), 'X-Backend-Storage-Policy-Index': info['storage_policy_index'], 'user-agent': 'container-server %s' % os.getpid(), 'referer': req.as_referer() }) if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(account_ip, account_port, account_device, account_partition, 'PUT', new_path, account_headers) with Timeout(self.node_timeout): account_response = conn.getresponse() account_response.read() if account_response.status == HTTP_NOT_FOUND: account_404s += 1 elif not is_success(account_response.status): self.logger.error( _('ERROR Account update failed ' 'with %(ip)s:%(port)s/%(device)s (will retry ' 'later): Response %(status)s %(reason)s'), { 'ip': account_ip, 'port': account_port, 'device': account_device, 'status': account_response.status, 'reason': account_response.reason }) except (Exception, Timeout): self.logger.exception( _('ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later)'), { 'ip': account_ip, 'port': account_port, 'device': account_device }) if updates and account_404s == len(updates): return HTTPNotFound(req=req) else: return None
def account_info(self, account, autocreate=False): """ Get account information, and also verify that the account exists. :param account: name of the account to get the info for :returns: tuple of (account partition, account nodes, container_count) or (None, None, None) if it does not exist """ partition, nodes = self.app.account_ring.get_nodes(account) # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses if self.app.memcache: cache_key = get_account_memcache_key(account) cache_value = self.app.memcache.get(cache_key) if not isinstance(cache_value, dict): result_code = cache_value container_count = 0 else: result_code = cache_value['status'] container_count = cache_value['container_count'] if result_code == HTTP_OK: return partition, nodes, container_count elif result_code == HTTP_NOT_FOUND and not autocreate: return None, None, None result_code = 0 container_count = 0 attempts_left = len(nodes) path = '/%s' % account headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} iternodes = self.iter_nodes(partition, nodes, self.app.account_ring) while attempts_left > 0: try: node = iternodes.next() except StopIteration: break attempts_left -= 1 try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], partition, 'HEAD', path, headers) with Timeout(self.app.node_timeout): resp = conn.getresponse() body = resp.read() if is_success(resp.status): result_code = HTTP_OK container_count = int( resp.getheader('x-account-container-count') or 0) break elif resp.status == HTTP_NOT_FOUND: if result_code == 0: result_code = HTTP_NOT_FOUND elif result_code != HTTP_NOT_FOUND: result_code = -1 elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) continue else: result_code = -1 except (Exception, Timeout): self.exception_occurred( node, _('Account'), _('Trying to get account info for %s') % path) if result_code == HTTP_NOT_FOUND and autocreate: if len(account) > MAX_ACCOUNT_NAME_LENGTH: return None, None, None headers = { 'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close' } resp = self.make_requests(Request.blank('/v1' + path), self.app.account_ring, partition, 'PUT', path, [headers] * len(nodes)) if not is_success(resp.status_int): self.app.logger.warning('Could not autocreate account %r' % \ path) return None, None, None result_code = HTTP_OK if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND): if result_code == HTTP_OK: cache_timeout = self.app.recheck_account_existence else: cache_timeout = self.app.recheck_account_existence * 0.1 self.app.memcache.set(cache_key, { 'status': result_code, 'container_count': container_count }, timeout=cache_timeout) if result_code == HTTP_OK: return partition, nodes, container_count return None, None, None
def container_info(self, account, container, account_autocreate=False): """ Get container information and thusly verify container existance. This will also make a call to account_info to verify that the account exists. :param account: account name for the container :param container: container name to look up :returns: tuple of (container partition, container nodes, container read acl, container write acl, container sync key) or (None, None, None, None, None) if the container does not exist """ partition, nodes = self.app.container_ring.get_nodes( account, container) path = '/%s/%s' % (account, container) if self.app.memcache: cache_key = get_container_memcache_key(account, container) cache_value = self.app.memcache.get(cache_key) if isinstance(cache_value, dict): status = cache_value['status'] read_acl = cache_value['read_acl'] write_acl = cache_value['write_acl'] sync_key = cache_value.get('sync_key') versions = cache_value.get('versions') if status == HTTP_OK: return partition, nodes, read_acl, write_acl, sync_key, \ versions elif status == HTTP_NOT_FOUND: return None, None, None, None, None, None if not self.account_info(account, autocreate=account_autocreate)[1]: return None, None, None, None, None, None result_code = 0 read_acl = None write_acl = None sync_key = None container_size = None versions = None attempts_left = len(nodes) headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} iternodes = self.iter_nodes(partition, nodes, self.app.container_ring) while attempts_left > 0: try: node = iternodes.next() except StopIteration: break attempts_left -= 1 try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], partition, 'HEAD', path, headers) with Timeout(self.app.node_timeout): resp = conn.getresponse() body = resp.read() if is_success(resp.status): result_code = HTTP_OK read_acl = resp.getheader('x-container-read') write_acl = resp.getheader('x-container-write') sync_key = resp.getheader('x-container-sync-key') container_size = \ resp.getheader('X-Container-Object-Count') versions = resp.getheader('x-versions-location') break elif resp.status == HTTP_NOT_FOUND: if result_code == 0: result_code = HTTP_NOT_FOUND elif result_code != HTTP_NOT_FOUND: result_code = -1 elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) continue else: result_code = -1 except (Exception, Timeout): self.exception_occurred( node, _('Container'), _('Trying to get container info for %s') % path) if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND): if result_code == HTTP_OK: cache_timeout = self.app.recheck_container_existence else: cache_timeout = self.app.recheck_container_existence * 0.1 self.app.memcache.set(cache_key, { 'status': result_code, 'read_acl': read_acl, 'write_acl': write_acl, 'sync_key': sync_key, 'container_size': container_size, 'versions': versions }, timeout=cache_timeout) if result_code == HTTP_OK: return partition, nodes, read_acl, write_acl, sync_key, versions return None, None, None, None, None, None
def GETorHEAD_base(self, req, server_type, partition, nodes, path, attempts): """ Base handler for HTTP GET or HEAD requests. :param req: webob.Request object :param server_type: server type :param partition: partition :param nodes: nodes :param path: path for the request :param attempts: number of attempts to try :returns: webob.Response object """ statuses = [] reasons = [] bodies = [] source = None sources = [] newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES nodes = iter(nodes) while len(statuses) < attempts: try: node = nodes.next() except StopIteration: break if self.error_limited(node): continue try: with ConnectionTimeout(self.app.conn_timeout): headers = dict(req.headers) headers['Connection'] = 'close' conn = http_connect(node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, query_string=req.query_string) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _('Trying to %(method)s %(path)s') % { 'method': req.method, 'path': req.path }) continue if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) continue if is_success(possible_source.status) or \ is_redirection(possible_source.status): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(HTTP_NOT_FOUND) reasons.append('') bodies.append('') possible_source.read() continue if newest: if sources: ts = float( source.getheader('x-put-timestamp') or source.getheader('x-timestamp') or 0) pts = float( possible_source.getheader('x-put-timestamp') or possible_source.getheader('x-timestamp') or 0) if pts > ts: sources.insert(0, possible_source) else: sources.append(possible_source) else: sources.insert(0, possible_source) source = sources[0] statuses.append(source.status) reasons.append(source.reason) bodies.append('') continue else: source = possible_source break statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) if is_server_error(possible_source.status): self.error_occurred(node, _('ERROR %(status)d %(body)s ' \ 'From %(type)s Server') % {'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type}) if source: if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): if newest: # we need to close all hanging swift_conns sources.pop(0) for src in sources: self.close_swift_conn(src) res = Response(request=req, conditional_response=True) res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn update_headers(res, source.getheaders()) # Used by container sync feature if res.environ is None: res.environ = dict() res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') update_headers(res, {'accept-ranges': 'bytes'}) res.status = source.status res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res elif is_success(source.status) or is_redirection(source.status): res = status_map[source.status](request=req) update_headers(res, source.getheaders()) # Used by container sync feature if res.environ is None: res.environ = dict() res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') update_headers(res, {'accept-ranges': 'bytes'}) res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res return self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method))