Beispiel #1
0
    def container_info(self, account, container, account_autocreate=False):
        """
        Get container information and thusly verify container existence.
        This will also make a call to account_info to verify that the
        account exists.

        :param account: account name for the container
        :param container: container name to look up
        :returns: dict containing at least container partition ('partition'),
                  container nodes ('containers'), container read
                  acl ('read_acl'), container write acl ('write_acl'),
                  and container sync key ('sync_key').
                  Values are set to None if the container does not exist.
        """
        part, nodes = self.app.container_ring.get_nodes(account, container)
        path = '/%s/%s' % (account, container)
        container_info = {'status': 0, 'read_acl': None,
                          'write_acl': None, 'sync_key': None,
                          'count': None, 'bytes': None,
                          'versions': None, 'partition': None,
                          'nodes': None}
        if self.app.memcache:
            cache_key = get_container_memcache_key(account, container)
            cache_value = self.app.memcache.get(cache_key)
            if isinstance(cache_value, dict):
                if 'container_size' in cache_value:
                    cache_value['count'] = cache_value['container_size']
                if is_success(cache_value['status']):
                    container_info.update(cache_value)
                    container_info['partition'] = part
                    container_info['nodes'] = nodes
                return container_info
        if not self.account_info(account, autocreate=account_autocreate)[1]:
            return container_info
        attempts_left = len(nodes)
        headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
        for node in self.iter_nodes(part, nodes, self.app.container_ring):
            try:
                start_node_timing = time.time()
                with ConnectionTimeout(self.app.conn_timeout):
                    conn = http_connect(node['ip'], node['port'],
                                        node['device'], part, 'HEAD',
                                        path, headers)
                self.app.set_node_timing(node, time.time() - start_node_timing)
                with Timeout(self.app.node_timeout):
                    resp = conn.getresponse()
                    resp.read()
                if is_success(resp.status):
                    container_info.update(
                        headers_to_container_info(resp.getheaders()))
                    break
                elif resp.status == HTTP_NOT_FOUND:
                    container_info['status'] = HTTP_NOT_FOUND
                else:
                    container_info['status'] = -1
                    if resp.status == HTTP_INSUFFICIENT_STORAGE:
                        self.error_limit(node)
            except (Exception, Timeout):
                self.exception_occurred(
                    node, _('Container'),
                    _('Trying to get container info for %s') % path)
            attempts_left -= 1
            if attempts_left <= 0:
                break
        if self.app.memcache:
            if container_info['status'] == HTTP_OK:
                self.app.memcache.set(
                    cache_key, container_info,
                    time=self.app.recheck_container_existence)
            elif container_info['status'] == HTTP_NOT_FOUND:
                self.app.memcache.set(
                    cache_key, container_info,
                    time=self.app.recheck_container_existence * 0.1)
        if container_info['status'] == HTTP_OK:
            container_info['partition'] = part
            container_info['nodes'] = nodes
        return container_info
Beispiel #2
0
    def GETorHEAD_base(self, req, server_type, partition, nodes, path,
                       attempts):
        """
        Base handler for HTTP GET or HEAD requests.

        :param req: swob.Request object
        :param server_type: server type
        :param partition: partition
        :param nodes: nodes
        :param path: path for the request
        :param attempts: number of attempts to try
        :returns: swob.Response object
        """
        statuses = []
        reasons = []
        bodies = []
        sources = []
        newest = config_true_value(req.headers.get('x-newest', 'f'))
        nodes = iter(nodes)
        while len(statuses) < attempts:
            try:
                node = nodes.next()
            except StopIteration:
                break
            if self.error_limited(node):
                continue
            start_node_timing = time.time()
            try:
                with ConnectionTimeout(self.app.conn_timeout):
                    headers = dict(req.headers)
                    headers['Connection'] = 'close'
                    conn = http_connect(
                        node['ip'], node['port'], node['device'], partition,
                        req.method, path, headers=headers,
                        query_string=req.query_string)
                self.app.set_node_timing(node, time.time() - start_node_timing)
                with Timeout(self.app.node_timeout):
                    possible_source = conn.getresponse()
                    # See NOTE: gate_conn at top of file about this.
                    possible_source.gate_conn = conn
            except (Exception, Timeout):
                self.exception_occurred(
                    node, server_type, _('Trying to %(method)s %(path)s') %
                    {'method': req.method, 'path': req.path})
                continue
            if self.is_good_source(possible_source):
                # 404 if we know we don't have a synced copy
                if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
                    statuses.append(HTTP_NOT_FOUND)
                    reasons.append('')
                    bodies.append('')
                    self.close_gate_conn(possible_source)
                else:
                    statuses.append(possible_source.status)
                    reasons.append(possible_source.reason)
                    bodies.append('')
                    sources.append(possible_source)
                    if not newest:  # one good source is enough
                        break
            else:
                statuses.append(possible_source.status)
                reasons.append(possible_source.reason)
                bodies.append(possible_source.read())
                if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
                    self.error_limit(node)
                elif is_server_error(possible_source.status):
                    self.error_occurred(node, _('ERROR %(status)d %(body)s '
                                                'From %(type)s Server') %
                                        {'status': possible_source.status,
                                         'body': bodies[-1][:1024],
                                         'type': server_type})
        if sources:
            sources.sort(key=source_key)
            source = sources.pop()
            for src in sources:
                self.close_gate_conn(src)
            res = Response(request=req, conditional_response=True)
            if req.method == 'GET' and \
                    source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
                res.app_iter = self._make_app_iter(node, source)
                # See NOTE: gate_conn at top of file about this.
                res.gate_conn = source.gate_conn
            res.status = source.status
            update_headers(res, source.getheaders())
            if not res.environ:
                res.environ = {}
            res.environ['gate_x_timestamp'] = \
                source.getheader('x-timestamp')
            res.accept_ranges = 'bytes'
            res.content_length = source.getheader('Content-Length')
            if source.getheader('Content-Type'):
                res.charset = None
                res.content_type = source.getheader('Content-Type')
            return res
        return self.best_response(req, statuses, reasons, bodies,
                                  '%s %s' % (server_type, req.method))
Beispiel #3
0
    def case_info(self, case, autocreate=False):
        """
        Get case information, and also verify that the account exists.

        :param case: name of the case to get the info for
        :returns: tuple of (case id, evidence_count)
                  or (None, None) if it does not exist
        """
        partition, nodes = self.app.account_ring.get_nodes(account)
        account_info = {'status': 0,
                        'container_count': 0,
                        'total_object_count': None,
                        'bytes': None,
                        'meta': {}}
        # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
        if self.app.memcache:
            cache_key = get_account_memcache_key(account)
            cache_value = self.app.memcache.get(cache_key)
            if not isinstance(cache_value, dict):
                result_code = cache_value
                container_count = 0
            else:
                result_code = cache_value['status']
                try:
                    container_count = int(cache_value['container_count'])
                except ValueError:
                    container_count = 0
            if result_code == HTTP_OK:
                return partition, nodes, container_count
            elif result_code == HTTP_NOT_FOUND and not autocreate:
                return None, None, None
        result_code = 0
        attempts_left = len(nodes)
        path = '/%s' % account
        headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
        iternodes = self.iter_nodes(partition, nodes, self.app.account_ring)
        while attempts_left > 0:
            try:
                node = iternodes.next()
            except StopIteration:
                break
            attempts_left -= 1
            try:
                start_node_timing = time.time()
                with ConnectionTimeout(self.app.conn_timeout):
                    conn = http_connect(node['ip'], node['port'],
                                        node['device'], partition, 'HEAD',
                                        path, headers)
                self.app.set_node_timing(node, time.time() - start_node_timing)
                with Timeout(self.app.node_timeout):
                    resp = conn.getresponse()
                    resp.read()
                    if is_success(resp.status):
                        result_code = HTTP_OK
                        account_info.update(
                            headers_to_account_info(resp.getheaders()))
                        break
                    elif resp.status == HTTP_NOT_FOUND:
                        if result_code == 0:
                            result_code = HTTP_NOT_FOUND
                        elif result_code != HTTP_NOT_FOUND:
                            result_code = -1
                    elif resp.status == HTTP_INSUFFICIENT_STORAGE:
                        self.error_limit(node)
                        continue
                    else:
                        result_code = -1
            except (Exception, Timeout):
                self.exception_occurred(node, _('Account'),
                                        _('Trying to get account info for %s')
                                        % path)
        if result_code == HTTP_NOT_FOUND and autocreate:
            if len(account) > MAX_ACCOUNT_NAME_LENGTH:
                return None, None, None
            headers = {'X-Timestamp': normalize_timestamp(time.time()),
                       'X-Trans-Id': self.trans_id,
                       'Connection': 'close'}
            resp = self.make_requests(Request.blank('/v1' + path),
                                      self.app.account_ring, partition, 'PUT',
                                      path, [headers] * len(nodes))
            if not is_success(resp.status_int):
                self.app.logger.warning('Could not autocreate account %r' %
                                        path)
                return None, None, None
            result_code = HTTP_OK
        if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND):
            if result_code == HTTP_OK:
                cache_timeout = self.app.recheck_account_existence
            else:
                cache_timeout = self.app.recheck_account_existence * 0.1
            account_info.update(status=result_code)
            self.app.memcache.set(cache_key,
                                  account_info,
                                  time=cache_timeout)
        if result_code == HTTP_OK:
            try:
                container_count = int(account_info['container_count'])
            except ValueError:
                container_count = 0
            return partition, nodes, container_count
        return None, None, None