示例#1
0
文件: recon.py 项目: jgmerritt/swift
    def get_diskusage(self):
        """get disk utilization statistics"""
        devices = []
        for entry in os.listdir(self.devices):
            if not os.path.isdir(os.path.join(self.devices, entry)):
                continue

            try:
                check_mount(self.devices, entry)
            except OSError as err:
                devices.append({'device': entry, 'mounted': str(err),
                                'size': '', 'used': '', 'avail': ''})
            except ValueError:
                devices.append({'device': entry, 'mounted': False,
                                'size': '', 'used': '', 'avail': ''})
            else:
                path = os.path.join(self.devices, entry)
                disk = os.statvfs(path)
                capacity = disk.f_bsize * disk.f_blocks
                available = disk.f_bsize * disk.f_bavail
                used = disk.f_bsize * (disk.f_blocks - disk.f_bavail)
                devices.append({'device': entry, 'mounted': True,
                                'size': capacity, 'used': used,
                                'avail': available})
        return devices
示例#2
0
    def test_check_drive_isdir(self):
        root = '/srv'
        path = 'sdb2'
        with mock_check_drive(isdir=True) as mocks:
            self.assertEqual('/srv/sdb2', constraints.check_dir(root, path))
            self.assertEqual('/srv/sdb2', constraints.check_drive(
                root, path, False))
            self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')],
                             mocks['isdir'].call_args_list)
            self.assertEqual([], mocks['ismount'].call_args_list)

        with mock_check_drive(isdir=True) as mocks:
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_mount(root, path)
            self.assertEqual(str(exc_mgr.exception),
                             '/srv/sdb2 is not mounted')

            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_drive(root, path, True)
            self.assertEqual(str(exc_mgr.exception),
                             '/srv/sdb2 is not mounted')

            self.assertEqual([], mocks['isdir'].call_args_list)
            self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')],
                             mocks['ismount'].call_args_list)
示例#3
0
    def test_check_drive_invalid_path(self):
        root = '/srv/'
        with mock_check_drive() as mocks:
            drive = 'foo?bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_dir(root, drive)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)

            drive = 'foo bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_mount(root, drive)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)

            drive = 'foo/bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_drive(root, drive, True)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)

            drive = 'foo%bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_drive(root, drive, False)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)
        self.assertEqual([], mocks['isdir'].call_args_list)
        self.assertEqual([], mocks['ismount'].call_args_list)
示例#4
0
    def test_check_drive_invalid_path(self):
        root = '/srv/'
        with mock_check_drive() as mocks:
            drive = 'foo?bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_dir(root, drive)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)

            drive = 'foo bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_mount(root, drive)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)

            drive = 'foo/bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_drive(root, drive, True)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)

            drive = 'foo%bar'
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_drive(root, drive, False)
            self.assertEqual(str(exc_mgr.exception),
                             '%s is not a valid drive name' % drive)
        self.assertEqual([], mocks['isdir'].call_args_list)
        self.assertEqual([], mocks['ismount'].call_args_list)
示例#5
0
    def get_diskusage(self):
        """get disk utilization statistics"""
        devices = []
        for entry in os.listdir(self.devices):
            if not os.path.isdir(os.path.join(self.devices, entry)):
                continue

            try:
                check_mount(self.devices, entry)
            except OSError as err:
                devices.append({'device': entry, 'mounted': str(err),
                                'size': '', 'used': '', 'avail': ''})
            except ValueError:
                devices.append({'device': entry, 'mounted': False,
                                'size': '', 'used': '', 'avail': ''})
            else:
                path = os.path.join(self.devices, entry)
                disk = os.statvfs(path)
                capacity = disk.f_bsize * disk.f_blocks
                available = disk.f_bsize * disk.f_bavail
                used = disk.f_bsize * (disk.f_blocks - disk.f_bavail)
                devices.append({'device': entry, 'mounted': True,
                                'size': capacity, 'used': used,
                                'avail': available})
        return devices
示例#6
0
    def test_check_drive_isdir(self):
        root = '/srv'
        path = 'sdb2'
        with mock_check_drive(isdir=True) as mocks:
            self.assertEqual('/srv/sdb2', constraints.check_dir(root, path))
            self.assertEqual('/srv/sdb2',
                             constraints.check_drive(root, path, False))
            self.assertEqual([mock.call('/srv/sdb2'),
                              mock.call('/srv/sdb2')],
                             mocks['isdir'].call_args_list)
            self.assertEqual([], mocks['ismount'].call_args_list)

        with mock_check_drive(isdir=True) as mocks:
            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_mount(root, path)
            self.assertEqual(str(exc_mgr.exception),
                             '/srv/sdb2 is not mounted')

            with self.assertRaises(ValueError) as exc_mgr:
                constraints.check_drive(root, path, True)
            self.assertEqual(str(exc_mgr.exception),
                             '/srv/sdb2 is not mounted')

            self.assertEqual([], mocks['isdir'].call_args_list)
            self.assertEqual([mock.call('/srv/sdb2'),
                              mock.call('/srv/sdb2')],
                             mocks['ismount'].call_args_list)
示例#7
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     req_timestamp = valid_timestamp(req)
     if 'x-container-sync-to' in req.headers:
         err, sync_to, realm, realm_key = validate_sync_to(
             req.headers['x-container-sync-to'], self.allowed_sync_hosts,
             self.realms_conf)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     metadata = {}
     metadata.update(
         (key, (value, req_timestamp.internal))
         for key, value in req.headers.iteritems()
         if key.lower() in self.save_headers or
         is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata, validate_metadata=True)
     return HTTPNoContent(request=req)
示例#8
0
 def get_diskusage(self):
     """get disk utilization statistics"""
     devices = []
     for entry in os.listdir(self.devices):
         if check_mount(self.devices, entry):
             path = os.path.join(self.devices, entry)
             disk = os.statvfs(path)
             capacity = disk.f_bsize * disk.f_blocks
             available = disk.f_bsize * disk.f_bavail
             used = disk.f_bsize * (disk.f_blocks - disk.f_bavail)
             devices.append({
                 'device': entry,
                 'mounted': True,
                 'size': capacity,
                 'used': used,
                 'avail': available
             })
         else:
             devices.append({
                 'device': entry,
                 'mounted': False,
                 'size': '',
                 'used': '',
                 'avail': ''
             })
     return devices
示例#9
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive,
                                         part,
                                         account,
                                         container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     info, is_deleted = broker.get_info_is_deleted()
     headers = gen_resp_headers(info, is_deleted=is_deleted)
     if is_deleted:
         return HTTPNotFound(request=req, headers=headers)
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.items()
         if value != '' and (key.lower() in self.save_headers
                             or is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     resp = HTTPNoContent(request=req, headers=headers, charset='utf-8')
     resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp']))
     return resp
示例#10
0
 def GET(self, req):
     """Handle HTTP GET request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     path = get_param(req, 'path')
     prefix = get_param(req, 'prefix')
     delimiter = get_param(req, 'delimiter')
     if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
         # delimiters can be made more flexible later
         return HTTPPreconditionFailed(body='Bad delimiter')
     marker = get_param(req, 'marker', '')
     end_marker = get_param(req, 'end_marker')
     limit = constraints.CONTAINER_LISTING_LIMIT
     given_limit = get_param(req, 'limit')
     if given_limit and given_limit.isdigit():
         limit = int(given_limit)
         if limit > constraints.CONTAINER_LISTING_LIMIT:
             return HTTPPreconditionFailed(
                 request=req,
                 body='Maximum limit is %d'
                 % constraints.CONTAINER_LISTING_LIMIT)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     info = broker.get_info()
     container_list = broker.list_objects_iter(limit, marker, end_marker,
                                               prefix, delimiter, path)
     return self.create_listing(req, out_content_type, info,
                                broker.metadata, container_list, container)
示例#11
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing or bad timestamp',
                               request=req, content_type='text/plain')
     if 'x-container-sync-to' in req.headers:
         err = validate_sync_to(req.headers['x-container-sync-to'],
                                self.allowed_sync_hosts)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     metadata = {}
     metadata.update(
         (key, (value, timestamp)) for key, value in req.headers.iteritems()
         if key.lower() in self.save_headers or
         key.lower().startswith('x-container-meta-'))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata)
     return HTTPNoContent(request=req)
示例#12
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive,
                                         part,
                                         account,
                                         container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     info = broker.get_info()
     headers = {
         'X-Container-Object-Count': info['object_count'],
         'X-Container-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp'],
     }
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.iteritems()
         if value != '' and (key.lower() in self.save_headers
                             or is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset='utf-8')
示例#13
0
class AccountController(object):
    """WSGI controller for the account server."""

    def __init__(self, conf):
        self.logger = get_logger(conf, log_route='account-server')
        self.root = conf.get('devices', '/srv/node')
        self.mount_check = conf.get('mount_check', 'true').lower() in \
                              ('true', 't', '1', 'on', 'yes', 'y')
        self.replicator_rpc = \
            ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check)

    def _get_account_broker(self, drive, part, account):
        hsh = hash_path(account)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        return AccountBroker(db_path, account=account, logger=self.logger)

    def DELETE(self, req):
        """Handle HTTP DELETE request."""
        try:
            drive, part, account = split_path(unquote(req.path), 3)
        except ValueError, err:
            return HTTPBadRequest(body=str(err), content_type='text/plain',
                                                    request=req)
        if self.mount_check and not check_mount(self.root, drive):
            return Response(status='507 %s is not mounted' % drive)
        if 'x-timestamp' not in req.headers or \
                    not check_float(req.headers['x-timestamp']):
            return HTTPBadRequest(body='Missing timestamp', request=req,
                        content_type='text/plain')
        broker = self._get_account_broker(drive, part, account)
        if broker.is_deleted():
            return HTTPNotFound(request=req)
        broker.delete_db(req.headers['x-timestamp'])
        return HTTPNoContent(request=req)
示例#14
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     req_timestamp = valid_timestamp(req)
     if 'x-container-sync-to' in req.headers:
         err, sync_to, realm, realm_key = validate_sync_to(
             req.headers['x-container-sync-to'], self.allowed_sync_hosts,
             self.realms_conf)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     broker.update_put_timestamp(req_timestamp.internal)
     metadata = {}
     metadata.update((key, (value, req_timestamp.internal))
                     for key, value in req.headers.items()
                     if key.lower() in self.save_headers
                     or is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata, validate_metadata=True)
         self._update_sync_store(broker, 'POST')
     return HTTPNoContent(request=req)
示例#15
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing or bad timestamp',
                               request=req,
                               content_type='text/plain')
     if 'x-container-sync-to' in req.headers:
         err = validate_sync_to(req.headers['x-container-sync-to'],
                                self.allowed_sync_hosts)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     metadata = {}
     metadata.update((key, (value, timestamp))
                     for key, value in req.headers.iteritems()
                     if key.lower() in self.save_headers
                     or is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata)
     return HTTPNoContent(request=req)
示例#16
0
文件: server.py 项目: HoO-Group/swift
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     info = broker.get_info()
     headers = {
         'X-Container-Object-Count': info['object_count'],
         'X-Container-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp'],
     }
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.iteritems()
         if value != '' and (key.lower() in self.save_headers or
                             is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset='utf-8')
示例#17
0
    def GET(self, req):
        """Handle HTTP GET request."""
        drive, part, account = split_and_validate_path(req, 3)
        prefix = get_param(req, 'prefix')
        delimiter = get_param(req, 'delimiter')
        if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
            # delimiters can be made more flexible later
            return HTTPPreconditionFailed(body='Bad delimiter')
        limit = constraints.ACCOUNT_LISTING_LIMIT
        given_limit = get_param(req, 'limit')
        if given_limit and given_limit.isdigit():
            limit = int(given_limit)
            if limit > constraints.ACCOUNT_LISTING_LIMIT:
                return HTTPPreconditionFailed(
                    request=req,
                    body='Maximum limit is %d' %
                    constraints.ACCOUNT_LISTING_LIMIT)
        marker = get_param(req, 'marker', '')
        end_marker = get_param(req, 'end_marker')
        out_content_type = get_listing_content_type(req)

        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        broker = self._get_account_broker(drive,
                                          part,
                                          account,
                                          pending_timeout=0.1,
                                          stale_reads_ok=True)
        if broker.is_deleted():
            return self._deleted_response(broker, req, HTTPNotFound)
        return account_listing_response(account, req, out_content_type, broker,
                                        limit, marker, end_marker, prefix,
                                        delimiter)
示例#18
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     #logging.info("...head...")
     drive, part, account = split_and_validate_path(req, 3)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_account_broker(drive,
                                       part,
                                       account,
                                       pending_timeout=0.1,
                                       stale_reads_ok=True)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     info = broker.get_info()
     headers = {
         'X-Account-Container-Count': info['container_count'],
         'X-Account-Object-Count': info['object_count'],
         'X-Account-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp']
     }
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.iteritems()
         if value != '')
     headers['Content-Type'] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset='utf-8')
    def find_and_process(self):
        src_filename = time.strftime(self.filename_format)
        working_dir = os.path.join(self.target_dir, ".%-stats_tmp" % self.stats_type)
        shutil.rmtree(working_dir, ignore_errors=True)
        mkdirs(working_dir)
        tmp_filename = os.path.join(working_dir, src_filename)
        hasher = hashlib.md5()
        try:
            with open(tmp_filename, "wb") as statfile:
                statfile.write(self.get_header())
                for device in os.listdir(self.devices):
                    if self.mount_check and not check_mount(self.devices, device):
                        self.logger.error(_("Device %s is not mounted, skipping.") % device)
                        continue
                    db_dir = os.path.join(self.devices, device, self.data_dir)
                    if not os.path.exists(db_dir):
                        self.logger.debug(_("Path %s does not exist, skipping.") % db_dir)
                        continue
                    for root, dirs, files in os.walk(db_dir, topdown=False):
                        for filename in files:
                            if filename.endswith(".db"):
                                db_path = os.path.join(root, filename)
                                try:
                                    line_data = self.get_data(db_path)
                                except sqlite3.Error, err:
                                    self.logger.info(_("Error accessing db %s: %s") % (db_path, err))
                                    continue
                                if line_data:
                                    statfile.write(line_data)
                                    hasher.update(line_data)

            src_filename += hasher.hexdigest()
            renamer(tmp_filename, os.path.join(self.target_dir, src_filename))
示例#20
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, disk_chunk_size=65536,
                 bytes_per_sync=(512 * 1024 * 1024),
                 iter_hook=None, threadpool=None, obj_dir='objects',
                 mount_check=False):
        if mount_check and not check_mount(path, device):
            raise DiskFileDeviceUnavailable()
        self.disk_chunk_size = disk_chunk_size
        self.bytes_per_sync = bytes_per_sync
        self.iter_hook = iter_hook
        self.name = '/' + '/'.join((account, container, obj))
        name_hash = hash_path(account, container, obj)
        self.datadir = join(
            path, device, storage_directory(obj_dir, partition, name_hash))
        self.device_path = join(path, device)
        self.tmpdir = join(path, device, 'tmp')
        self.logger = logger
        self._metadata = None
        self.data_file = None
        self._data_file_size = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.suppress_file_closing = False
        self._verify_close = False
        self.threadpool = threadpool or ThreadPool(nthreads=0)

        # FIXME(clayg): this attribute is set after open and affects the
        # behavior of the class (i.e. public interface)
        self.keep_cache = False
示例#21
0
    def GET(self, req):
        """Handle HTTP GET request."""
        drive, part, account = split_and_validate_path(req, 3)
        prefix = get_param(req, "prefix")
        delimiter = get_param(req, "delimiter")
        if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
            # delimiters can be made more flexible later
            return HTTPPreconditionFailed(body="Bad delimiter")
        limit = constraints.ACCOUNT_LISTING_LIMIT
        given_limit = get_param(req, "limit")
        if given_limit and given_limit.isdigit():
            limit = int(given_limit)
            if limit > constraints.ACCOUNT_LISTING_LIMIT:
                return HTTPPreconditionFailed(
                    request=req, body="Maximum limit is %d" % constraints.ACCOUNT_LISTING_LIMIT
                )
        marker = get_param(req, "marker", "")
        end_marker = get_param(req, "end_marker")
        out_content_type = get_listing_content_type(req)

        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True)
        if broker.is_deleted():
            return self._deleted_response(broker, req, HTTPNotFound)
        return account_listing_response(
            account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter
        )
示例#22
0
文件: server.py 项目: sagarjha/swift
 def PUT(self, req):
     """Handle HTTP PUT request."""
     drive, part, account, container = split_and_validate_path(req, 3, 4)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     if container:   # put account container
         pending_timeout = None
         container_policy_index = req.headers.get(POLICY_INDEX, 0)
         if 'x-trans-id' in req.headers:
             pending_timeout = 3
         broker = self._get_account_broker(drive, part, account,
                                           pending_timeout=pending_timeout)
         if account.startswith(self.auto_create_account_prefix) and \
                 not os.path.exists(broker.db_file):
             try:
                 broker.initialize(normalize_timestamp(
                     req.headers.get('x-timestamp') or time.time()))
             except DatabaseAlreadyExists:
                 pass
         if req.headers.get('x-account-override-deleted', 'no').lower() != \
                 'yes' and broker.is_deleted():
             return HTTPNotFound(request=req)
         broker.put_container(container, req.headers['x-put-timestamp'],
                              req.headers['x-delete-timestamp'],
                              req.headers['x-object-count'],
                              req.headers['x-bytes-used'],
                              container_policy_index)
         if req.headers['x-delete-timestamp'] > \
                 req.headers['x-put-timestamp']:
             return HTTPNoContent(request=req)
         else:
             return HTTPCreated(request=req)
     else:   # put account
         broker = self._get_account_broker(drive, part, account)
         timestamp = normalize_timestamp(req.headers['x-timestamp'])
         if not os.path.exists(broker.db_file):
             try:
                 broker.initialize(timestamp)
                 created = True
             except DatabaseAlreadyExists:
                 created = False
         elif broker.is_status_deleted():
             return self._deleted_response(broker, req, HTTPForbidden,
                                           body='Recently deleted')
         else:
             created = broker.is_deleted()
             broker.update_put_timestamp(timestamp)
             if broker.is_deleted():
                 return HTTPConflict(request=req)
         metadata = {}
         metadata.update((key, (value, timestamp))
                         for key, value in req.headers.iteritems()
                         if is_sys_or_user_meta('account', key))
         if metadata:
             broker.update_metadata(metadata)
         if created:
             return HTTPCreated(request=req)
         else:
             return HTTPAccepted(request=req)
示例#23
0
文件: recon.py 项目: masoud91/swift
    def get_unmounted(self):
        """list unmounted (failed?) devices"""
        mountlist = []
        for entry in os.listdir(self.devices):
            if not os.path.isdir(os.path.join(self.devices, entry)):
                continue

            try:
                check_mount(self.devices, entry)
            except OSError as err:
                mounted = str(err)
            except ValueError:
                mounted = False
            else:
                continue
            mountlist.append({'device': entry, 'mounted': mounted})
        return mountlist
示例#24
0
文件: recon.py 项目: jgmerritt/swift
    def get_unmounted(self):
        """list unmounted (failed?) devices"""
        mountlist = []
        for entry in os.listdir(self.devices):
            if not os.path.isdir(os.path.join(self.devices, entry)):
                continue

            try:
                check_mount(self.devices, entry)
            except OSError as err:
                mounted = str(err)
            except ValueError:
                mounted = False
            else:
                continue
            mountlist.append({'device': entry, 'mounted': mounted})
        return mountlist
示例#25
0
文件: recon.py 项目: noodley/swift
 def get_unmounted(self):
     """list unmounted (failed?) devices"""
     mountlist = []
     for entry in os.listdir(self.devices):
         mpoint = {"device": entry, "mounted": check_mount(self.devices, entry)}
         if not mpoint["mounted"]:
             mountlist.append(mpoint)
     return mountlist
示例#26
0
class AccountController(object):
    """WSGI controller for the account server."""
    def __init__(self, conf):
        self.logger = get_logger(conf, log_route='account-server')
        self.root = conf.get('devices', '/srv/node')
        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
        replication_server = conf.get('replication_server', None)
        if replication_server is None:
            allowed_methods = [
                'DELETE', 'PUT', 'HEAD', 'GET', 'REPLICATE', 'POST'
            ]
        else:
            replication_server = config_true_value(replication_server)
            if replication_server:
                allowed_methods = ['REPLICATE']
            else:
                allowed_methods = ['DELETE', 'PUT', 'HEAD', 'GET', 'POST']
        self.replication_server = replication_server
        self.allowed_methods = allowed_methods
        self.replicator_rpc = ReplicatorRpc(self.root,
                                            DATADIR,
                                            AccountBroker,
                                            self.mount_check,
                                            logger=self.logger)
        self.auto_create_account_prefix = \
            conf.get('auto_create_account_prefix') or '.'
        swift.common.db.DB_PREALLOCATION = \
            config_true_value(conf.get('db_preallocation', 'f'))

    def _get_account_broker(self, drive, part, account):
        hsh = hash_path(account)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        return AccountBroker(db_path, account=account, logger=self.logger)

    @public
    @timing_stats()
    def DELETE(self, req):
        """Handle HTTP DELETE request."""
        try:
            drive, part, account = req.split_path(3)
            validate_device_partition(drive, part)
        except ValueError, err:
            return HTTPBadRequest(body=str(err),
                                  content_type='text/plain',
                                  request=req)
        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        if 'x-timestamp' not in req.headers or \
                not check_float(req.headers['x-timestamp']):
            return HTTPBadRequest(body='Missing timestamp',
                                  request=req,
                                  content_type='text/plain')
        broker = self._get_account_broker(drive, part, account)
        if broker.is_deleted():
            return HTTPNotFound(request=req)
        broker.delete_db(req.headers['x-timestamp'])
        return HTTPNoContent(request=req)
示例#27
0
 def test_check_drive_invalid_path(self):
     root = '/srv/'
     with mock_check_drive() as mocks:
         self.assertIsNone(constraints.check_dir(root, 'foo?bar'))
         self.assertIsNone(constraints.check_mount(root, 'foo bar'))
         self.assertIsNone(constraints.check_drive(root, 'foo/bar', True))
         self.assertIsNone(constraints.check_drive(root, 'foo%bar', False))
     self.assertEqual([], mocks['isdir'].call_args_list)
     self.assertEqual([], mocks['ismount'].call_args_list)
示例#28
0
 def test_check_drive_invalid_path(self):
     root = '/srv/'
     with mock_check_drive() as mocks:
         self.assertIsNone(constraints.check_dir(root, 'foo?bar'))
         self.assertIsNone(constraints.check_mount(root, 'foo bar'))
         self.assertIsNone(constraints.check_drive(root, 'foo/bar', True))
         self.assertIsNone(constraints.check_drive(root, 'foo%bar', False))
     self.assertEqual([], mocks['isdir'].call_args_list)
     self.assertEqual([], mocks['ismount'].call_args_list)
示例#29
0
 def __init__(self, root, account, fs_object=None):
     self.root = root
     self.account = account
     self.datadir = os.path.join(self.root, self.account)
     if not check_mount(root, account):
         check_valid_account(account, fs_object)
     self.metadata = read_metadata(self.datadir)
     if not self.metadata or not validate_account(self.metadata):
         self.metadata = create_account_metadata(self.datadir)
示例#30
0
 def get_unmounted(self):
     """list unmounted (failed?) devices"""
     mountlist = []
     for entry in os.listdir(self.devices):
         mpoint = {'device': entry,
                   'mounted': check_mount(self.devices, entry)}
         if not mpoint['mounted']:
             mountlist.append(mpoint)
     return mountlist
示例#31
0
 def PUT(self, req):
     """Handle HTTP PUT request."""
     drive, part, account, container = split_and_validate_path(req, 3, 4)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     if container:   # put account container
         pending_timeout = None
         if 'x-trans-id' in req.headers:
             pending_timeout = 3
         broker = self._get_account_broker(drive, part, account,
                                           pending_timeout=pending_timeout)
         if account.startswith(self.auto_create_account_prefix) and \
                 not os.path.exists(broker.db_file):
             try:
                 broker.initialize(normalize_timestamp(
                     req.headers.get('x-timestamp') or time.time()))
             except DatabaseAlreadyExists:
                 pass
         if req.headers.get('x-account-override-deleted', 'no').lower() != \
                 'yes' and broker.is_deleted():
             return HTTPNotFound(request=req)
         broker.put_container(container, req.headers['x-put-timestamp'],
                              req.headers['x-delete-timestamp'],
                              req.headers['x-object-count'],
                              req.headers['x-bytes-used'])
         if req.headers['x-delete-timestamp'] > \
                 req.headers['x-put-timestamp']:
             return HTTPNoContent(request=req)
         else:
             return HTTPCreated(request=req)
     else:   # put account
         broker = self._get_account_broker(drive, part, account)
         timestamp = normalize_timestamp(req.headers['x-timestamp'])
         if not os.path.exists(broker.db_file):
             try:
                 broker.initialize(timestamp)
                 created = True
             except DatabaseAlreadyExists:
                 created = False
         elif broker.is_status_deleted():
             return self._deleted_response(broker, req, HTTPForbidden,
                                           body='Recently deleted')
         else:
             created = broker.is_deleted()
             broker.update_put_timestamp(timestamp)
             if broker.is_deleted():
                 return HTTPConflict(request=req)
         metadata = {}
         metadata.update((key, (value, timestamp))
                         for key, value in req.headers.iteritems()
                         if key.lower().startswith('x-account-meta-'))
         if metadata:
             broker.update_metadata(metadata)
         if created:
             return HTTPCreated(request=req)
         else:
             return HTTPAccepted(request=req)
示例#32
0
 def __init__(self, root, account, fs_object = None):
     self.root = root
     self.account = account
     self.datadir = os.path.join(self.root, self.account)
     if not check_mount(root, account):
         check_valid_account(account, fs_object)
     self.metadata = read_metadata(self.datadir)
     if not self.metadata or not validate_account(self.metadata):
         self.metadata = create_account_metadata(self.datadir)
示例#33
0
 def __init__(self,
              path,
              device,
              partition,
              account,
              container,
              logger,
              uid=DEFAULT_UID,
              gid=DEFAULT_GID,
              fs_object=None):
     self.root = path
     device = account
     if container:
         self.name = container
     else:
         self.name = None
     if self.name:
         self.datadir = os.path.join(path, account, self.name)
     else:
         self.datadir = os.path.join(path, device)
     self.account = account
     self.device_path = os.path.join(path, device)
     if not check_mount(path, device):
         check_valid_account(account, fs_object)
     self.logger = logger
     self.metadata = {}
     self.uid = int(uid)
     self.gid = int(gid)
     # Create a dummy db_file in /etc/swift
     self.db_file = '/etc/swift/db_file.db'
     if not os.path.exists(self.db_file):
         file(self.db_file, 'w+')
     self.dir_exists = os.path.exists(self.datadir)
     if self.dir_exists:
         try:
             self.metadata = read_metadata(self.datadir)
         except EOFError:
             create_container_metadata(self.datadir)
     else:
         return
     if container:
         if not self.metadata:
             create_container_metadata(self.datadir)
             self.metadata = read_metadata(self.datadir)
         else:
             if not validate_container(self.metadata):
                 create_container_metadata(self.datadir)
                 self.metadata = read_metadata(self.datadir)
     else:
         if not self.metadata:
             create_account_metadata(self.datadir)
             self.metadata = read_metadata(self.datadir)
         else:
             if not validate_account(self.metadata):
                 create_account_metadata(self.datadir)
                 self.metadata = read_metadata(self.datadir)
示例#34
0
 def test_check_mount(self):
     self.assertFalse(constraints.check_mount('', ''))
     with mock.patch("swift.common.constraints.ismount", MockTrue()):
         self.assertTrue(constraints.check_mount('/srv', '1'))
         self.assertTrue(constraints.check_mount('/srv', 'foo-bar'))
         self.assertTrue(constraints.check_mount('/srv', '003ed03c-242a-4b2f-bee9-395f801d1699'))
         self.assertFalse(constraints.check_mount('/srv', 'foo bar'))
         self.assertFalse(constraints.check_mount('/srv', 'foo/bar'))
         self.assertFalse(constraints.check_mount('/srv', 'foo?bar'))
示例#35
0
 def test_check_mount(self):
     self.assertFalse(constraints.check_mount("", ""))
     with mock.patch("swift.common.utils.ismount", MockTrue()):
         self.assertTrue(constraints.check_mount("/srv", "1"))
         self.assertTrue(constraints.check_mount("/srv", "foo-bar"))
         self.assertTrue(constraints.check_mount("/srv", "003ed03c-242a-4b2f-bee9-395f801d1699"))
         self.assertFalse(constraints.check_mount("/srv", "foo bar"))
         self.assertFalse(constraints.check_mount("/srv", "foo/bar"))
         self.assertFalse(constraints.check_mount("/srv", "foo?bar"))
示例#36
0
 def DELETE(self, req):
     """Handle HTTP DELETE request."""
     drive, part, account = split_and_validate_path(req, 3)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     req_timestamp = valid_timestamp(req)
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     broker.delete_db(req_timestamp.internal)
     return self._deleted_response(broker, req, HTTPNoContent)
示例#37
0
 def DELETE(self, req):
     """Handle HTTP DELETE request."""
     drive, part, account = split_and_validate_path(req, 3)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     req_timestamp = valid_timestamp(req)
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     broker.delete_db(req_timestamp.internal)
     return self._deleted_response(broker, req, HTTPNoContent)
示例#38
0
 def DELETE(self, req):
     """Handle HTTP DELETE request."""
     drive, part, account = split_and_validate_path(req, 3)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     if "x-timestamp" not in req.headers or not check_float(req.headers["x-timestamp"]):
         return HTTPBadRequest(body="Missing timestamp", request=req, content_type="text/plain")
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     broker.delete_db(req.headers["x-timestamp"])
     return self._deleted_response(broker, req, HTTPNoContent)
示例#39
0
 def test_check_mount(self):
     self.assertFalse(constraints.check_mount('', ''))
     constraints.os = MockTrue()  # mock os module
     self.assertTrue(constraints.check_mount('/srv', '1'))
     self.assertTrue(constraints.check_mount('/srv', 'foo-bar'))
     self.assertTrue(constraints.check_mount('/srv', '003ed03c-242a-4b2f-bee9-395f801d1699'))
     self.assertFalse(constraints.check_mount('/srv', 'foo bar'))
     self.assertFalse(constraints.check_mount('/srv', 'foo/bar'))
     self.assertFalse(constraints.check_mount('/srv', 'foo?bar'))
     reload(constraints)  # put it back
示例#40
0
文件: recon.py 项目: bkolli/swift
 def get_unmounted(self):
     """list unmounted (failed?) devices"""
     mountlist = []
     for entry in os.listdir(self.devices):
         try:
             mounted = check_mount(self.devices, entry)
         except OSError as err:
             mounted = str(err)
         mpoint = {'device': entry, 'mounted': mounted}
         if mpoint['mounted'] is not True:
             mountlist.append(mpoint)
     return mountlist
示例#41
0
 def PUT(self, req):
     """Handle HTTP PUT request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing timestamp', request=req,
                               content_type='text/plain')
     if 'x-container-sync-to' in req.headers:
         err, sync_to, realm, realm_key = validate_sync_to(
             req.headers['x-container-sync-to'], self.allowed_sync_hosts,
             self.realms_conf)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     broker = self._get_container_broker(drive, part, account, container)
     if obj:     # put container object
         if account.startswith(self.auto_create_account_prefix) and \
                 not os.path.exists(broker.db_file):
             try:
                 broker.initialize(timestamp)
             except DatabaseAlreadyExists:
                 pass
         if not os.path.exists(broker.db_file):
             return HTTPNotFound()
         broker.put_object(obj, timestamp, int(req.headers['x-size']),
                           req.headers['x-content-type'],
                           req.headers['x-etag'])
         return HTTPCreated(request=req)
     else:   # put container
         created = self._update_or_create(req, broker, timestamp)
         metadata = {}
         metadata.update(
             (key, (value, timestamp))
             for key, value in req.headers.iteritems()
             if key.lower() in self.save_headers or
             is_sys_or_user_meta('container', key))
         if metadata:
             if 'X-Container-Sync-To' in metadata:
                 if 'X-Container-Sync-To' not in broker.metadata or \
                         metadata['X-Container-Sync-To'][0] != \
                         broker.metadata['X-Container-Sync-To'][0]:
                     broker.set_x_container_sync_points(-1, -1)
             broker.update_metadata(metadata)
         resp = self.account_update(req, account, container, broker)
         if resp:
             return resp
         if created:
             return HTTPCreated(request=req)
         else:
             return HTTPAccepted(request=req)
示例#42
0
 def test_check_mount(self):
     self.assertFalse(constraints.check_mount('', ''))
     constraints.os = MockTrue()  # mock os module
     self.assertTrue(constraints.check_mount('/srv', '1'))
     self.assertTrue(constraints.check_mount('/srv', 'foo-bar'))
     self.assertTrue(constraints.check_mount('/srv', '003ed03c-242a-4b2f-bee9-395f801d1699'))
     self.assertFalse(constraints.check_mount('/srv', 'foo bar'))
     self.assertFalse(constraints.check_mount('/srv', 'foo/bar'))
     self.assertFalse(constraints.check_mount('/srv', 'foo?bar'))
     reload(constraints)  # put it back
示例#43
0
    def GET(self, req):
        """Handle HTTP GET request."""
        # 从原始请求中获取相关信息
        drive, part, account, container, obj = split_and_validate_path(
            req, 4, 5, True)
        path = get_param(req, 'path')
        prefix = get_param(req, 'prefix')
        delimiter = get_param(req, 'delimiter')
        if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
            # delimiters can be made more flexible later
            return HTTPPreconditionFailed(body='Bad delimiter')
        marker = get_param(req, 'marker', '')
        end_marker = get_param(req, 'end_marker')
        limit = constraints.CONTAINER_LISTING_LIMIT
        given_limit = get_param(req, 'limit')
        # 如果设置的对象数量限制超过系统定义的阈值,则报错412
        if given_limit and given_limit.isdigit():
            limit = int(given_limit)
            if limit > constraints.CONTAINER_LISTING_LIMIT:
                return HTTPPreconditionFailed(
                    request=req,
                    body='Maximum limit is %d'
                    % constraints.CONTAINER_LISTING_LIMIT)
        out_content_type = get_listing_content_type(req)
        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)

        # 返回一个container broker实例,用于代理其数据块操作
        broker = self._get_container_broker(drive, part, account, container,
                                            pending_timeout=0.1,
                                            stale_reads_ok=True)

        # info = SELECT account, container, created_at, put_timestamp,
        #                     delete_timestamp, status_changed_at,
        #                     object_count, bytes_used,
        #                     reported_put_timestamp, reported_delete_timestamp,
        #                     reported_object_count, reported_bytes_used, hash,
        #                     id, %s, %s
        #                     FROM container_stat
        info, is_deleted = broker.get_info_is_deleted()
        resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
        # 如果container元数据不存在,则报错404
        if is_deleted:
            return HTTPNotFound(request=req, headers=resp_headers)

        # 获取objects的排序列表
        container_list = broker.list_objects_iter(
            limit, marker, end_marker, prefix, delimiter, path,
            storage_policy_index=info['storage_policy_index'])

        # 根据json和xml的不同创建列表
        return self.create_listing(req, out_content_type, info, resp_headers,
                                   broker.metadata, container_list, container)
示例#44
0
文件: server.py 项目: r0mik/swift
 def REPLICATE(self, req):
     """
     Handle HTTP REPLICATE request (json-encoded RPC calls for replication.)
     """
     post_args = split_and_validate_path(req, 3)
     drive, partition, hash = post_args
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     try:
         args = json.load(req.environ['wsgi.input'])
     except ValueError, err:
         return HTTPBadRequest(body=str(err), content_type='text/plain')
示例#45
0
 def test_check_mount(self):
     self.assertFalse(constraints.check_mount('', ''))
     with mock.patch("swift.common.utils.ismount", MockTrue()):
         self.assertTrue(constraints.check_mount('/srv', '1'))
         self.assertTrue(constraints.check_mount('/srv', 'foo-bar'))
         self.assertTrue(constraints.check_mount(
             '/srv', '003ed03c-242a-4b2f-bee9-395f801d1699'))
         self.assertFalse(constraints.check_mount('/srv', 'foo bar'))
         self.assertFalse(constraints.check_mount('/srv', 'foo/bar'))
         self.assertFalse(constraints.check_mount('/srv', 'foo?bar'))
示例#46
0
 def REPLICATE(self, req):
     """
     Handle HTTP REPLICATE request (json-encoded RPC calls for replication.)
     """
     post_args = split_and_validate_path(req, 3)
     drive, partition, hash = post_args
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     try:
         args = json.load(req.environ['wsgi.input'])
     except ValueError, err:
         return HTTPBadRequest(body=str(err), content_type='text/plain')
示例#47
0
 def get_unmounted(self):
     """list unmounted (failed?) devices"""
     mountlist = []
     for entry in os.listdir(self.devices):
         try:
             mounted = check_mount(self.devices, entry)
         except OSError as err:
             mounted = str(err)
         mpoint = {'device': entry, 'mounted': mounted}
         if mpoint['mounted'] is not True:
             mountlist.append(mpoint)
     return mountlist
示例#48
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account = split_and_validate_path(req, 3)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     headers = get_response_headers(broker)
     headers["Content-Type"] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset="utf-8")
示例#49
0
文件: server.py 项目: a4a881d4/swift
class AccountController(object):
    """WSGI controller for the account server."""
    def __init__(self, conf):
        self.logger = get_logger(conf, log_route='account-server')
        self.root = conf.get('devices', '/srv/node')
        self.mount_check = conf.get('mount_check', 'true').lower() in \
                              ('true', 't', '1', 'on', 'yes', 'y')
        self.replicator_rpc = ReplicatorRpc(self.root,
                                            DATADIR,
                                            AccountBroker,
                                            self.mount_check,
                                            logger=self.logger)
        self.auto_create_account_prefix = \
            conf.get('auto_create_account_prefix') or '.'
        swift.common.db.DB_PREALLOCATION = \
            conf.get('db_preallocation', 'f').lower() in TRUE_VALUES

    def _get_account_broker(self, drive, part, account):
        hsh = hash_path(account)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        return AccountBroker(db_path, account=account, logger=self.logger)

    @public
    def DELETE(self, req):
        """Handle HTTP DELETE request."""
        start_time = time.time()
        try:
            drive, part, account = split_path(unquote(req.path), 3)
            validate_device_partition(drive, part)
        except ValueError, err:
            self.logger.increment('DELETE.errors')
            return HTTPBadRequest(body=str(err),
                                  content_type='text/plain',
                                  request=req)
        if self.mount_check and not check_mount(self.root, drive):
            self.logger.increment('DELETE.errors')
            return HTTPInsufficientStorage(drive=drive, request=req)
        if 'x-timestamp' not in req.headers or \
                    not check_float(req.headers['x-timestamp']):
            self.logger.increment('DELETE.errors')
            return HTTPBadRequest(body='Missing timestamp',
                                  request=req,
                                  content_type='text/plain')
        broker = self._get_account_broker(drive, part, account)
        if broker.is_deleted():
            self.logger.timing_since('DELETE.timing', start_time)
            return HTTPNotFound(request=req)
        broker.delete_db(req.headers['x-timestamp'])
        self.logger.timing_since('DELETE.timing', start_time)
        return HTTPNoContent(request=req)
示例#50
0
 def find_and_process(self):
     src_filename = time.strftime(self.filename_format)
     working_dir = os.path.join(self.target_dir, '.stats_tmp')
     shutil.rmtree(working_dir, ignore_errors=True)
     mkdirs(working_dir)
     tmp_filename = os.path.join(working_dir, src_filename)
     hasher = hashlib.md5()
     with open(tmp_filename, 'wb') as statfile:
         # csv has the following columns:
         # Account Name, Container Count, Object Count, Bytes Used
         for device in os.listdir(self.devices):
             if self.mount_check and not check_mount(self.devices, device):
                 self.logger.error(
                     _("Device %s is not mounted, skipping.") % device)
                 continue
             accounts = os.path.join(self.devices,
                                     device,
                                     account_server_data_dir)
             if not os.path.exists(accounts):
                 self.logger.debug(_("Path %s does not exist, skipping.") %
                     accounts)
                 continue
             for root, dirs, files in os.walk(accounts, topdown=False):
                 for filename in files:
                     if filename.endswith('.db'):
                         db_path = os.path.join(root, filename)
                         broker = AccountBroker(db_path)
                         if not broker.is_deleted():
                             (account_name,
                             _junk, _junk, _junk,
                             container_count,
                             object_count,
                             bytes_used,
                             _junk, _junk) = broker.get_info()
                             line_data = '"%s",%d,%d,%d\n' % (
                                 account_name, container_count,
                                 object_count, bytes_used)
                             statfile.write(line_data)
                             hasher.update(line_data)
     file_hash = hasher.hexdigest()
     hash_index = src_filename.find('*')
     if hash_index < 0:
         # if there is no * in the target filename, the uploader probably
         # won't work because we are crafting a filename that doesn't
         # fit the pattern
         src_filename = '_'.join([src_filename, file_hash])
     else:
         parts = src_filename[:hash_index], src_filename[hash_index + 1:]
         src_filename = ''.join([parts[0], file_hash, parts[1]])
     renamer(tmp_filename, os.path.join(self.target_dir, src_filename))
     shutil.rmtree(working_dir, ignore_errors=True)
示例#51
0
    def get_dev_path(self, device):
        """
        Return the path to a device, checking to see that it is a proper mount
        point based on a configuration parameter.

        :param device: name of target device
        :returns: full path to the device, None if the path to the device is
                  not a proper mount point.
        """
        if self.mount_check and not check_mount(self.devices, device):
            dev_path = None
        else:
            dev_path = os.path.join(self.devices, device)
        return dev_path
示例#52
0
文件: diskfile.py 项目: yohsuke/swift
    def get_dev_path(self, device):
        """
        Return the path to a device, checking to see that it is a proper mount
        point based on a configuration parameter.

        :param device: name of target device
        :returns: full path to the device, None if the path to the device is
                  not a proper mount point.
        """
        if self.mount_check and not check_mount(self.devices, device):
            dev_path = None
        else:
            dev_path = os.path.join(self.devices, device)
        return dev_path
示例#53
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account = split_and_validate_path(req, 3)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_account_broker(drive, part, account,
                                       pending_timeout=0.1,
                                       stale_reads_ok=True)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     headers = get_response_headers(broker)
     headers['Content-Type'] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset='utf-8')
示例#54
0
文件: recon.py 项目: noodley/swift
 def get_diskusage(self):
     """get disk utilization statistics"""
     devices = []
     for entry in os.listdir(self.devices):
         if check_mount(self.devices, entry):
             path = "%s/%s" % (self.devices, entry)
             disk = os.statvfs(path)
             capacity = disk.f_bsize * disk.f_blocks
             available = disk.f_bsize * disk.f_bavail
             used = disk.f_bsize * (disk.f_blocks - disk.f_bavail)
             devices.append({"device": entry, "mounted": True, "size": capacity, "used": used, "avail": available})
         else:
             devices.append({"device": entry, "mounted": False, "size": "", "used": "", "avail": ""})
     return devices
示例#55
0
 def DELETE(self, req):
     """Handle HTTP DELETE request."""
     drive, part, account = split_and_validate_path(req, 3)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing timestamp', request=req,
                               content_type='text/plain')
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     broker.delete_db(req.headers['x-timestamp'])
     return self._deleted_response(broker, req, HTTPNoContent)
示例#56
0
    def GET(self, req):
        """Handle HTTP GET request."""
        drive, part, account, container, obj = split_and_validate_path(
            req, 4, 5, True)

        # import pydevd
        # pydevd.settrace('172.29.132.122', port=5678, stdoutToServer=True, stderrToServer=True)
        path = get_param(req, 'path')
        prefix = get_param(req, 'prefix')
        delimiter = get_param(req, 'delimiter')
        if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
            # delimiters can be made more flexible later
            return HTTPPreconditionFailed(body='Bad delimiter')
        marker = get_param(req, 'marker', '')
        end_marker = get_param(req, 'end_marker')
        limit = constraints.CONTAINER_LISTING_LIMIT
        given_limit = get_param(req, 'limit')
        reverse = config_true_value(get_param(req, 'reverse'))
        if given_limit and given_limit.isdigit():
            limit = int(given_limit)
            if limit > constraints.CONTAINER_LISTING_LIMIT:
                return HTTPPreconditionFailed(
                    request=req,
                    body='Maximum limit is %d' %
                    constraints.CONTAINER_LISTING_LIMIT)
        out_content_type = get_listing_content_type(req)
        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        broker = self._get_container_broker(drive,
                                            part,
                                            account,
                                            container,
                                            pending_timeout=0.1,
                                            stale_reads_ok=True)
        info, is_deleted = broker.get_info_is_deleted()
        resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
        if is_deleted:
            return HTTPNotFound(request=req, headers=resp_headers)
        container_list = broker.list_objects_iter(
            limit,
            marker,
            end_marker,
            prefix,
            delimiter,
            path,
            storage_policy_index=info['storage_policy_index'],
            reverse=reverse)
        return self.create_listing(req, out_content_type, info, resp_headers,
                                   broker.metadata, container_list, container)
示例#57
0
    def REPLICATE(self, request):
        """
        Handle REPLICATE requests for the Swift Object Server.  This is used
        by the object replicator to get hashes for directories.
        """
        device, partition, suffix = split_and_validate_path(request, 2, 3, True)

        if self.mount_check and not check_mount(self.devices, device):
            return HTTPInsufficientStorage(drive=device, request=request)
        path = os.path.join(self.devices, device, DATADIR, partition)
        if not os.path.exists(path):
            mkdirs(path)
        suffixes = suffix.split("-") if suffix else []
        _junk, hashes = self.threadpools[device].force_run_in_thread(get_hashes, path, recalculate=suffixes)
        return Response(body=pickle.dumps(hashes))
示例#58
0
文件: recon.py 项目: yinyalan/swift
    def get_unmounted(self):
        """list unmounted (failed?) devices"""
        mountlist = []
        for entry in os.listdir(self.devices):
            if not os.path.isdir(os.path.join(self.devices, entry)):
                continue

            try:
                mounted = bool(check_mount(self.devices, entry))
            except OSError as err:
                mounted = str(err)
            mpoint = {'device': entry, 'mounted': mounted}
            if mpoint['mounted'] is not True:
                mountlist.append(mpoint)
        return mountlist