def GET(self, req): """Handle HTTP GET request.""" drive, part, account = split_and_validate_path(req, 3) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') limit = constraints.ACCOUNT_LISTING_LIMIT given_limit = get_param(req, 'limit') reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.ACCOUNT_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter, reverse)
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = split_and_validate_path(req, 3) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') limit = constraints.ACCOUNT_LISTING_LIMIT given_limit = get_param(req, 'limit') reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.ACCOUNT_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter, reverse)
def test_check_drive_isdir(self): root = '/srv' path = 'sdb2' with mock_check_drive(isdir=True) as mocks: self.assertEqual('/srv/sdb2', constraints.check_dir(root, path)) self.assertEqual('/srv/sdb2', constraints.check_drive( root, path, False)) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['isdir'].call_args_list) self.assertEqual([], mocks['ismount'].call_args_list) with mock_check_drive(isdir=True) as mocks: with self.assertRaises(ValueError) as exc_mgr: constraints.check_mount(root, path) self.assertEqual(str(exc_mgr.exception), '/srv/sdb2 is not mounted') with self.assertRaises(ValueError) as exc_mgr: constraints.check_drive(root, path, True) self.assertEqual(str(exc_mgr.exception), '/srv/sdb2 is not mounted') self.assertEqual([], mocks['isdir'].call_args_list) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['ismount'].call_args_list)
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = get_account_name_and_placement(req) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') reverse = config_true_value(get_param(req, 'reverse')) limit = constrain_req_limit(req, constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter, reverse)
def test_check_drive_invalid_path(self): root = '/srv/' with mock_check_drive() as mocks: drive = 'foo?bar' with self.assertRaises(ValueError) as exc_mgr: constraints.check_dir(root, drive) self.assertEqual(str(exc_mgr.exception), '%s is not a valid drive name' % drive) drive = 'foo bar' with self.assertRaises(ValueError) as exc_mgr: constraints.check_mount(root, drive) self.assertEqual(str(exc_mgr.exception), '%s is not a valid drive name' % drive) drive = 'foo/bar' with self.assertRaises(ValueError) as exc_mgr: constraints.check_drive(root, drive, True) self.assertEqual(str(exc_mgr.exception), '%s is not a valid drive name' % drive) drive = 'foo%bar' with self.assertRaises(ValueError) as exc_mgr: constraints.check_drive(root, drive, False) self.assertEqual(str(exc_mgr.exception), '%s is not a valid drive name' % drive) self.assertEqual([], mocks['isdir'].call_args_list) self.assertEqual([], mocks['ismount'].call_args_list)
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = get_obj_name_and_placement(req) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=headers) headers.update( (str_to_wsgi(key), str_to_wsgi(value)) for key, (value, timestamp) in broker.metadata.items() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type resp = HTTPNoContent(request=req, headers=headers, charset='utf-8') resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp'])) return resp
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=headers) headers.update( (str_to_wsgi(key), str_to_wsgi(value)) for key, (value, timestamp) in broker.metadata.items() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type resp = HTTPNoContent(request=req, headers=headers, charset='utf-8') resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp'])) return resp
def test_check_drive_isdir(self): root = '/srv' path = 'sdb2' with mock_check_drive(isdir=True) as mocks: self.assertEqual('/srv/sdb2', constraints.check_dir(root, path)) self.assertEqual('/srv/sdb2', constraints.check_drive(root, path, False)) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['isdir'].call_args_list) self.assertEqual([], mocks['ismount'].call_args_list) with mock_check_drive(isdir=True) as mocks: with self.assertRaises(ValueError) as exc_mgr: constraints.check_mount(root, path) self.assertEqual(str(exc_mgr.exception), '/srv/sdb2 is not mounted') with self.assertRaises(ValueError) as exc_mgr: constraints.check_drive(root, path, True) self.assertEqual(str(exc_mgr.exception), '/srv/sdb2 is not mounted') self.assertEqual([], mocks['isdir'].call_args_list) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['ismount'].call_args_list)
def test_check_drive_invalid_path(self): root = '/srv/' with mock_check_drive() as mocks: self.assertIsNone(constraints.check_dir(root, 'foo?bar')) self.assertIsNone(constraints.check_mount(root, 'foo bar')) self.assertIsNone(constraints.check_drive(root, 'foo/bar', True)) self.assertIsNone(constraints.check_drive(root, 'foo%bar', False)) self.assertEqual([], mocks['isdir'].call_args_list) self.assertEqual([], mocks['ismount'].call_args_list)
def DELETE(self, req): """Handle HTTP DELETE request.""" drive, part, account = split_and_validate_path(req, 3) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) req_timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) broker.delete_db(req_timestamp.internal) return self._deleted_response(broker, req, HTTPNoContent)
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) self._update_metadata(req, broker, req_timestamp) return HTTPNoContent(request=req)
def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account = split_and_validate_path(req, 3) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) headers = get_response_headers(broker) headers['Content-Type'] = out_content_type return HTTPNoContent(request=req, headers=headers, charset='utf-8')
def test_check_drive_isdir(self): root = '/srv' path = 'sdb2' with mock_check_drive(isdir=True) as mocks: self.assertEqual('/srv/sdb2', constraints.check_dir(root, path)) self.assertEqual('/srv/sdb2', constraints.check_drive( root, path, False)) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['isdir'].call_args_list) self.assertEqual([], mocks['ismount'].call_args_list) with mock_check_drive(isdir=True) as mocks: self.assertIsNone(constraints.check_mount(root, path)) self.assertIsNone(constraints.check_drive(root, path, True)) self.assertEqual([], mocks['isdir'].call_args_list) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['ismount'].call_args_list)
def dispatch(self, replicate_args, args): if not hasattr(args, 'pop'): return HTTPBadRequest(body='Invalid object type') op = args.pop(0) drive, partition, hsh = replicate_args try: dev_path = check_drive(self.root, drive, self.mount_check) except ValueError: return Response(status='507 %s is not mounted' % drive) db_file = os.path.join(dev_path, storage_directory(self.datadir, partition, hsh), hsh + '.db') if op == 'rsync_then_merge': return self.rsync_then_merge(drive, db_file, args) if op == 'complete_rsync': return self.complete_rsync(drive, db_file, args) else: # someone might be about to rsync a db to us, # make sure there's a tmp dir to receive it. mkdirs(os.path.join(self.root, drive, 'tmp')) if not self._db_file_exists(db_file): return HTTPNotFound() return getattr(self, op)(self.broker_class(db_file, logger=self.logger), args)
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) if not check_drive(self.root, drive, self.mount_check): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) broker.update_put_timestamp(req_timestamp.internal) metadata = {} metadata.update( (key, (value, req_timestamp.internal)) for key, value in req.headers.items() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata, validate_metadata=True) self._update_sync_store(broker, 'POST') return HTTPNoContent(request=req)
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) if not check_drive(self.root, drive, self.mount_check): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) broker.update_put_timestamp(req_timestamp.internal) metadata = {} metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.items() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata, validate_metadata=True) self._update_sync_store(broker, 'POST') return HTTPNoContent(request=req)
def REPLICATE(self, req): """ Handle HTTP REPLICATE request (json-encoded RPC calls for replication.) """ post_args = split_and_validate_path(req, 3) drive, partition, hash = post_args try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) try: args = json.load(req.environ['wsgi.input']) except ValueError as err: return HTTPBadRequest(body=str(err), content_type='text/plain') ret = self.replicator_rpc.dispatch(post_args, args) ret.request = req return ret
def DELETE(self, req): """Handle HTTP DELETE request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) # policy index is only relevant for delete_obj (and transitively for # auto create accounts) obj_policy_index = self.get_and_validate_policy_index(req) or 0 broker = self._get_container_broker(drive, part, account, container) if account.startswith(self.auto_create_account_prefix) and obj and \ not os.path.exists(broker.db_file): try: broker.initialize(req_timestamp.internal, obj_policy_index) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() if obj: # delete object # redirect if a shard range exists for the object name redirect = self._redirect_to_shard(req, broker, obj) if redirect: return redirect broker.delete_object(obj, req.headers.get('x-timestamp'), obj_policy_index) return HTTPNoContent(request=req) else: # delete container if not broker.empty(): return HTTPConflict(request=req) existed = Timestamp(broker.get_info()['put_timestamp']) and \ not broker.is_deleted() broker.delete_db(req_timestamp.internal) if not broker.is_deleted(): return HTTPConflict(request=req) self._update_sync_store(broker, 'DELETE') resp = self.account_update(req, account, container, broker) if resp: return resp if existed: return HTTPNoContent(request=req) return HTTPNotFound()
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) metadata = {} metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata, validate_metadata=True) return HTTPNoContent(request=req)
def test_check_drive_isdir(self): root = '/srv' path = 'sdb2' with mock_check_drive(isdir=True) as mocks: self.assertEqual('/srv/sdb2', constraints.check_dir(root, path)) self.assertEqual('/srv/sdb2', constraints.check_drive(root, path, False)) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['isdir'].call_args_list) self.assertEqual([], mocks['ismount'].call_args_list) with mock_check_drive(isdir=True) as mocks: self.assertIsNone(constraints.check_mount(root, path)) self.assertIsNone(constraints.check_drive(root, path, True)) self.assertEqual([], mocks['isdir'].call_args_list) self.assertEqual([mock.call('/srv/sdb2'), mock.call('/srv/sdb2')], mocks['ismount'].call_args_list)
def REPLICATE(self, req): """ Handle HTTP REPLICATE request (json-encoded RPC calls for replication.) """ post_args = split_and_validate_path(req, 3) drive, partition, hash = post_args try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) try: args = json.load(req.environ['wsgi.input']) except ValueError as err: return HTTPBadRequest(body=str(err), content_type='text/plain') ret = self.replicator_rpc.dispatch(post_args, args) ret.request = req return ret
def run_once(self, *args, **kwargs): """Run a replication pass once.""" self._zero_stats() dirs = [] ips = whataremyips(self.bind_ip) if not ips: self.logger.error(_('ERROR Failed to get my own IPs?')) return if self.handoffs_only: self.logger.warning( 'Starting replication pass with handoffs_only enabled. ' 'This mode is not intended for normal ' 'operation; use handoffs_only with care.') self._local_device_ids = set() found_local = False for node in self.ring.devs: if node and is_local_device(ips, self.port, node['replication_ip'], node['replication_port']): found_local = True if not check_drive(self.root, node['device'], self.mount_check): self._add_failure_stats([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in self.ring.devs if failure_dev ]) self.logger.warning( _('Skipping %(device)s as it is not mounted') % node) continue unlink_older_than( os.path.join(self.root, node['device'], 'tmp'), time.time() - self.reclaim_age) datadir = os.path.join(self.root, node['device'], self.datadir) if os.path.isdir(datadir): self._local_device_ids.add(node['id']) filt = (self.handoffs_only_filter(node['id']) if self.handoffs_only else None) dirs.append((datadir, node['id'], filt)) if not found_local: self.logger.error( "Can't find itself %s with port %s in ring " "file, not replicating", ", ".join(ips), self.port) self.logger.info(_('Beginning replication run')) for part, object_file, node_id in roundrobin_datadirs(dirs): self.cpool.spawn_n(self._replicate_object, part, object_file, node_id) self.cpool.waitall() self.logger.info(_('Replication run OVER')) if self.handoffs_only: self.logger.warning( 'Finished replication pass with handoffs_only enabled. ' 'If handoffs_only is no longer required, disable it.') self._report_stats()
def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) metadata = {} metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata, validate_metadata=True) return HTTPNoContent(request=req)
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) broker.update_put_timestamp(req_timestamp.internal) self._update_metadata(req, broker, req_timestamp, 'POST') return HTTPNoContent(request=req)
def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) broker.update_put_timestamp(req_timestamp.internal) self._update_metadata(req, broker, req_timestamp, 'POST') return HTTPNoContent(request=req)
def run_once(self, *args, **kwargs): """ Main entry point when running the reaper in 'once' mode, where it will do a single pass over all accounts on the server. This is called repeatedly by :func:`run_forever`. This will call :func:`reap_device` once for each device on the server. """ self.logger.debug('Begin devices pass: %s', self.devices) begin = time() try: for device in os.listdir(self.devices): try: check_drive(self.devices, device, self.mount_check) except ValueError as err: self.logger.increment('errors') self.logger.debug('Skipping: %s', err) continue self.reap_device(device) except (Exception, Timeout): self.logger.exception("Exception in top-level account reaper " "loop") elapsed = time() - begin self.logger.info('Devices pass completed: %.02fs', elapsed)
def UPDATE(self, req): """ Handle HTTP UPDATE request (merge_items RPCs coming from the proxy.) """ drive, part, account, container = split_and_validate_path(req, 4) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) requested_policy_index = self.get_and_validate_policy_index(req) broker = self._get_container_broker(drive, part, account, container) self._maybe_autocreate(broker, req_timestamp, account, requested_policy_index) try: objs = json.load(req.environ['wsgi.input']) except ValueError as err: return HTTPBadRequest(body=str(err), content_type='text/plain') broker.merge_items(objs) return HTTPAccepted(request=req)
def UPDATE(self, req): """ Handle HTTP UPDATE request (merge_items RPCs coming from the proxy.) """ drive, part, account, container = get_container_name_and_placement(req) req_timestamp = valid_timestamp(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) requested_policy_index = self.get_and_validate_policy_index(req) broker = self._get_container_broker(drive, part, account, container) self._maybe_autocreate(broker, req_timestamp, account, requested_policy_index) try: objs = json.load(req.environ['wsgi.input']) except ValueError as err: return HTTPBadRequest(body=str(err), content_type='text/plain') broker.merge_items(objs) return HTTPAccepted(request=req)
def run_once(self, *args, **kwargs): """ Main entry point when running the reaper in 'once' mode, where it will do a single pass over all accounts on the server. This is called repeatedly by :func:`run_forever`. This will call :func:`reap_device` once for each device on the server. """ self.logger.debug('Begin devices pass: %s', self.devices) begin = time() try: for device in os.listdir(self.devices): try: check_drive(self.devices, device, self.mount_check) except ValueError as err: self.logger.increment('errors') self.logger.debug('Skipping: %s', err) continue self.reap_device(device) except (Exception, Timeout): self.logger.exception(_("Exception in top-level account reaper " "loop")) elapsed = time() - begin self.logger.info(_('Devices pass completed: %.02fs'), elapsed)
def run_forever(self, *args, **kwargs): """Run the updater continuously.""" time.sleep(random() * self.interval) while True: self.logger.info(_('Begin object update sweep')) begin = time.time() pids = [] # read from container ring to ensure it's fresh self.get_container_ring().get_nodes('') for device in self._listdir(self.devices): try: dev_path = check_drive(self.devices, device, self.mount_check) except ValueError as err: # We don't count this as an error. The occasional # unmounted drive is part of normal cluster operations, # so a simple warning is sufficient. self.logger.warning('Skipping: %s', err) continue while len(pids) >= self.updater_workers: pids.remove(os.wait()[0]) pid = os.fork() if pid: pids.append(pid) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) eventlet_monkey_patch() self.stats.reset() forkbegin = time.time() self.object_sweep(dev_path) elapsed = time.time() - forkbegin self.logger.info(('Object update sweep of %(device)s ' 'completed: %(elapsed).02fs, %(stats)s'), { 'device': device, 'elapsed': elapsed, 'stats': self.stats }) sys.exit() while pids: pids.remove(os.wait()[0]) elapsed = time.time() - begin self.logger.info(_('Object update sweep completed: %.02fs'), elapsed) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger) if elapsed < self.interval: time.sleep(self.interval - elapsed)
def run_forever(self, *args, **kwargs): """Run the updater continuously.""" time.sleep(random() * self.interval) while True: self.logger.info(_('Begin object update sweep')) begin = time.time() pids = [] # read from container ring to ensure it's fresh self.get_container_ring().get_nodes('') for device in self._listdir(self.devices): if not check_drive(self.devices, device, self.mount_check): self.logger.increment('errors') self.logger.warning(_('Skipping %s as it is not mounted'), device) continue while len(pids) >= self.concurrency: pids.remove(os.wait()[0]) pid = os.fork() if pid: pids.append(pid) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) eventlet_monkey_patch() self.successes = 0 self.failures = 0 forkbegin = time.time() self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - forkbegin self.logger.info( _('Object update sweep of %(device)s' ' completed: %(elapsed).02fs, %(success)s successes' ', %(fail)s failures'), { 'device': device, 'elapsed': elapsed, 'success': self.successes, 'fail': self.failures }) sys.exit() while pids: pids.remove(os.wait()[0]) elapsed = time.time() - begin self.logger.info(_('Object update sweep completed: %.02fs'), elapsed) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger) if elapsed < self.interval: time.sleep(self.interval - elapsed)
def GET(self, req): """Handle HTTP GET request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = constraints.CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.CONTAINER_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.CONTAINER_LISTING_LIMIT) out_content_type = listing_formats.get_listing_content_type(req) if not check_drive(self.root, drive, self.mount_check): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() resp_headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=resp_headers) container_list = broker.list_objects_iter( limit, marker, end_marker, prefix, delimiter, path, storage_policy_index=info['storage_policy_index'], reverse=reverse) return self.create_listing(req, out_content_type, info, resp_headers, broker.metadata, container_list, container)
def run_forever(self, *args, **kwargs): """Run the updater continuously.""" time.sleep(random() * self.interval) while True: self.logger.info(_('Begin object update sweep')) begin = time.time() pids = [] # read from container ring to ensure it's fresh self.get_container_ring().get_nodes('') for device in self._listdir(self.devices): if not check_drive(self.devices, device, self.mount_check): self.logger.increment('errors') self.logger.warning( _('Skipping %s as it is not mounted'), device) continue while len(pids) >= self.concurrency: pids.remove(os.wait()[0]) pid = os.fork() if pid: pids.append(pid) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) eventlet_monkey_patch() self.successes = 0 self.failures = 0 forkbegin = time.time() self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - forkbegin self.logger.info( _('Object update sweep of %(device)s' ' completed: %(elapsed).02fs, %(success)s successes' ', %(fail)s failures'), {'device': device, 'elapsed': elapsed, 'success': self.successes, 'fail': self.failures}) sys.exit() while pids: pids.remove(os.wait()[0]) elapsed = time.time() - begin self.logger.info(_('Object update sweep completed: %.02fs'), elapsed) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger) if elapsed < self.interval: time.sleep(self.interval - elapsed)
def run_forever(self, *args, **kwargs): """Run the updater continuously.""" time.sleep(random() * self.interval) while True: self.logger.info(_('Begin object update sweep')) begin = time.time() pids = [] # read from container ring to ensure it's fresh self.get_container_ring().get_nodes('') for device in self._listdir(self.devices): if not check_drive(self.devices, device, self.mount_check): # We don't count this as an error. The occasional # unmounted drive is part of normal cluster operations, # so a simple warning is sufficient. self.logger.warning( _('Skipping %s as it is not mounted'), device) continue while len(pids) >= self.concurrency: pids.remove(os.wait()[0]) pid = os.fork() if pid: pids.append(pid) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) eventlet_monkey_patch() self.stats.reset() forkbegin = time.time() self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - forkbegin self.logger.info( ('Object update sweep of %(device)s ' 'completed: %(elapsed).02fs, %(stats)s'), {'device': device, 'elapsed': elapsed, 'stats': self.stats}) sys.exit() while pids: pids.remove(os.wait()[0]) elapsed = time.time() - begin self.logger.info(_('Object update sweep completed: %.02fs'), elapsed) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger) if elapsed < self.interval: time.sleep(self.interval - elapsed)
def get_paths(self): """ Get paths to all of the partitions on each drive to be processed. :returns: a list of paths """ paths = [] for device in self._listdir(self.devices): dev_path = check_drive(self.devices, device, self.mount_check) if not dev_path: self.logger.warning(_('%s is not mounted'), device) continue con_path = os.path.join(dev_path, DATADIR) if not os.path.exists(con_path): continue for partition in self._listdir(con_path): paths.append(os.path.join(con_path, partition)) shuffle(paths) return paths
def run_once(self, *args, **kwargs): """Run a replication pass once.""" self._zero_stats() dirs = [] ips = whataremyips(self.bind_ip) if not ips: self.logger.error(_('ERROR Failed to get my own IPs?')) return self._local_device_ids = set() found_local = False for node in self.ring.devs: if node and is_local_device(ips, self.port, node['replication_ip'], node['replication_port']): found_local = True if not check_drive(self.root, node['device'], self.mount_check): self._add_failure_stats( [(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in self.ring.devs if failure_dev]) self.logger.warning( _('Skipping %(device)s as it is not mounted') % node) continue unlink_older_than( os.path.join(self.root, node['device'], 'tmp'), time.time() - self.reclaim_age) datadir = os.path.join(self.root, node['device'], self.datadir) if os.path.isdir(datadir): self._local_device_ids.add(node['id']) dirs.append((datadir, node['id'])) if not found_local: self.logger.error("Can't find itself %s with port %s in ring " "file, not replicating", ", ".join(ips), self.port) self.logger.info(_('Beginning replication run')) for part, object_file, node_id in roundrobin_datadirs(dirs): self.cpool.spawn_n( self._replicate_object, part, object_file, node_id) self.cpool.waitall() self.logger.info(_('Replication run OVER')) self._report_stats()
def run_once(self, *args, **kwargs): """Run the updater once.""" self.logger.info(_('Begin object update single threaded sweep')) begin = time.time() self.stats.reset() for device in self._listdir(self.devices): if not check_drive(self.devices, device, self.mount_check): # We don't count this as an error. The occasional unmounted # drive is part of normal cluster operations, so a simple # warning is sufficient. self.logger.warning( _('Skipping %s as it is not mounted'), device) continue self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - begin self.logger.info( ('Object update single-threaded sweep completed: ' '%(elapsed).02fs, %(stats)s'), {'elapsed': elapsed, 'stats': self.stats}) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger)
def run_once(self, *args, **kwargs): """Run the updater once.""" self.logger.info(_('Begin object update single threaded sweep')) begin = time.time() self.successes = 0 self.failures = 0 for device in self._listdir(self.devices): if not check_drive(self.devices, device, self.mount_check): self.logger.increment('errors') self.logger.warning( _('Skipping %s as it is not mounted'), device) continue self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - begin self.logger.info( _('Object update single threaded sweep completed: ' '%(elapsed).02fs, %(success)s successes, %(fail)s failures'), {'elapsed': elapsed, 'success': self.successes, 'fail': self.failures}) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger)
def dispatch(self, replicate_args, args): if not hasattr(args, 'pop'): return HTTPBadRequest(body='Invalid object type') op = args.pop(0) drive, partition, hsh = replicate_args if not check_drive(self.root, drive, self.mount_check): return Response(status='507 %s is not mounted' % drive) db_file = os.path.join(self.root, drive, storage_directory(self.datadir, partition, hsh), hsh + '.db') if op == 'rsync_then_merge': return self.rsync_then_merge(drive, db_file, args) if op == 'complete_rsync': return self.complete_rsync(drive, db_file, args) else: # someone might be about to rsync a db to us, # make sure there's a tmp dir to receive it. mkdirs(os.path.join(self.root, drive, 'tmp')) if not os.path.exists(db_file): return HTTPNotFound() return getattr(self, op)(self.broker_class(db_file), args)
def run_once(self, *args, **kwargs): """Run the updater once.""" self.logger.info(_('Begin object update single threaded sweep')) begin = time.time() self.successes = 0 self.failures = 0 for device in self._listdir(self.devices): if not check_drive(self.devices, device, self.mount_check): self.logger.increment('errors') self.logger.warning(_('Skipping %s as it is not mounted'), device) continue self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - begin self.logger.info( _('Object update single threaded sweep completed: ' '%(elapsed).02fs, %(success)s successes, %(fail)s failures'), { 'elapsed': elapsed, 'success': self.successes, 'fail': self.failures }) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger)
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) requested_policy_index = self.get_and_validate_policy_index(req) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object # obj put expects the policy_index header, default is for # legacy support during upgrade. obj_policy_index = requested_policy_index or 0 self._maybe_autocreate(broker, req_timestamp, account, obj_policy_index) # redirect if a shard exists for this object name response = self._redirect_to_shard(req, broker, obj) if response: return response broker.put_object(obj, req_timestamp.internal, int(req.headers['x-size']), wsgi_to_str(req.headers['x-content-type']), wsgi_to_str(req.headers['x-etag']), 0, obj_policy_index, wsgi_to_str(req.headers.get( 'x-content-type-timestamp')), wsgi_to_str(req.headers.get('x-meta-timestamp'))) return HTTPCreated(request=req) record_type = req.headers.get('x-backend-record-type', '').lower() if record_type == RECORD_TYPE_SHARD: try: # validate incoming data... shard_ranges = [ShardRange.from_dict(sr) for sr in json.loads(req.body)] except (ValueError, KeyError, TypeError) as err: return HTTPBadRequest('Invalid body: %r' % err) created = self._maybe_autocreate(broker, req_timestamp, account, requested_policy_index) self._update_metadata(req, broker, req_timestamp, 'PUT') if shard_ranges: # TODO: consider writing the shard ranges into the pending # file, but if so ensure an all-or-none semantic for the write broker.merge_shard_ranges(shard_ranges) else: # put container if requested_policy_index is None: # use the default index sent by the proxy if available new_container_policy = req.headers.get( 'X-Backend-Storage-Policy-Default', int(POLICIES.default)) else: new_container_policy = requested_policy_index created = self._update_or_create(req, broker, req_timestamp.internal, new_container_policy, requested_policy_index) self._update_metadata(req, broker, req_timestamp, 'PUT') resp = self.account_update(req, account, container, broker) if resp: return resp if created: return HTTPCreated(request=req, headers={'x-backend-storage-policy-index': broker.storage_policy_index}) else: return HTTPAccepted(request=req, headers={'x-backend-storage-policy-index': broker.storage_policy_index})
def run_once(self, *args, **kwargs): """Run a replication pass once.""" override_options = parse_override_options(once=True, **kwargs) devices_to_replicate = override_options.devices or Everything() partitions_to_replicate = override_options.partitions or Everything() self._zero_stats() dirs = [] ips = whataremyips(self.bind_ip) if not ips: self.logger.error(_('ERROR Failed to get my own IPs?')) return if self.handoffs_only: self.logger.warning( 'Starting replication pass with handoffs_only enabled. ' 'This mode is not intended for normal ' 'operation; use handoffs_only with care.') self._local_device_ids = set() found_local = False for node in self.ring.devs: if node and is_local_device(ips, self.port, node['replication_ip'], node['replication_port']): found_local = True try: dev_path = check_drive(self.root, node['device'], self.mount_check) except ValueError as err: self._add_failure_stats( [(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in self.ring.devs if failure_dev]) self.logger.warning('Skipping: %s', err) continue if node['device'] not in devices_to_replicate: self.logger.debug( 'Skipping device %s due to given arguments', node['device']) continue unlink_older_than( os.path.join(dev_path, 'tmp'), time.time() - self.reclaim_age) datadir = os.path.join(self.root, node['device'], self.datadir) if os.path.isdir(datadir): self._local_device_ids.add(node['id']) part_filt = self._partition_dir_filter( node['id'], partitions_to_replicate) dirs.append((datadir, node['id'], part_filt)) if not found_local: self.logger.error("Can't find itself %s with port %s in ring " "file, not replicating", ", ".join(ips), self.port) self.logger.info(_('Beginning replication run')) for part, object_file, node_id in self.roundrobin_datadirs(dirs): self.cpool.spawn_n( self._replicate_object, part, object_file, node_id) self.cpool.waitall() self.logger.info(_('Replication run OVER')) if self.handoffs_only: self.logger.warning( 'Finished replication pass with handoffs_only enabled. ' 'If handoffs_only is no longer required, disable it.') self._report_stats()
def replicate(self, override_devices=None, override_partitions=None, override_policies=None): """Run a replication pass""" self.start = time.time() self.suffix_count = 0 self.suffix_sync = 0 self.suffix_hash = 0 self.replication_count = 0 self.last_replication_count = -1 self.replication_cycle = (self.replication_cycle + 1) % 10 self.partition_times = [] self.my_replication_ips = self._get_my_replication_ips() self.all_devs_info = set() self.handoffs_remaining = 0 stats = eventlet.spawn(self.heartbeat) lockup_detector = eventlet.spawn(self.detect_lockups) eventlet.sleep() # Give spawns a cycle current_nodes = None try: self.run_pool = GreenPool(size=self.concurrency) jobs = self.collect_jobs(override_devices=override_devices, override_partitions=override_partitions, override_policies=override_policies) for job in jobs: current_nodes = job['nodes'] if override_devices and job['device'] not in override_devices: continue if override_partitions and \ job['partition'] not in override_partitions: continue dev_path = check_drive(self.devices_dir, job['device'], self.mount_check) if not dev_path: self._add_failure_stats([(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in job['nodes']]) self.logger.warning(_('%s is not mounted'), job['device']) continue if self.handoffs_first and not job['delete']: # in handoffs first mode, we won't process primary # partitions until rebalance was successful! if self.handoffs_remaining: self.logger.warning(_( "Handoffs first mode still has handoffs " "remaining. Aborting current " "replication pass.")) break if not self.check_ring(job['policy'].object_ring): self.logger.info(_("Ring change detected. Aborting " "current replication pass.")) return try: if isfile(job['path']): # Clean up any (probably zero-byte) files where a # partition should be. self.logger.warning( 'Removing partition directory ' 'which was a file: %s', job['path']) os.remove(job['path']) continue except OSError: continue if job['delete']: self.run_pool.spawn(self.update_deleted, job) else: self.run_pool.spawn(self.update, job) current_nodes = None with Timeout(self.lockup_timeout): self.run_pool.waitall() except (Exception, Timeout): if current_nodes: self._add_failure_stats([(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in current_nodes]) else: self._add_failure_stats(self.all_devs_info) self.logger.exception(_("Exception in top-level replication loop")) self.kill_coros() finally: stats.kill() lockup_detector.kill() self.stats_line() self.stats['attempted'] = self.replication_count
def build_replication_jobs(self, policy, ips, override_devices=None, override_partitions=None): """ Helper function for collect_jobs to build jobs for replication using replication style storage policy """ jobs = [] df_mgr = self._df_router[policy] self.all_devs_info.update( [(dev['replication_ip'], dev['device']) for dev in policy.object_ring.devs if dev]) data_dir = get_data_dir(policy) found_local = False for local_dev in [dev for dev in policy.object_ring.devs if (dev and is_local_device(ips, self.port, dev['replication_ip'], dev['replication_port']) and (override_devices is None or dev['device'] in override_devices))]: found_local = True dev_path = check_drive(self.devices_dir, local_dev['device'], self.mount_check) if not dev_path: self._add_failure_stats( [(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in policy.object_ring.devs if failure_dev]) self.logger.warning( _('%s is not mounted'), local_dev['device']) continue obj_path = join(dev_path, data_dir) tmp_path = join(dev_path, get_tmp_dir(policy)) unlink_older_than(tmp_path, time.time() - df_mgr.reclaim_age) if not os.path.exists(obj_path): try: mkdirs(obj_path) except Exception: self.logger.exception('ERROR creating %s' % obj_path) continue for partition in os.listdir(obj_path): if (override_partitions is not None and partition not in override_partitions): continue if (partition.startswith('auditor_status_') and partition.endswith('.json')): # ignore auditor status files continue part_nodes = None try: job_path = join(obj_path, partition) part_nodes = policy.object_ring.get_part_nodes( int(partition)) nodes = [node for node in part_nodes if node['id'] != local_dev['id']] jobs.append( dict(path=job_path, device=local_dev['device'], obj_path=obj_path, nodes=nodes, delete=len(nodes) > len(part_nodes) - 1, policy=policy, partition=partition, region=local_dev['region'])) except ValueError: if part_nodes: self._add_failure_stats( [(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in nodes]) else: self._add_failure_stats( [(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in policy.object_ring.devs if failure_dev]) continue if not found_local: self.logger.error("Can't find itself in policy with index %d with" " ips %s and with port %s in ring file, not" " replicating", int(policy), ", ".join(ips), self.port) return jobs
def replicate(self, override_devices=None, override_partitions=None, override_policies=None, start_time=None): """Run a replication pass""" if start_time is None: start_time = time.time() self.start = start_time self.last_replication_count = 0 self.replication_cycle = (self.replication_cycle + 1) % 10 self.partition_times = [] self.my_replication_ips = self._get_my_replication_ips() self.all_devs_info = set() self.handoffs_remaining = 0 stats = eventlet.spawn(self.heartbeat) eventlet.sleep() # Give spawns a cycle current_nodes = None dev_stats = None num_jobs = 0 try: self.run_pool = GreenPool(size=self.concurrency) jobs = self.collect_jobs(override_devices=override_devices, override_partitions=override_partitions, override_policies=override_policies) for job in jobs: dev_stats = self.stats_for_dev[job['device']] num_jobs += 1 current_nodes = job['nodes'] dev_path = check_drive(self.devices_dir, job['device'], self.mount_check) if not dev_path: dev_stats.add_failure_stats([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in job['nodes'] ]) self.logger.warning(_('%s is not mounted'), job['device']) continue if self.handoffs_first and not job['delete']: # in handoffs first mode, we won't process primary # partitions until rebalance was successful! if self.handoffs_remaining: self.logger.warning( _("Handoffs first mode still has handoffs " "remaining. Aborting current " "replication pass.")) break if not self.check_ring(job['policy'].object_ring): self.logger.info( _("Ring change detected. Aborting " "current replication pass.")) return try: if isfile(job['path']): # Clean up any (probably zero-byte) files where a # partition should be. self.logger.warning( 'Removing partition directory ' 'which was a file: %s', job['path']) os.remove(job['path']) continue except OSError: continue if job['delete']: self.run_pool.spawn(self.update_deleted, job) else: self.run_pool.spawn(self.update, job) current_nodes = None self.run_pool.waitall() except (Exception, Timeout) as err: if dev_stats: if current_nodes: dev_stats.add_failure_stats([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in current_nodes ]) else: dev_stats.add_failure_stats(self.all_devs_info) self.logger.exception( _("Exception in top-level replication loop: %s"), err) finally: stats.kill() self.stats_line()
def build_replication_jobs(self, policy, ips, override_devices=None, override_partitions=None): """ Helper function for collect_jobs to build jobs for replication using replication style storage policy """ jobs = [] df_mgr = self._df_router[policy] self.all_devs_info.update([(dev['replication_ip'], dev['device']) for dev in policy.object_ring.devs if dev]) data_dir = get_data_dir(policy) found_local = False for local_dev in [ dev for dev in policy.object_ring.devs if (dev and is_local_device(ips, self.port, dev['replication_ip'], dev['replication_port']) and (override_devices is None or dev['device'] in override_devices)) ]: found_local = True dev_path = check_drive(self.devices_dir, local_dev['device'], self.mount_check) local_dev_stats = self.stats_for_dev[local_dev['device']] if not dev_path: local_dev_stats.add_failure_stats([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in policy.object_ring.devs if failure_dev ]) self.logger.warning(_('%s is not mounted'), local_dev['device']) continue obj_path = join(dev_path, data_dir) tmp_path = join(dev_path, get_tmp_dir(policy)) unlink_older_than(tmp_path, time.time() - df_mgr.reclaim_age) if not os.path.exists(obj_path): try: mkdirs(obj_path) except Exception: self.logger.exception('ERROR creating %s' % obj_path) continue for partition in os.listdir(obj_path): if (override_partitions is not None and partition not in override_partitions): continue if (partition.startswith('auditor_status_') and partition.endswith('.json')): # ignore auditor status files continue part_nodes = None try: job_path = join(obj_path, partition) part_nodes = policy.object_ring.get_part_nodes( int(partition)) nodes = [ node for node in part_nodes if node['id'] != local_dev['id'] ] jobs.append( dict(path=job_path, device=local_dev['device'], obj_path=obj_path, nodes=nodes, delete=len(nodes) > len(part_nodes) - 1, policy=policy, partition=partition, region=local_dev['region'])) except ValueError: if part_nodes: local_dev_stats.add_failure_stats([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in nodes ]) else: local_dev_stats.add_failure_stats([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in policy.object_ring.devs if failure_dev ]) continue if not found_local: self.logger.error( "Can't find itself in policy with index %d with" " ips %s and with port %s in ring file, not" " replicating", int(policy), ", ".join(ips), self.port) return jobs
def GET(self, req): """ Handle HTTP GET request. The body of the response to a successful GET request contains a listing of either objects or shard ranges. The exact content of the listing is determined by a combination of request headers and query string parameters, as follows: * The type of the listing is determined by the ``X-Backend-Record-Type`` header. If this header has value ``shard`` then the response body will be a list of shard ranges; if this header has value ``auto``, and the container state is ``sharding`` or ``sharded``, then the listing will be a list of shard ranges; otherwise the response body will be a list of objects. * Both shard range and object listings may be constrained to a name range by the ``marker`` and ``end_marker`` query string parameters. Object listings will only contain objects whose names are greater than any ``marker`` value and less than any ``end_marker`` value. Shard range listings will only contain shard ranges whose namespace is greater than or includes any ``marker`` value and is less than or includes any ``end_marker`` value. * Shard range listings may also be constrained by an ``includes`` query string parameter. If this parameter is present the listing will only contain shard ranges whose namespace includes the value of the parameter; any ``marker`` or ``end_marker`` parameters are ignored * The length of an object listing may be constrained by the ``limit`` parameter. Object listings may also be constrained by ``prefix``, ``delimiter`` and ``path`` query string parameters. * Shard range listings will include deleted shard ranges if and only if the ``X-Backend-Include-Deleted`` header value is one of :attr:`swift.common.utils.TRUE_VALUES`. Object listings never include deleted objects. * Shard range listings may be constrained to include only shard ranges whose state is specified by a query string ``states`` parameter. If present, the ``states`` parameter should be a comma separated list of either the string or integer representation of :data:`~swift.common.utils.ShardRange.STATES`. Two alias values may be used in a ``states`` parameter value: ``listing`` will cause the listing to include all shard ranges in a state suitable for contributing to an object listing; ``updating`` will cause the listing to include all shard ranges in a state suitable to accept an object update. If either of these aliases is used then the shard range listing will if necessary be extended with a synthesised 'filler' range in order to satisfy the requested name range when insufficient actual shard ranges are found. Any 'filler' shard range will cover the otherwise uncovered tail of the requested name range and will point back to the same container. * Listings are not normally returned from a deleted container. However, the ``X-Backend-Override-Deleted`` header may be used with a value in :attr:`swift.common.utils.TRUE_VALUES` to force a shard range listing to be returned from a deleted container whose DB file still exists. :param req: an instance of :class:`swift.common.swob.Request` :returns: an instance of :class:`swift.common.swob.Response` """ drive, part, account, container, obj = get_obj_name_and_placement(req) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = constraints.CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.CONTAINER_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.CONTAINER_LISTING_LIMIT) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() record_type = req.headers.get('x-backend-record-type', '').lower() if record_type == 'auto' and info.get('db_state') in (SHARDING, SHARDED): record_type = 'shard' if record_type == 'shard': override_deleted = info and config_true_value( req.headers.get('x-backend-override-deleted', False)) resp_headers = gen_resp_headers(info, is_deleted=is_deleted and not override_deleted) if is_deleted and not override_deleted: return HTTPNotFound(request=req, headers=resp_headers) resp_headers['X-Backend-Record-Type'] = 'shard' includes = get_param(req, 'includes') states = get_param(req, 'states') fill_gaps = False if states: states = list_from_csv(states) fill_gaps = any(('listing' in states, 'updating' in states)) try: states = broker.resolve_shard_range_states(states) except ValueError: return HTTPBadRequest(request=req, body='Bad state') include_deleted = config_true_value( req.headers.get('x-backend-include-deleted', False)) container_list = broker.get_shard_ranges( marker, end_marker, includes, reverse, states=states, include_deleted=include_deleted, fill_gaps=fill_gaps) else: resp_headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=resp_headers) resp_headers['X-Backend-Record-Type'] = 'object' # Use the retired db while container is in process of sharding, # otherwise use current db src_broker = broker.get_brokers()[0] container_list = src_broker.list_objects_iter( limit, marker, end_marker, prefix, delimiter, path, storage_policy_index=info['storage_policy_index'], reverse=reverse, allow_reserved=req.allow_reserved_names) return self.create_listing(req, out_content_type, info, resp_headers, broker.metadata, container_list, container)
def replicate(self, override_devices=None, override_partitions=None, override_policies=None, start_time=None): """Run a replication pass""" if start_time is None: start_time = time.time() self.start = start_time self.last_replication_count = 0 self.replication_cycle = (self.replication_cycle + 1) % 10 self.partition_times = [] self.my_replication_ips = self._get_my_replication_ips() self.all_devs_info = set() self.handoffs_remaining = 0 stats = eventlet.spawn(self.heartbeat) eventlet.sleep() # Give spawns a cycle current_nodes = None dev_stats = None num_jobs = 0 try: self.run_pool = GreenPool(size=self.concurrency) jobs = self.collect_jobs(override_devices=override_devices, override_partitions=override_partitions, override_policies=override_policies) for job in jobs: dev_stats = self.stats_for_dev[job['device']] num_jobs += 1 current_nodes = job['nodes'] try: check_drive(self.devices_dir, job['device'], self.mount_check) except ValueError as err: dev_stats.add_failure_stats([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in job['nodes']]) self.logger.warning("%s", err) continue if self.handoffs_first and not job['delete']: # in handoffs first mode, we won't process primary # partitions until rebalance was successful! if self.handoffs_remaining: self.logger.warning(_( "Handoffs first mode still has handoffs " "remaining. Aborting current " "replication pass.")) break if not self.check_ring(job['policy'].object_ring): self.logger.info(_("Ring change detected. Aborting " "current replication pass.")) return try: if isfile(job['path']): # Clean up any (probably zero-byte) files where a # partition should be. self.logger.warning( 'Removing partition directory ' 'which was a file: %s', job['path']) os.remove(job['path']) continue except OSError: continue if job['delete']: self.run_pool.spawn(self.update_deleted, job) else: self.run_pool.spawn(self.update, job) current_nodes = None self.run_pool.waitall() except (Exception, Timeout) as err: if dev_stats: if current_nodes: dev_stats.add_failure_stats( [(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in current_nodes]) else: dev_stats.add_failure_stats(self.all_devs_info) self.logger.exception( _("Exception in top-level replication loop: %s"), err) finally: stats.kill() self.stats_line()
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container, obj = get_obj_name_and_placement(req) req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) requested_policy_index = self.get_and_validate_policy_index(req) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object # obj put expects the policy_index header, default is for # legacy support during upgrade. obj_policy_index = requested_policy_index or 0 self._maybe_autocreate(broker, req_timestamp, account, obj_policy_index) # redirect if a shard exists for this object name response = self._redirect_to_shard(req, broker, obj) if response: return response broker.put_object( obj, req_timestamp.internal, int(req.headers['x-size']), wsgi_to_str(req.headers['x-content-type']), wsgi_to_str(req.headers['x-etag']), 0, obj_policy_index, wsgi_to_str(req.headers.get('x-content-type-timestamp')), wsgi_to_str(req.headers.get('x-meta-timestamp'))) return HTTPCreated(request=req) record_type = req.headers.get('x-backend-record-type', '').lower() if record_type == RECORD_TYPE_SHARD: try: # validate incoming data... shard_ranges = [ ShardRange.from_dict(sr) for sr in json.loads(req.body) ] except (ValueError, KeyError, TypeError) as err: return HTTPBadRequest('Invalid body: %r' % err) created = self._maybe_autocreate(broker, req_timestamp, account, requested_policy_index) self._update_metadata(req, broker, req_timestamp, 'PUT') if shard_ranges: # TODO: consider writing the shard ranges into the pending # file, but if so ensure an all-or-none semantic for the write broker.merge_shard_ranges(shard_ranges) else: # put container if requested_policy_index is None: # use the default index sent by the proxy if available new_container_policy = req.headers.get( 'X-Backend-Storage-Policy-Default', int(POLICIES.default)) else: new_container_policy = requested_policy_index created = self._update_or_create(req, broker, req_timestamp.internal, new_container_policy, requested_policy_index) self._update_metadata(req, broker, req_timestamp, 'PUT') resp = self.account_update(req, account, container, broker) if resp: return resp if created: return HTTPCreated(request=req, headers={ 'x-backend-storage-policy-index': broker.storage_policy_index }) else: return HTTPAccepted(request=req, headers={ 'x-backend-storage-policy-index': broker.storage_policy_index })
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = split_and_validate_path(req, 3, 4) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container if 'x-timestamp' not in req.headers: timestamp = Timestamp.now() else: timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = \ req.headers.get('X-Backend-Storage-Policy-Index', 0) if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used'], container_policy_index) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} if six.PY2: metadata.update((key, (value, timestamp.internal)) for key, value in req.headers.items() if is_sys_or_user_meta('account', key)) else: for key, value in req.headers.items(): if is_sys_or_user_meta('account', key): # Cast to native strings, so that json inside # updata_metadata eats the data. try: value = value.encode('latin-1').decode('utf-8') except UnicodeDecodeError: raise HTTPBadRequest( 'Metadata must be valid UTF-8') metadata[key] = (value, timestamp.internal) if metadata: broker.update_metadata(metadata, validate_metadata=True) if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req)
def GET(self, req): """ Handle HTTP GET request. The body of the response to a successful GET request contains a listing of either objects or shard ranges. The exact content of the listing is determined by a combination of request headers and query string parameters, as follows: * The type of the listing is determined by the ``X-Backend-Record-Type`` header. If this header has value ``shard`` then the response body will be a list of shard ranges; if this header has value ``auto``, and the container state is ``sharding`` or ``sharded``, then the listing will be a list of shard ranges; otherwise the response body will be a list of objects. * Both shard range and object listings may be constrained to a name range by the ``marker`` and ``end_marker`` query string parameters. Object listings will only contain objects whose names are greater than any ``marker`` value and less than any ``end_marker`` value. Shard range listings will only contain shard ranges whose namespace is greater than or includes any ``marker`` value and is less than or includes any ``end_marker`` value. * Shard range listings may also be constrained by an ``includes`` query string parameter. If this parameter is present the listing will only contain shard ranges whose namespace includes the value of the parameter; any ``marker`` or ``end_marker`` parameters are ignored * The length of an object listing may be constrained by the ``limit`` parameter. Object listings may also be constrained by ``prefix``, ``delimiter`` and ``path`` query string parameters. * Shard range listings will include deleted shard ranges if and only if the ``X-Backend-Include-Deleted`` header value is one of :attr:`swift.common.utils.TRUE_VALUES`. Object listings never include deleted objects. * Shard range listings may be constrained to include only shard ranges whose state is specified by a query string ``states`` parameter. If present, the ``states`` parameter should be a comma separated list of either the string or integer representation of :data:`~swift.common.utils.ShardRange.STATES`. Two alias values may be used in a ``states`` parameter value: ``listing`` will cause the listing to include all shard ranges in a state suitable for contributing to an object listing; ``updating`` will cause the listing to include all shard ranges in a state suitable to accept an object update. If either of these aliases is used then the shard range listing will if necessary be extended with a synthesised 'filler' range in order to satisfy the requested name range when insufficient actual shard ranges are found. Any 'filler' shard range will cover the otherwise uncovered tail of the requested name range and will point back to the same container. * Listings are not normally returned from a deleted container. However, the ``X-Backend-Override-Deleted`` header may be used with a value in :attr:`swift.common.utils.TRUE_VALUES` to force a shard range listing to be returned from a deleted container whose DB file still exists. :param req: an instance of :class:`swift.common.swob.Request` :returns: an instance of :class:`swift.common.swob.Response` """ drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = constraints.CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.CONTAINER_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.CONTAINER_LISTING_LIMIT) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() record_type = req.headers.get('x-backend-record-type', '').lower() if record_type == 'auto' and info.get('db_state') in (SHARDING, SHARDED): record_type = 'shard' if record_type == 'shard': override_deleted = info and config_true_value( req.headers.get('x-backend-override-deleted', False)) resp_headers = gen_resp_headers( info, is_deleted=is_deleted and not override_deleted) if is_deleted and not override_deleted: return HTTPNotFound(request=req, headers=resp_headers) resp_headers['X-Backend-Record-Type'] = 'shard' includes = get_param(req, 'includes') states = get_param(req, 'states') fill_gaps = False if states: states = list_from_csv(states) fill_gaps = any(('listing' in states, 'updating' in states)) try: states = broker.resolve_shard_range_states(states) except ValueError: return HTTPBadRequest(request=req, body='Bad state') include_deleted = config_true_value( req.headers.get('x-backend-include-deleted', False)) container_list = broker.get_shard_ranges( marker, end_marker, includes, reverse, states=states, include_deleted=include_deleted, fill_gaps=fill_gaps) else: resp_headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=resp_headers) resp_headers['X-Backend-Record-Type'] = 'object' # Use the retired db while container is in process of sharding, # otherwise use current db src_broker = broker.get_brokers()[0] container_list = src_broker.list_objects_iter( limit, marker, end_marker, prefix, delimiter, path, storage_policy_index=info['storage_policy_index'], reverse=reverse) return self.create_listing(req, out_content_type, info, resp_headers, broker.metadata, container_list, container)
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = split_and_validate_path(req, 3, 4) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container if 'x-timestamp' not in req.headers: timestamp = Timestamp.now() else: timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = \ req.headers.get('X-Backend-Storage-Policy-Index', 0) if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used'], container_policy_index) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) self._update_metadata(req, broker, timestamp) if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req)