def run_quarantine_range_etag(self): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'RANGE') # Stash the on disk data for future comparison - this may not equal # 'VERIFY' if for example the proxy has crypto enabled backend_data = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) base_headers = {'X-Backend-Storage-Policy-Index': self.policy.idx} for header, result in [({'Range': 'bytes=0-2'}, backend_data[0:3]), ({'Range': 'bytes=1-11'}, backend_data[1:]), ({'Range': 'bytes=0-11'}, backend_data)]: req_headers = base_headers.copy() req_headers.update(header) odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers=req_headers)[-1] self.assertEqual(odata, result) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEqual(err.http_status, 404)
def run_quarantine_range_etag(self): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'RANGE') metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) base_headers = {'X-Backend-Storage-Policy-Index': self.policy.idx} for header, result in [({'Range': 'bytes=0-2'}, 'RAN'), ({'Range': 'bytes=1-11'}, 'ANGE'), ({'Range': 'bytes=0-11'}, 'RANGE')]: req_headers = base_headers.copy() req_headers.update(header) odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers=req_headers)[-1] self.assertEqual(odata, result) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEqual(err.http_status, 404)
def run_quarantine(self): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'VERIFY') # Stash the on disk data for future comparison - this may not equal # 'VERIFY' if for example the proxy has crypto enabled backend_data = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] self.assertEqual(odata, backend_data) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEqual(err.http_status, 404)
def run_quarantine_range_etag(self): container = "container-range-%s" % uuid4() obj = "object-range-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "RANGE") metadata = read_metadata(data_file) metadata["ETag"] = "badetag" write_metadata(data_file, metadata) base_headers = {"X-Backend-Storage-Policy-Index": self.policy.idx} for header, result in [ ({"Range": "bytes=0-2"}, "RAN"), ({"Range": "bytes=1-11"}, "ANGE"), ({"Range": "bytes=0-11"}, "RANGE"), ]: req_headers = base_headers.copy() req_headers.update(header) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=req_headers)[-1] self.assertEquals(odata, result) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={"X-Backend-Storage-Policy-Index": self.policy.idx} ) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404)
def object_audit(self, path, device, partition): """ Audits the given object path. :param path: a path to an object :param device: the device the path is on :param partition: the partition the path is on """ try: try: name = diskfile.read_metadata(path)['name'] except (Exception, Timeout) as exc: raise AuditException('Error when reading metadata: %s' % exc) _junk, account, container, obj = name.split('/', 3) df = diskfile.DiskFile(self.devices, device, partition, account, container, obj, self.logger, keep_data_fp=True) try: try: obj_size = df.get_data_file_size() except DiskFileNotExist: return except DiskFileError as e: raise AuditException(str(e)) if self.stats_sizes: self.record_stats(obj_size) if self.zero_byte_only_at_fps and obj_size: self.passes += 1 return for chunk in df: self.bytes_running_time = ratelimit_sleep( self.bytes_running_time, self.max_bytes_per_second, incr_by=len(chunk)) self.bytes_processed += len(chunk) self.total_bytes_processed += len(chunk) df.close() if df.quarantined_dir: self.quarantines += 1 self.logger.error( _("ERROR Object %(path)s failed audit and will be " "quarantined: ETag and file's md5 do not match"), {'path': path}) finally: df.close(verify_file=False) except AuditException as err: self.logger.increment('quarantines') self.quarantines += 1 self.logger.error(_('ERROR Object %(obj)s failed audit and will ' 'be quarantined: %(err)s'), {'obj': path, 'err': err}) diskfile.quarantine_renamer( os.path.join(self.devices, device), path) return except (Exception, Timeout): self.logger.increment('errors') self.errors += 1 self.logger.exception(_('ERROR Trying to audit %s'), path) return self.passes += 1
def object_audit(self, path, device, partition): """ Audits the given object path. :param path: a path to an object :param device: the device the path is on :param partition: the partition the path is on """ try: try: name = diskfile.read_metadata(path)['name'] except (Exception, Timeout) as exc: raise AuditException('Error when reading metadata: %s' % exc) _junk, account, container, obj = name.split('/', 3) df = self.diskfile_mgr.get_diskfile( device, partition, account, container, obj) try: with df.open(): metadata = df.get_metadata() obj_size = int(metadata['Content-Length']) if self.stats_sizes: self.record_stats(obj_size) if self.zero_byte_only_at_fps and obj_size: self.passes += 1 return reader = df.reader() with closing(reader): for chunk in reader: chunk_len = len(chunk) self.bytes_running_time = ratelimit_sleep( self.bytes_running_time, self.max_bytes_per_second, incr_by=chunk_len) self.bytes_processed += chunk_len self.total_bytes_processed += chunk_len if reader.was_quarantined: self.quarantines += 1 self.logger.error(_('ERROR Object %(obj)s failed audit and' ' was quarantined: %(err)s'), {'obj': path, 'err': reader.was_quarantined}) return except DiskFileNotExist: return except DiskFileQuarantined as err: self.quarantines += 1 self.logger.error(_('ERROR Object %(obj)s failed audit and was' ' quarantined: %(err)s'), {'obj': path, 'err': err}) except AuditException as err: self.logger.increment('quarantines') self.quarantines += 1 self.logger.error(_('ERROR Object %(obj)s failed audit and will' ' be quarantined: %(err)s'), {'obj': path, 'err': err}) diskfile.quarantine_renamer( os.path.join(self.devices, device), path) return self.passes += 1
def run_quarantine_zero_byte_head(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_head_object(onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") except client.ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_get(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEqual(err.http_status, 404)
def run_quarantine(self): container = "container-%s" % uuid4() obj = "object-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "VERIFY") with open(data_file) as fpointer: metadata = read_metadata(fpointer) metadata["ETag"] = "badetag" with open(data_file) as fpointer: write_metadata(fpointer, metadata) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] self.assertEquals(odata, "VERIFY") try: direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") except client.ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine(self): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'VERIFY') metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) odata = direct_client.direct_get_object( onode, opart, self.account, container, obj)[-1] self.assertEquals(odata, 'VERIFY') try: direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") except client.ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine(self): container = "container-%s" % uuid4() obj = "object-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "VERIFY") metadata = read_metadata(data_file) metadata["ETag"] = "badetag" write_metadata(data_file, metadata) odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={"X-Backend-Storage-Policy-Index": self.policy.idx} )[-1] self.assertEquals(odata, "VERIFY") try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={"X-Backend-Storage-Policy-Index": self.policy.idx} ) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_post(self): container = "container-zbyte-%s" % uuid4() obj = "object-zbyte-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "DATA") metadata = read_metadata(data_file) unlink(data_file) with open(data_file, "w") as fpointer: write_metadata(fpointer, metadata) try: headers = { "X-Object-Meta-1": "One", "X-Object-Meta-Two": "Two", "X-Backend-Storage-Policy-Index": self.policy.idx, } direct_client.direct_post_object( onode, opart, self.account, container, obj, headers=headers, conn_timeout=1, response_timeout=1 ) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_post(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: headers = {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two', POLICY_INDEX: self.policy.idx} direct_client.direct_post_object( onode, opart, self.account, container, obj, headers=headers, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_head(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_head_object( onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_range_etag(self): container = "container-range-%s" % uuid4() obj = "object-range-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "RANGE") with open(data_file) as fpointer: metadata = read_metadata(fpointer) metadata["ETag"] = "badetag" with open(data_file) as fpointer: write_metadata(fpointer, metadata) for header, result in [ ({"Range": "bytes=0-2"}, "RAN"), ({"Range": "bytes=1-11"}, "ANGE"), ({"Range": "bytes=0-11"}, "RANGE"), ]: odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=header)[-1] self.assertEquals(odata, result) try: direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") except client.ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_range_etag(self): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'RANGE') metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) for header, result in [({'Range': 'bytes=0-2'}, 'RAN'), ({'Range': 'bytes=1-11'}, 'ANGE'), ({'Range': 'bytes=0-11'}, 'RANGE')]: odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers=header)[-1] self.assertEquals(odata, result) try: direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") except client.ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_range_etag(self): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file( container, obj, 'RANGE') metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) base_headers = {'X-Backend-Storage-Policy-Index': self.policy.idx} for header, result in [({ 'Range': 'bytes=0-2' }, 'RAN'), ({ 'Range': 'bytes=1-11' }, 'ANGE'), ({ 'Range': 'bytes=0-11' }, 'RANGE')]: req_headers = base_headers.copy() req_headers.update(header) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=req_headers)[-1] self.assertEquals(odata, result) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine(self): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() onode, opart, data_file = self._setup_data_file( container, obj, 'VERIFY') # Stash the on disk data for future comparison - this may not equal # 'VERIFY' if for example the proxy has crypto enabled backend_data = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] self.assertEqual(odata, backend_data) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) raise Exception("Did not quarantine object") except ClientException as err: self.assertEqual(err.http_status, 404)
def run_quarantine_zero_byte_post(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_post_object(onode, opart, self.account, container, obj, { 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two' }, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") except client.ClientException as err: self.assertEquals(err.http_status, 404)
def run_quarantine_range_etag(self): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'RANGE') with open(data_file) as fpointer: metadata = read_metadata(fpointer) metadata['ETag'] = 'badetag' with open(data_file) as fpointer: write_metadata(fpointer, metadata) for header, result in [({'Range': 'bytes=0-2'}, 'RAN'), ({'Range': 'bytes=1-11'}, 'ANGE'), ({'Range': 'bytes=0-11'}, 'RANGE')]: odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers=header)[-1] self.assertEquals(odata, result) try: direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") except client.ClientException, err: self.assertEquals(err.http_status, 404)
def run_quarantine_zero_byte_post(self): container = "container-zbyte-%s" % uuid4() obj = "object-zbyte-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "DATA") with open(data_file) as fpointer: metadata = read_metadata(fpointer) unlink(data_file) with open(data_file, "w") as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_post_object( onode, opart, self.account, container, obj, {"X-Object-Meta-1": "One", "X-Object-Meta-Two": "Two"}, conn_timeout=1, response_timeout=1, ) raise Exception("Did not quarantine object") except client.ClientException as err: self.assertEquals(err.http_status, 404)
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', policy_name=''): """ Display information about an object read from the datafile. Optionally verify the datafile content matches the ETag metadata. :param datafile: path on disk to object file :param check_etag: boolean, will read datafile content and verify computed checksum matches value stored in metadata. :param swift_dir: the path on disk to rings :param policy_name: optionally the name to use when finding the ring """ if not os.path.exists(datafile): print "Data file doesn't exist" raise InfoSystemExit() if not datafile.startswith(('/', './')): datafile = './' + datafile policy_index = None ring = None datadir = DATADIR_BASE # try to extract policy index from datafile disk path policy_index = int(extract_policy(datafile) or POLICIES.legacy) try: if policy_index: datadir += '-' + str(policy_index) ring = Ring(swift_dir, ring_name='object-' + str(policy_index)) elif policy_index == 0: ring = Ring(swift_dir, ring_name='object') except IOError: # no such ring pass if policy_name: policy = POLICIES.get_by_name(policy_name) if policy: policy_index_for_name = policy.idx if (policy_index is not None and policy_index_for_name is not None and policy_index != policy_index_for_name): print 'Attention: Ring does not match policy!' print 'Double check your policy name!' if not ring and policy_index_for_name: ring = POLICIES.get_object_ring(policy_index_for_name, swift_dir) datadir = get_data_dir(policy_index_for_name) with open(datafile, 'rb') as fp: try: metadata = read_metadata(fp) except EOFError: print "Invalid metadata" raise InfoSystemExit() etag = metadata.pop('ETag', '') length = metadata.pop('Content-Length', '') path = metadata.get('name', '') print_obj_metadata(metadata) # Optional integrity check; it's useful, but slow. file_len = None if check_etag: h = md5() file_len = 0 while True: data = fp.read(64 * 1024) if not data: break h.update(data) file_len += len(data) h = h.hexdigest() if etag: if h == etag: print 'ETag: %s (valid)' % etag else: print ("ETag: %s doesn't match file hash of %s!" % (etag, h)) else: print 'ETag: Not found in metadata' else: print 'ETag: %s (not checked)' % etag file_len = os.fstat(fp.fileno()).st_size if length: if file_len == int(length): print 'Content-Length: %s (valid)' % length else: print ("Content-Length: %s doesn't match file length of %s" % (length, file_len)) else: print 'Content-Length: Not found in metadata' account, container, obj = path.split('/', 3)[1:] if ring: print_ring_locations(ring, datadir, account, container, obj, policy_index=policy_index)
def object_audit(self, path, device, partition): """ Audits the given object path. :param path: a path to an object :param device: the device the path is on :param partition: the partition the path is on """ try: try: name = diskfile.read_metadata(path)['name'] except (Exception, Timeout) as exc: raise AuditException('Error when reading metadata: %s' % exc) _junk, account, container, obj = name.split('/', 3) df = self.diskfile_mgr.get_diskfile(device, partition, account, container, obj) try: with df.open(): metadata = df.get_metadata() obj_size = int(metadata['Content-Length']) if self.stats_sizes: self.record_stats(obj_size) if self.zero_byte_only_at_fps and obj_size: self.passes += 1 return reader = df.reader() with closing(reader): for chunk in reader: chunk_len = len(chunk) self.bytes_running_time = ratelimit_sleep( self.bytes_running_time, self.max_bytes_per_second, incr_by=chunk_len) self.bytes_processed += chunk_len self.total_bytes_processed += chunk_len if reader.was_quarantined: self.quarantines += 1 self.logger.error( _('ERROR Object %(obj)s failed audit and' ' was quarantined: %(err)s'), { 'obj': path, 'err': reader.was_quarantined }) return except DiskFileNotExist: return except DiskFileQuarantined as err: self.quarantines += 1 self.logger.error( _('ERROR Object %(obj)s failed audit and was' ' quarantined: %(err)s'), { 'obj': path, 'err': err }) except AuditException as err: self.logger.increment('quarantines') self.quarantines += 1 self.logger.error( _('ERROR Object %(obj)s failed audit and will' ' be quarantined: %(err)s'), { 'obj': path, 'err': err }) diskfile.quarantine_renamer(os.path.join(self.devices, device), path) return self.passes += 1
def object_audit(self, path, device, partition): """ Audits the given object path. :param path: a path to an object :param device: the device the path is on :param partition: the partition the path is on """ try: try: name = diskfile.read_metadata(path)['name'] except (Exception, Timeout) as exc: raise AuditException('Error when reading metadata: %s' % exc) _junk, account, container, obj = name.split('/', 3) df = diskfile.DiskFile(self.devices, device, partition, account, container, obj, self.logger, keep_data_fp=True) try: try: obj_size = df.get_data_file_size() except DiskFileNotExist: return except DiskFileError as e: raise AuditException(str(e)) if self.stats_sizes: self.record_stats(obj_size) if self.zero_byte_only_at_fps and obj_size: self.passes += 1 return for chunk in df: self.bytes_running_time = ratelimit_sleep( self.bytes_running_time, self.max_bytes_per_second, incr_by=len(chunk)) self.bytes_processed += len(chunk) self.total_bytes_processed += len(chunk) df.close() if df.quarantined_dir: self.quarantines += 1 self.logger.error( _("ERROR Object %(path)s failed audit and will be " "quarantined: ETag and file's md5 do not match"), {'path': path}) finally: df.close(verify_file=False) except AuditException as err: self.logger.increment('quarantines') self.quarantines += 1 self.logger.error( _('ERROR Object %(obj)s failed audit and will ' 'be quarantined: %(err)s'), { 'obj': path, 'err': err }) diskfile.quarantine_renamer(os.path.join(self.devices, device), path) return except (Exception, Timeout): self.logger.increment('errors') self.errors += 1 self.logger.exception(_('ERROR Trying to audit %s'), path) return self.passes += 1