def run_quarantine_range_etag(self): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file( container, obj, 'RANGE') with open(data_file) as fp: metadata = read_metadata(fp) metadata['ETag'] = 'badetag' with open(data_file) as fp: write_metadata(fp, metadata) for header, result in [({ 'Range': 'bytes=0-2' }, 'RAN'), ({ 'Range': 'bytes=1-11' }, 'ANGE'), ({ 'Range': 'bytes=0-11' }, 'RANGE')]: odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=header)[-1] self.assertEquals(odata, result) try: resp = direct_client.direct_get_object(onode, opart, self.account, container, obj) raise "Did not quarantine object" except client.ClientException, e: self.assertEquals(e.http_status, 404)
def run_quarantine_zero_byte_post(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') with open(data_file) as fp: metadata = read_metadata(fp) os.unlink(data_file) with open(data_file, 'w') as fp: write_metadata(fp, metadata) try: resp = direct_client.direct_post_object( onode, opart, self.account, container, obj, { 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two' }, conn_timeout=1, response_timeout=1) raise "Did not quarantine object" except client.ClientException, e: self.assertEquals(e.http_status, 404)
def test_object_audit_diff_data(self): self.auditor = auditor.AuditorWorker(self.conf) data = '0' * 1024 etag = md5() timestamp = str(normalize_timestamp(time.time())) with self.disk_file.mkstemp() as (fd, tmppath): os.write(fd, data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(fd).st_size), } self.disk_file.put(fd, tmppath, metadata) pre_quarantines = self.auditor.quarantines # remake so it will have metadata self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', self.logger) self.auditor.object_audit( os.path.join(self.disk_file.datadir, timestamp + '.data'), 'sda', '0') self.assertEquals(self.auditor.quarantines, pre_quarantines) etag = md5() etag.update('1' + '0' * 1023) etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(fd, metadata) self.auditor.object_audit( os.path.join(self.disk_file.datadir, timestamp + '.data'), 'sda', '0') self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
def setup_bad_zero_byte(self, with_ts=False): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 ts_file_path = '' if with_ts: name_hash = hash_path('a', 'c', 'o') dir_path = os.path.join(self.devices, 'sda', storage_directory(DATADIR, '0', name_hash)) ts_file_path = os.path.join(dir_path, '99999.ts') if not os.path.exists(dir_path): mkdirs(dir_path) fp = open(ts_file_path, 'w') fp.close() etag = md5() with self.disk_file.mkstemp() as (fd, tmppath): etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': str(normalize_timestamp(time.time())), 'Content-Length': 10, } self.disk_file.put(fd, tmppath, metadata) etag = md5() etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(fd, metadata) if self.disk_file.data_file: return self.disk_file.data_file return ts_file_path
def test_object_run_fast_track_non_zero(self): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 data = '0' * 1024 etag = md5() with self.disk_file.mkstemp() as (fd, tmppath): os.write(fd, data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': str(normalize_timestamp(time.time())), 'Content-Length': str(os.fstat(fd).st_size), } self.disk_file.put(fd, tmppath, metadata) etag = md5() etag.update('1' + '0' * 1023) etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(fd, metadata) quarantine_path = os.path.join(self.devices, 'sda', 'quarantined', 'objects') self.auditor.run_once(zero_byte_fps=50) self.assertFalse(os.path.isdir(quarantine_path)) self.auditor.run_once() self.assertTrue(os.path.isdir(quarantine_path))
def run_quarantine_zero_byte_get(self): container = "container-zbyte-%s" % uuid4() obj = "object-zbyte-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "DATA") with open(data_file) as fp: metadata = read_metadata(fp) os.unlink(data_file) with open(data_file, "w") as fp: write_metadata(fp, metadata) try: resp = direct_client.direct_get_object( onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1 ) raise "Did not quarantine object" except client.ClientException, e: self.assertEquals(e.http_status, 404)
def run_quarantine(self): container = "container-%s" % uuid4() obj = "object-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "VERIFY") with open(data_file) as fp: metadata = read_metadata(fp) metadata["ETag"] = "badetag" with open(data_file) as fp: write_metadata(fp, metadata) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] self.assertEquals(odata, "VERIFY") try: resp = direct_client.direct_get_object(onode, opart, self.account, container, obj) raise "Did not quarantine object" except client.ClientException, e: self.assertEquals(e.http_status, 404)
def run_quarantine(self): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() onode, opart, data_file = self._setup_data_file( container, obj, 'VERIFY') with open(data_file) as fp: metadata = read_metadata(fp) metadata['ETag'] = 'badetag' with open(data_file) as fp: write_metadata(fp, metadata) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] self.assertEquals(odata, 'VERIFY') try: resp = direct_client.direct_get_object(onode, opart, self.account, container, obj) raise "Did not quarantine object" except client.ClientException, e: self.assertEquals(e.http_status, 404)
def run_quarantine_range_etag(self): container = "container-range-%s" % uuid4() obj = "object-range-%s" % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, "RANGE") with open(data_file) as fp: metadata = read_metadata(fp) metadata["ETag"] = "badetag" with open(data_file) as fp: write_metadata(fp, metadata) for header, result in [ ({"Range": "bytes=0-2"}, "RAN"), ({"Range": "bytes=1-11"}, "ANGE"), ({"Range": "bytes=0-11"}, "RANGE"), ]: odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=header)[-1] self.assertEquals(odata, result) try: resp = direct_client.direct_get_object(onode, opart, self.account, container, obj) raise "Did not quarantine object" except client.ClientException, e: self.assertEquals(e.http_status, 404)