def run_quarantine_range_etag(self):
container = 'container-range-%s' % uuid4()
obj = 'object-range-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(
container, obj, 'RANGE')
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata['ETag'] = 'badetag'
with open(data_file) as fp:
write_metadata(fp, metadata)
for header, result in [({
'Range': 'bytes=0-2'
}, 'RAN'), ({
'Range': 'bytes=1-11'
}, 'ANGE'), ({
'Range': 'bytes=0-11'
}, 'RANGE')]:
odata = direct_client.direct_get_object(onode,
opart,
self.account,
container,
obj,
headers=header)[-1]
self.assertEquals(odata, result)
try:
resp = direct_client.direct_get_object(onode, opart, self.account,
container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def run_quarantine_zero_byte_post(self):
container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
with open(data_file) as fp:
metadata = read_metadata(fp)
os.unlink(data_file)
with open(data_file, 'w') as fp:
write_metadata(fp, metadata)
try:
resp = direct_client.direct_post_object(
onode,
opart,
self.account,
container,
obj, {
'X-Object-Meta-1': 'One',
'X-Object-Meta-Two': 'Two'
},
conn_timeout=1,
response_timeout=1)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def object_audit(self, path, device, partition):
"""
Audits the given object path.
:param path: a path to an object
:param device: the device the path is on
:param partition: the partition the path is on
"""
try:
if not path.endswith('.data'):
return
try:
name = object_server.read_metadata(path)['name']
except (Exception, Timeout), exc:
raise AuditException('Error when reading metadata: %s' % exc)
_junk, account, container, obj = name.split('/', 3)
df = object_server.DiskFile(self.devices,
device,
partition,
account,
container,
obj,
self.logger,
keep_data_fp=True)
try:
if df.data_file is None:
# file is deleted, we found the tombstone
return
try:
obj_size = df.get_data_file_size()
except DiskFileError, e:
raise AuditException(str(e))
except DiskFileNotExist:
return
if self.zero_byte_only_at_fps and obj_size:
self.passes += 1
return
for chunk in df:
self.bytes_running_time = ratelimit_sleep(
self.bytes_running_time,
self.max_bytes_per_second,
incr_by=len(chunk))
self.bytes_processed += len(chunk)
self.total_bytes_processed += len(chunk)
df.close()
if df.quarantined_dir:
self.quarantines += 1
self.logger.error(
_("ERROR Object %(path)s failed audit and will be "
"quarantined: ETag and file's md5 do not match"),
{'path': path})
def run_quarantine_zero_byte_get(self):
container = "container-zbyte-%s" % uuid4()
obj = "object-zbyte-%s" % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, "DATA")
with open(data_file) as fp:
metadata = read_metadata(fp)
os.unlink(data_file)
with open(data_file, "w") as fp:
write_metadata(fp, metadata)
try:
resp = direct_client.direct_get_object(
onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1
)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def run_quarantine(self):
container = "container-%s" % uuid4()
obj = "object-%s" % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, "VERIFY")
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata["ETag"] = "badetag"
with open(data_file) as fp:
write_metadata(fp, metadata)
odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1]
self.assertEquals(odata, "VERIFY")
try:
resp = direct_client.direct_get_object(onode, opart, self.account, container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def object_audit(self, path, device, partition):
"""
Audits the given object path.
:param path: a path to an object
:param device: the device the path is on
:param partition: the partition the path is on
"""
try:
if not path.endswith('.data'):
return
try:
name = object_server.read_metadata(path)['name']
except (Exception, Timeout), exc:
raise AuditException('Error when reading metadata: %s' % exc)
_junk, account, container, obj = name.split('/', 3)
df = object_server.DiskFile(self.devices, device, partition,
account, container, obj, self.logger,
keep_data_fp=True)
try:
if df.data_file is None:
# file is deleted, we found the tombstone
return
try:
obj_size = df.get_data_file_size()
except DiskFileError, e:
raise AuditException(str(e))
except DiskFileNotExist:
return
if self.zero_byte_only_at_fps and obj_size:
self.passes += 1
return
for chunk in df:
self.bytes_running_time = ratelimit_sleep(
self.bytes_running_time, self.max_bytes_per_second,
incr_by=len(chunk))
self.bytes_processed += len(chunk)
self.total_bytes_processed += len(chunk)
df.close()
if df.quarantined_dir:
self.quarantines += 1
self.logger.error(
_("ERROR Object %(path)s failed audit and will be "
"quarantined: ETag and file's md5 do not match"),
{'path': path})
def run_quarantine(self):
container = 'container-%s' % uuid4()
obj = 'object-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(
container, obj, 'VERIFY')
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata['ETag'] = 'badetag'
with open(data_file) as fp:
write_metadata(fp, metadata)
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
self.assertEquals(odata, 'VERIFY')
try:
resp = direct_client.direct_get_object(onode, opart, self.account,
container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def run_quarantine_range_etag(self):
container = "container-range-%s" % uuid4()
obj = "object-range-%s" % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, "RANGE")
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata["ETag"] = "badetag"
with open(data_file) as fp:
write_metadata(fp, metadata)
for header, result in [
({"Range": "bytes=0-2"}, "RAN"),
({"Range": "bytes=1-11"}, "ANGE"),
({"Range": "bytes=0-11"}, "RANGE"),
]:
odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=header)[-1]
self.assertEquals(odata, result)
try:
resp = direct_client.direct_get_object(onode, opart, self.account, container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
