def object_audit(self, path, device, partition):
"""
Audits the given object path.
:param path: a path to an object
:param device: the device the path is on
:param partition: the partition the path is on
"""
try:
if not path.endswith('.data'):
return
try:
name = object_server.read_metadata(path)['name']
except Exception, exc:
raise AuditException('Error when reading metadata: %s' % exc)
_junk, account, container, obj = name.split('/', 3)
df = object_server.DiskFile(self.devices, device, partition,
account, container, obj, self.logger,
keep_data_fp=True)
if df.data_file is None:
# file is deleted, we found the tombstone
return
try:
obj_size = df.get_data_file_size()
except DiskFileError, e:
raise AuditException(str(e))
def object_audit(self, path, device, partition):
"""
Audits the given object path.
:param path: a path to an object
:param device: the device the path is on
:param partition: the partition the path is on
"""
try:
try:
name = diskfile.read_metadata(path)['name']
except (Exception, Timeout) as exc:
raise AuditException('Error when reading metadata: %s' % exc)
_junk, account, container, obj = name.split('/', 3)
df = diskfile.DiskFile(self.devices, device, partition, account,
container, obj, self.logger)
df.open()
try:
try:
obj_size = df.get_data_file_size()
except DiskFileNotExist:
return
except DiskFileError as e:
raise AuditException(str(e))
if self.stats_sizes:
self.record_stats(obj_size)
if self.zero_byte_only_at_fps and obj_size:
self.passes += 1
return
for chunk in df:
self.bytes_running_time = ratelimit_sleep(
self.bytes_running_time,
self.max_bytes_per_second,
incr_by=len(chunk))
self.bytes_processed += len(chunk)
self.total_bytes_processed += len(chunk)
df.close()
if df.quarantined_dir:
self.quarantines += 1
self.logger.error(
_("ERROR Object %(path)s failed audit and will be "
"quarantined: ETag and file's md5 do not match"),
{'path': path})
finally:
df.close(verify_file=False)
except AuditException as err:
self.logger.increment('quarantines')
self.quarantines += 1
self.logger.error(
_('ERROR Object %(obj)s failed audit and will '
'be quarantined: %(err)s'), {
'obj': path,
'err': err
})
diskfile.quarantine_renamer(os.path.join(self.devices, device),
path)
return
self.passes += 1
def object_audit(self, path, device, partition):
"""
Audits the given object path.
:param path: a path to an object
:param device: the device the path is on
:param partition: the partition the path is on
"""
try:
if not path.endswith('.data'):
return
try:
name = object_server.read_metadata(path)['name']
except (Exception, Timeout), exc:
raise AuditException('Error when reading metadata: %s' % exc)
_junk, account, container, obj = name.split('/', 3)
df = object_server.DiskFile(self.devices,
device,
partition,
account,
container,
obj,
self.logger,
keep_data_fp=True)
try:
if df.data_file is None:
# file is deleted, we found the tombstone
return
try:
obj_size = df.get_data_file_size()
except DiskFileError, e:
raise AuditException(str(e))
except DiskFileNotExist:
return
if self.zero_byte_only_at_fps and obj_size:
self.passes += 1
return
for chunk in df:
self.bytes_running_time = ratelimit_sleep(
self.bytes_running_time,
self.max_bytes_per_second,
incr_by=len(chunk))
self.bytes_processed += len(chunk)
self.total_bytes_processed += len(chunk)
df.close()
if df.quarantined_dir:
self.quarantines += 1
self.logger.error(
_("ERROR Object %(path)s failed audit and will be "
"quarantined: ETag and file's md5 do not match"),
{'path': path})
def object_audit(self, path, device, partition):
"""
Audits the given object path.
:param path: a path to an object
:param device: the device the path is on
:param partition: the partition the path is on
"""
try:
if not path.endswith('.data'):
return
try:
name = object_server.read_metadata(path)['name']
except Exception, exc:
raise AuditException('Error when reading metadata: %s' % exc)
_junk, account, container, obj = name.split('/', 3)
df = object_server.DiskFile(self.devices,
device,
partition,
account,
container,
obj,
keep_data_fp=True)
if df.data_file is None:
# file is deleted, we found the tombstone
return
obj_size = os.path.getsize(df.data_file)
if obj_size != int(df.metadata['Content-Length']):
raise AuditException('Content-Length of %s does not match '
'file size of %s' %
(int(df.metadata['Content-Length']),
os.path.getsize(df.data_file)))
if self.zero_byte_only_at_fps and obj_size:
return
etag = md5()
for chunk in df:
self.bytes_running_time = ratelimit_sleep(
self.bytes_running_time,
self.max_bytes_per_second,
incr_by=len(chunk))
etag.update(chunk)
self.bytes_processed += len(chunk)
self.total_bytes_processed += len(chunk)
etag = etag.hexdigest()
if etag != df.metadata['ETag']:
raise AuditException("ETag of %s does not match file's md5 of "
"%s" % (df.metadata['ETag'], etag))
def object_audit(self, path, device, partition):
"""
Audits the given object path.
:param path: a path to an object
:param device: the device the path is on
:param partition: the partition the path is on
"""
try:
try:
name = diskfile.read_metadata(path)['name']
except (Exception, Timeout) as exc:
raise AuditException('Error when reading metadata: %s' % exc)
_junk, account, container, obj = name.split('/', 3)
df = self.diskfile_mgr.get_diskfile(device, partition, account,
container, obj)
try:
with df.open():
metadata = df.get_metadata()
obj_size = int(metadata['Content-Length'])
if self.stats_sizes:
self.record_stats(obj_size)
if self.zero_byte_only_at_fps and obj_size:
self.passes += 1
return
reader = df.reader()
with closing(reader):
for chunk in reader:
chunk_len = len(chunk)
self.bytes_running_time = ratelimit_sleep(
self.bytes_running_time,
self.max_bytes_per_second,
incr_by=chunk_len)
self.bytes_processed += chunk_len
self.total_bytes_processed += chunk_len
if reader.was_quarantined:
self.quarantines += 1
self.logger.error(
_('ERROR Object %(obj)s failed audit and'
' was quarantined: %(err)s'), {
'obj': path,
'err': reader.was_quarantined
})
return
except DiskFileNotExist:
return
except DiskFileQuarantined as err:
self.quarantines += 1
self.logger.error(
_('ERROR Object %(obj)s failed audit and was'
' quarantined: %(err)s'), {
'obj': path,
'err': err
})
except AuditException as err:
self.logger.increment('quarantines')
self.quarantines += 1
self.logger.error(
_('ERROR Object %(obj)s failed audit and will'
' be quarantined: %(err)s'), {
'obj': path,
'err': err
})
diskfile.quarantine_renamer(os.path.join(self.devices, device),
path)
return
self.passes += 1