def _verify_data_file(self, fp, current_time):
"""
Verify the metadata's name value matches what we think the object is
named.
:raises DiskFileCollision: if the metadata stored name does not match
the referenced name of the file
:raises DiskFileNotExist: if the object has expired
:raises DiskFileQuarantined: if data inconsistencies were detected
between the metadata and the file-system
metadata
"""
try:
mname = self._metadata['name']
except KeyError:
raise self._quarantine(self._name, "missing name metadata")
else:
if mname != self._name:
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
try:
x_delete_at = int(self._metadata['X-Delete-At'])
except KeyError:
pass
except ValueError:
# Quarantine, the x-delete-at key is present but not an
# integer.
raise self._quarantine(
self._name, "bad metadata x-delete-at value %s" %
(self._metadata['X-Delete-At']))
else:
if current_time is None:
current_time = time.time()
if x_delete_at <= current_time:
raise DiskFileNotExist('Expired')
try:
metadata_size = int(self._metadata['Content-Length'])
except KeyError:
raise self._quarantine(self._name,
"missing content-length in metadata")
except ValueError:
# Quarantine, the content-length key is present but not an
# integer.
raise self._quarantine(
self._name, "bad metadata content-length value %s" %
(self._metadata['Content-Length']))
try:
fp.seek(0, 2)
obj_size = fp.tell()
fp.seek(0, 0)
except OSError as err:
# Quarantine, we can't successfully stat the file.
raise self._quarantine(self._name, "not stat-able: %s" % err)
if obj_size != metadata_size:
raise self._quarantine(
self._name, "metadata content-length %s does"
" not match actual object size %s" % (metadata_size, obj_size))
return fp
def _verify_name(self):
"""
Verify the metadata's name value matches what we think the object is
named.
"""
try:
mname = self.metadata['name']
except KeyError:
pass
else:
if mname != self.name:
self.logger.error(_('Client path %(client)s does not match '
'path stored in object metadata %(meta)s'),
{'client': self.name, 'meta': mname})
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
def _verify_data_file(self):
"""
Verify the metadata's name value matches what we think the object is
named.
:raises DiskFileCollision: if the metadata stored name does not match
the referenced name of the file
:raises DiskFileNotExist: if the object has expired
:raises DiskFileQuarantined: if data inconsistencies were detected
between the metadata and the file-system
metadata
"""
try:
mname = self._metadata['name']
except KeyError:
self._quarantine("missing name metadata")
else:
if mname != self._name:
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
try:
x_delete_at = int(self._metadata['X-Delete-At'])
except KeyError:
pass
except ValueError:
# Quarantine, the x-delete-at key is present but not an
# integer.
self._quarantine("bad metadata x-delete-at value %s" %
(self._metadata['X-Delete-At']))
else:
if x_delete_at <= time.time():
raise DiskFileNotExist('Expired')
try:
metadata_size = int(self._metadata['Content-Length'])
except KeyError:
self._quarantine("missing content-length in metadata")
except ValueError:
# Quarantine, the content-length key is present but not an
# integer.
self._quarantine("bad metadata content-length value %s" %
(self._metadata['Content-Length']))
obj_size = self._fs_inst.size(self._name)
if obj_size != metadata_size:
self._quarantine("metadata content-length %s does"
" not match actual object size %s" %
(metadata_size, obj_size))
def __init__(self,
path,
device,
partition,
account,
container,
obj,
logger,
keep_data_fp=False,
disk_chunk_size=65536,
bytes_per_sync=(512 * 1024 * 1024),
iter_hook=None,
threadpool=None,
obj_dir='objects',
mount_check=False,
disallowed_metadata_keys=None):
if mount_check and not check_mount(path, device):
raise DiskFileDeviceUnavailable()
self.disk_chunk_size = disk_chunk_size
self.bytes_per_sync = bytes_per_sync
self.iter_hook = iter_hook
self.name = '/' + '/'.join((account, container, obj))
name_hash = hash_path(account, container, obj)
self.datadir = join(path, device,
storage_directory(obj_dir, partition, name_hash))
self.device_path = join(path, device)
self.tmpdir = join(path, device, 'tmp')
self.logger = logger
self.disallowed_metadata_keys = disallowed_metadata_keys or []
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.suppress_file_closing = False
self.threadpool = threadpool or ThreadPool(nthreads=0)
if not exists(self.datadir):
return
files = sorted(os.listdir(self.datadir), reverse=True)
meta_file = None
for afile in files:
if afile.endswith('.ts'):
self.data_file = None
with open(join(self.datadir, afile)) as mfp:
self.metadata = read_metadata(mfp)
self.metadata['deleted'] = True
break
if afile.endswith('.meta') and not meta_file:
meta_file = join(self.datadir, afile)
if afile.endswith('.data') and not self.data_file:
self.data_file = join(self.datadir, afile)
break
if not self.data_file:
return
self.fp = open(self.data_file, 'rb')
self.metadata = read_metadata(self.fp)
if not keep_data_fp:
self.close(verify_file=False)
if meta_file:
with open(meta_file) as mfp:
for key in self.metadata.keys():
if key.lower() not in self.disallowed_metadata_keys:
del self.metadata[key]
self.metadata.update(read_metadata(mfp))
if 'name' in self.metadata:
if self.metadata['name'] != self.name:
self.logger.error(
_('Client path %(client)s does not match '
'path stored in object metadata %(meta)s'), {
'client': self.name,
'meta': self.metadata['name']
})
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
def __init__(self,
path,
device,
partition,
account,
container,
obj,
logger,
keep_data_fp=False,
disk_chunk_size=65536,
bytes_per_sync=(512 * 1024 * 1024),
iter_hook=None):
self.disk_chunk_size = disk_chunk_size
self.bytes_per_sync = bytes_per_sync
self.iter_hook = iter_hook
self.name = '/' + '/'.join((account, container, obj))
name_hash = hash_path(account, container, obj)
self.datadir = os.path.join(
path, device, storage_directory(DATADIR, partition, name_hash))
self.device_path = os.path.join(path, device)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.meta_file = None
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.suppress_file_closing = False
if not os.path.exists(self.datadir):
return
files = sorted(os.listdir(self.datadir), reverse=True)
for afile in files:
if afile.endswith('.ts'):
self.data_file = self.meta_file = None
self.metadata = {'deleted': True}
return
if afile.endswith('.meta') and not self.meta_file:
self.meta_file = os.path.join(self.datadir, afile)
if afile.endswith('.data') and not self.data_file:
self.data_file = os.path.join(self.datadir, afile)
break
if not self.data_file:
return
self.fp = open(self.data_file, 'rb')
self.metadata = read_metadata(self.fp)
if not keep_data_fp:
self.close(verify_file=False)
if self.meta_file:
with open(self.meta_file) as mfp:
for key in self.metadata.keys():
if key.lower() not in DISALLOWED_HEADERS:
del self.metadata[key]
self.metadata.update(read_metadata(mfp))
if 'name' in self.metadata:
if self.metadata['name'] != self.name:
self.logger.error(
_('Client path %(client)s does not match '
'path stored in object metadata %(meta)s'), {
'client': self.name,
'meta': self.metadata['name']
})
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
def _verify_data_file(self, data_file, fp):
"""
Verify the metadata's name value matches what we think the object is
named.
:param data_file: data file name being consider, used when quarantines
occur
:param fp: open file pointer so that we can `fstat()` the file to
verify the on-disk size with Content-Length metadata value
:raises DiskFileCollision: if the metadata stored name does not match
the referenced name of the file
:raises DiskFileNotExist: if the object has expired
:raises DiskFileQuarantined: if data inconsistencies were detected
between the metadata and the file-system
metadata
"""
try:
mname = self._metadata['name']
except KeyError:
self._quarantine(data_file, "missing name metadata")
else:
if mname != self._name:
self._logger.error(
_('Client path %(client)s does not match '
'path stored in object metadata %(meta)s'), {
'client': self._name,
'meta': mname
})
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
try:
x_delete_at = int(self._metadata['X-Delete-At'])
except KeyError:
pass
except ValueError:
# Quarantine, the x-delete-at key is present but not an
# integer.
self._quarantine(
data_file, "bad metadata x-delete-at value %s" %
(self._metadata['X-Delete-At']))
else:
if x_delete_at <= time.time():
raise DiskFileNotExist('Expired')
try:
metadata_size = int(self._metadata['Content-Length'])
except KeyError:
self._quarantine(data_file, "missing content-length in metadata")
except ValueError:
# Quarantine, the content-length key is present but not an
# integer.
self._quarantine(
data_file, "bad metadata content-length value %s" %
(self._metadata['Content-Length']))
fd = fp.fileno()
try:
statbuf = os.fstat(fd)
except OSError as err:
# Quarantine, we can't successfully stat the file.
self._quarantine(data_file, "not stat-able: %s" % err)
else:
obj_size = statbuf.st_size
if metadata_size is not None and obj_size != metadata_size:
self._quarantine(
data_file, "metadata content-length %s does"
" not match actual object size %s" %
(metadata_size, statbuf.st_size))
return obj_size