def _create_object_metadata(self, file_path):
if self._etag is None:
self._etag = md5().hexdigest() if self._is_dir \
else get_etag(file_path)
if self._file_has_changed or (X_TIMESTAMP not in self._metadata):
timestamp = normalize_timestamp(self._stat.st_mtime)
else:
timestamp = self._metadata[X_TIMESTAMP]
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: timestamp,
X_CONTENT_TYPE: DIR_TYPE if self._is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR_NON_OBJECT if self._is_dir else FILE,
X_CONTENT_LENGTH: 0 if self._is_dir else self._stat.st_size,
X_ETAG: self._etag}
# Add X_MTIME key if object is a file
if not self._is_dir:
metadata[X_MTIME] = normalize_timestamp(self._stat.hpss_st_mtime)
meta_new = self._metadata.copy()
meta_new.update(metadata)
if self._metadata != meta_new:
write_metadata(file_path, meta_new)
# Avoid additional read_metadata() later
self._metadata = meta_new
def test_get_etag(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('123' * utils.CHUNK_SIZE)
tf.file.flush()
hd = utils.get_etag(tf.name)
tf.file.seek(0)
md5 = hashlib.md5()
while True:
chunk = tf.file.read(utils.CHUNK_SIZE)
if not chunk:
break
md5.update(chunk)
assert hd == md5.hexdigest()
def _validate_object_metadata(self):
# Has no Swift specific metadata saved as xattr. Probably because
# object was added/replaced through filesystem interface.
if not self._metadata and not self._is_dir:
self._file_has_changed = True
return False
required_keys = \
(X_TIMESTAMP, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_ETAG,
# SOF specific keys
X_TYPE, X_OBJECT_TYPE)
if not all(k in self._metadata for k in required_keys):
# At least one of the required keys does not exist
return False
if not self._is_dir:
# X_MTIME is a new key added recently, newer objects will
# have the key set during PUT.
if X_MTIME in self._metadata:
# Check if the file has been modified through filesystem
# interface by comparing mtime stored in xattr during PUT
# and current mtime of file.
obj_stat = do_stat(self._data_file)
if normalize_timestamp(self._metadata[X_MTIME]) != \
normalize_timestamp(obj_stat.hpss_st_mtime):
self._file_has_changed = True
return False
else:
# Without X_MTIME key, comparing md5sum is the only way
# to determine if file has changed or not. This is inefficient
# but there's no other way!
self._etag = get_etag(self._data_file)
if self._etag != self._metadata[X_ETAG]:
self._file_has_changed = True
return False
else:
# Checksums are same; File has not changed. For the next
# GET request on same file, we don't compute md5sum again!
# This is achieved by setting X_MTIME to mtime in
# _create_object_metadata()
return False
if self._metadata[X_TYPE] == OBJECT:
return True
return False
def test_get_etag_empty(self):
tf = tempfile.NamedTemporaryFile()
hd = utils.get_etag(tf.name)
assert hd == hashlib.md5().hexdigest()