def _create_object_metadata(self, file_path): if self._etag is None: self._etag = md5().hexdigest() if self._is_dir \ else get_etag(file_path) if self._file_has_changed or (X_TIMESTAMP not in self._metadata): timestamp = normalize_timestamp(self._stat.st_mtime) else: timestamp = self._metadata[X_TIMESTAMP] metadata = { X_TYPE: OBJECT, X_TIMESTAMP: timestamp, X_CONTENT_TYPE: DIR_TYPE if self._is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR_NON_OBJECT if self._is_dir else FILE, X_CONTENT_LENGTH: 0 if self._is_dir else self._stat.st_size, X_ETAG: self._etag} # Add X_MTIME key if object is a file if not self._is_dir: metadata[X_MTIME] = normalize_timestamp(self._stat.hpss_st_mtime) meta_new = self._metadata.copy() meta_new.update(metadata) if self._metadata != meta_new: write_metadata(file_path, meta_new) # Avoid additional read_metadata() later self._metadata = meta_new
def test_get_etag(self): tf = tempfile.NamedTemporaryFile() tf.file.write('123' * utils.CHUNK_SIZE) tf.file.flush() hd = utils.get_etag(tf.name) tf.file.seek(0) md5 = hashlib.md5() while True: chunk = tf.file.read(utils.CHUNK_SIZE) if not chunk: break md5.update(chunk) assert hd == md5.hexdigest()
def _validate_object_metadata(self): # Has no Swift specific metadata saved as xattr. Probably because # object was added/replaced through filesystem interface. if not self._metadata and not self._is_dir: self._file_has_changed = True return False required_keys = \ (X_TIMESTAMP, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_ETAG, # SOF specific keys X_TYPE, X_OBJECT_TYPE) if not all(k in self._metadata for k in required_keys): # At least one of the required keys does not exist return False if not self._is_dir: # X_MTIME is a new key added recently, newer objects will # have the key set during PUT. if X_MTIME in self._metadata: # Check if the file has been modified through filesystem # interface by comparing mtime stored in xattr during PUT # and current mtime of file. obj_stat = do_stat(self._data_file) if normalize_timestamp(self._metadata[X_MTIME]) != \ normalize_timestamp(obj_stat.hpss_st_mtime): self._file_has_changed = True return False else: # Without X_MTIME key, comparing md5sum is the only way # to determine if file has changed or not. This is inefficient # but there's no other way! self._etag = get_etag(self._data_file) if self._etag != self._metadata[X_ETAG]: self._file_has_changed = True return False else: # Checksums are same; File has not changed. For the next # GET request on same file, we don't compute md5sum again! # This is achieved by setting X_MTIME to mtime in # _create_object_metadata() return False if self._metadata[X_TYPE] == OBJECT: return True return False
def test_get_etag_empty(self): tf = tempfile.NamedTemporaryFile() hd = utils.get_etag(tf.name) assert hd == hashlib.md5().hexdigest()