def _generate_hash(self):
# Generate an FNV hash by creating a byte array containing the
# modification time of as well as the first and last 1 MB of data in
# every output file
ret = bytearray()
for pfile in self.data_files:
# only look at "real" files, not "fake" files generated by the
# chunking system
if pfile.start not in (0, None):
continue
try:
mtime = os.path.getmtime(pfile.filename)
except OSError as e:
if e.errno == errno.ENOENT:
# this is an in-memory file so we return with a dummy
# value
return -1
else:
raise
ret.extend(str(mtime).encode("utf-8"))
size = os.path.getsize(pfile.filename)
if size > 1e6:
size = int(1e6)
with open(pfile.filename, "rb") as fh:
# read in first and last 1 MB of data
data = fh.read(size)
fh.seek(-size, os.SEEK_END)
data = fh.read(size)
ret.extend(data)
return fnv_hash(ret)
def unit_system_id(self):
"""
This is a unique identifier for the unit registry created
from a FNV hash. It is needed to register a dataset's code
unit system in the unit system registry.
"""
if self._unit_system_id is None:
hash_data = bytearray()
for k, v in self.lut.items():
hash_data.extend(k.encode('ascii'))
hash_data.extend(repr(v).encode('ascii'))
self._unit_system_id = "us_%d" % fnv_hash(hash_data)
return self._unit_system_id