def _masked(value):
parts = str((
value.shape,
xxh64_hexdigest(value.data),
xxh64_hexdigest(value.mask),
))
return xxh64_hexdigest(parts)
def _hexdigest(value):
"""
Return a hexidecimal string hash representation of the provided value.
Calculates a 64-bit non-cryptographic hash of the provided value,
and returns the hexdigest string representation of the calculated hash.
"""
# Special case: deal with numpy arrays.
if ma.isMaskedArray(value):
parts = (
value.shape,
xxh64_hexdigest(value.data),
xxh64_hexdigest(value.mask),
)
value = str(parts)
elif isinstance(value, np.ndarray):
parts = (value.shape, xxh64_hexdigest(value))
value = str(parts)
try:
# Calculate single-shot hash to avoid allocating state on the heap
result = xxh64_hexdigest(value)
except TypeError:
# xxhash expects a bytes-like object, so try hashing the
# string representation of the provided value instead, but
# also fold in the object type...
parts = (type(value), value)
result = xxh64_hexdigest(str(parts))
return result
def calc_dir_signature(dir_path):
dir_signatures = []
for root_dir, dirs, files in os.walk(
os.path.abspath(os.path.expanduser(dir_path))):
for filename in files:
filepath = os.path.join(root_dir, filename)
with open(filepath, 'rb') as f:
dir_signatures.append(xxhash.xxh64_hexdigest(f.read()))
dir_signatures.sort()
sigs_as_string = '\n'.join(dir_signatures) + '\n'
return xxhash.xxh64_hexdigest(sigs_as_string)
def write_data(self,
array: np.ndarray,
*,
remote_operation: bool = False) -> bytes:
"""writes array data to disk in the numpy_00 fmtBackend
Parameters
----------
array : np.ndarray
tensor to write to disk
remote_operation : bool, optional, kwarg only
True if writing in a remote operation, otherwise False. Default is
False
Returns
-------
bytes
db hash record value specifying location information
"""
checksum = xxh64_hexdigest(array)
if self.w_uid in self.wFp:
self.hIdx += 1
if self.hIdx >= COLLECTION_SIZE:
self.wFp[self.w_uid].flush()
self._create_schema(remote_operation=remote_operation)
else:
self._create_schema(remote_operation=remote_operation)
destSlc = (self.hIdx, *[slice(0, x) for x in array.shape])
self.wFp[self.w_uid][destSlc] = array
self.wFp[self.w_uid].flush()
return numpy_10_encode(self.w_uid, checksum, self.hIdx, array.shape)
def define_anonymous_type(node: Cursor, bv: bn.BinaryView) -> bn.Type:
# An anonymous type must be either a Struct\UNION\ENUM.
# In order to simplify working with binaryNinja, an anonymized type is de-anonymized:
# The name of the anonymous type is a hash of its location in the source file prepended by 'anon_'
bn.log.log_debug(f'define_anonymous_type: Processing {node.type.spelling}')
struct = bn.Structure()
struct.width = node.type.get_size()
struct.alignment = node.type.get_align()
struct_name = 'anon_' + xxhash.xxh64_hexdigest(node.type.spelling)
for field in node.type.get_fields():
bn_field_type = bv.get_type_by_name(field.spelling)
field_name = field.spelling
if not bn_field_type:
# Need to define the field type
# if field.is_anonymous():
# field_name, bn_field_type = define_anonymous_type(field, bv)
# else:
field_name, bn_field_type = define_type(field.get_definition(), bv)
bn.log.log_debug(f'define_anonymous_type: Appending field - {bn_field_type} {field_name}')
struct.append(bn_field_type, field_name)
# Check if the underlying struct is a union
if node.type.kind == TypeKind.ELABORATED:
if node.type.get_named_type().get_declaration().kind == CursorKind.UNION_DECL:
# set type to union
struct.type = bn.StructureType.UnionStructureType
return struct_name, bn.Type.structure_type(struct)
def handle_file(self, path, base_dir, ignore_patterns):
for ignore_pattern in ignore_patterns:
if type(ignore_pattern) == tuple:
relative_folder, ignore_pattern = ignore_pattern
relative_path = os.path.relpath(path, relative_folder)
else:
relative_path = path
if fnmatch.fnmatch(relative_path, ignore_pattern):
return
storage_registry = self.storage_registry
with open(path, "rb") as f:
buffer = f.read()
digest = xxhash.xxh64_hexdigest(buffer) # type: str
friendly_path = os.path.relpath(path, base_dir)
if friendly_path not in storage_registry["files"]:
storage_registry["files"][friendly_path] = {}
original_dir = os.path.dirname(path)
file_name = os.path.basename(path)
rel_original_dir = os.path.relpath(original_dir, base_dir)
mkdir_p(os.path.join(self.storage_dir, "files", rel_original_dir))
target_rel_path = os.path.join("files", rel_original_dir, file_name + "." + digest)
is_new = False
if digest not in storage_registry["files"][friendly_path]:
# save file to storage
is_new = True
target_full_path = os.path.join(self.storage_dir, target_rel_path)
shutil.copy(path, target_full_path)
storage_registry["files"][friendly_path][digest] = target_full_path
if is_new:
print("New file found: {}...{}".format(friendly_path, digest))
else:
print("Existing file found: {}...{}".format(friendly_path, digest))
return is_new, friendly_path, digest, target_rel_path
def read_data(self, hashVal: NUMPY_10_DataHashSpec) -> np.ndarray:
"""Read data from disk written in the numpy_00 fmtBackend
Parameters
----------
hashVal : NUMPY_10_DataHashSpec
record specification stored in the db
Returns
-------
np.ndarray
tensor data stored at the provided hashVal specification.
Raises
------
RuntimeError
If the recorded checksum does not match the received checksum.
Notes
-----
TO AVOID DATA LOSS / CORRUPTION:
* On a read operation, we copy memmap subarray tensor data to a new
`np.ndarray` instance so as to prevent writes on a raw memmap result
slice (a `np.memmap` instance) from propogating to data on disk.
* This is an issue for reads from a write-enabled checkout where data
was just written, since the np flag "WRITEABLE" and "OWNDATA" will be
true, and writes to the returned array would be overwrite that data
slice on disk.
* For read-only checkouts, modifications to the resultant array would
perform a "copy on write"-like operation which would be propogated to
all future reads of the subarray from that process, but which would
not be persisted to disk.
"""
srcSlc = (hashVal.collection_idx,
*[slice(0, x) for x in hashVal.shape])
try:
res = self.Fp[hashVal.uid][srcSlc]
except TypeError:
self.Fp[hashVal.uid] = self.Fp[hashVal.uid]()
res = self.Fp[hashVal.uid][srcSlc]
except KeyError:
process_dir = self.STAGEDIR if self.mode == 'a' else self.STOREDIR
if Path(process_dir, f'{hashVal.uid}.npy').is_file():
file_pth = self.DATADIR.joinpath(f'{hashVal.uid}.npy')
self.rFp[hashVal.uid] = open_memmap(file_pth, 'r')
res = self.Fp[hashVal.uid][srcSlc]
else:
raise
out = np.array(res, dtype=res.dtype, order='C')
if xxh64_hexdigest(out) != hashVal.checksum:
raise RuntimeError(
f'DATA CORRUPTION Checksum {xxh64_hexdigest(out)} != recorded {hashVal}'
)
return out
def hash(df):
"""Compute the hash of a pandas DataFrame.
This only considers the index, data, and column names.
"""
dataset_hash = xxhash.xxh64_hexdigest(np.ascontiguousarray(df.values))
dataset_hash += joblib.hashing.hash(df.index)
dataset_hash += joblib.hashing.hash(df.columns)
return dataset_hash
def hexdigest(item):
"""
Calculate a hexidecimal string hash representation of the provided item.
Calculates a 64-bit non-cryptographic hash of the provided item, using
the extremely fast ``xxhash`` hashing algorithm, and returns the hexdigest
string representation of the hash.
This provides a means to compare large and/or complex objects through
simple string hexdigest comparison.
Args:
* item (object):
The item that requires to have its hexdigest calculated.
Returns:
The string hexidecimal representation of the item's 64-bit hash.
"""
# Special case: deal with numpy arrays.
if ma.isMaskedArray(item):
parts = (
item.shape,
xxh64_hexdigest(item.data),
xxh64_hexdigest(item.mask),
)
item = str(parts)
elif isinstance(item, np.ndarray):
parts = (item.shape, xxh64_hexdigest(item))
item = str(parts)
try:
# Calculate single-shot hash to avoid allocating state on the heap
result = xxh64_hexdigest(item)
except TypeError:
# xxhash expects a bytes-like object, so try hashing the
# string representation of the provided item instead, but
# also fold in the object type...
parts = (type(item), item)
result = xxh64_hexdigest(str(parts))
return result
def hash(cube):
cube_hash = ""
if isinstance(cube.data, np.ma.core.MaskedArray):
cube_hash += _ma_hasher.hash(cube.data)
else:
cube_hash += xxhash.xxh64_hexdigest(cube.data)
for coord in cube.coords():
cube_hash += joblib.hashing.hash(coord)
cube_hash += joblib.hashing.hash(cube.metadata)
return cube_hash
def compute_xxhash64_digest_filepath(filepath):
try:
import xxhash
import os
assert os.path.exists(filepath)
with open(filepath, 'rb') as file_:
digest = xxhash.xxh64_hexdigest(file_.read())
except Exception:
digest = None
return digest
def hash_key(key: Tuple[Optional[str], Any]) -> str:
"""
Creates a key from a picklable document and an optional humann readable
prefix.
"""
prefix, document = key
raw = [
prefix,
xxh64_hexdigest(pickle.dumps(document))
if document is not None else None,
]
return ":".join(filter(None, raw))
def o_encrypt(self, data):
a = base64.b64decode(data)
i = 16
s = max((len(a) - 2 * i) // 3, 0)
u = a[s:s + i]
a = a[0:s] + a[s + i:]
sec_key = xxhash.xxh64_hexdigest(u, 41405)
print(sec_key)
text = rc4(a, sec_key)
data = plistlib.loads(text, fmt=FMT_BINARY)
return data
def write_data(self,
array: np.ndarray,
*,
remote_operation: bool = False) -> bytes:
"""verifies correctness of array data and performs write operation.
Parameters
----------
array : np.ndarray
tensor to write to group.
remote_operation : optional, kwarg only, bool
If this is a remote process which is adding data, any necessary
hdf5 dataset files will be created in the remote data dir instead
of the stage directory. (default is False, which is for a regular
access process)
Returns
-------
bytes
string identifying the collection dataset and collection dim-0 index
which the array can be accessed at.
"""
checksum = xxh64_hexdigest(array)
if self.w_uid in self.wFp:
self.hIdx += 1
if self.hIdx >= self.hMaxSize:
self.hIdx = 0
self.hNextPath += 1
self.hColsRemain -= 1
if self.hColsRemain <= 1:
self.wFp[self.w_uid]['/'].attrs.modify(
'next_location', (self.hNextPath, self.hIdx))
self.wFp[self.w_uid]['/'].attrs.modify(
'collections_remaining', self.hColsRemain)
self.wFp[self.w_uid].flush()
self._create_schema(remote_operation=remote_operation)
else:
self._create_schema(remote_operation=remote_operation)
srcSlc = None
destSlc = (self.slcExpr[self.hIdx], self.slcExpr[0:array.size])
flat_arr = np.ravel(array)
self.wFp[self.w_uid][f'/{self.hNextPath}'].write_direct(
flat_arr, srcSlc, destSlc)
hashVal = hdf5_00_encode(uid=self.w_uid,
checksum=checksum,
dataset=self.hNextPath,
dataset_idx=self.hIdx,
shape=array.shape)
return hashVal
def generate_project_config(base_path):
project_config = {}
project_paths = {}
project_config['paths'] = project_paths
base_path = os.path.abspath(base_path)
project_config['project_name'] = os.path.basename(base_path)
project_config['project_id'] = xxh64_hexdigest(base_path + str(randint))
project_paths['project_path'] = base_path
project_paths['log_path'] = os.path.join(base_path, 'logs/')
project_paths['components_path'] = os.path.join(base_path, 'components/')
project_paths['workflows_path'] = os.path.join(base_path, 'workflows/')
project_paths['data_path'] = os.path.join(base_path, 'data/')
project_paths['models_path'] = os.path.join(base_path, 'models/')
return project_config
def add(self, value):
"""
Adds the item to the HyperLogLog
"""
# h: D -> {0,1} ** 64
# x = h(v)
# j = <x_0x_1..x_{p-1}>
# w = <x_{p}x_{p+1}..>
# M[j] = max(M[j], rho(w))
x = long(xxhash.xxh64_hexdigest(value)[:16], 16)
# x = long(sha1(bytes(value.encode('utf8'))).hexdigest()[:16], 16)
j = x & (self._m - 1)
w = x >> self._p
self._M[j] = max(self._M[j], get_rho(w, 64 - self._p))
def _json_hash_encode(row: Series) -> Series:
"""
Take a DataFrame row, add serialized JSON and hash
Parameters
----------
row: Series
a DataFrame row
Returns
-------
Series
the row with the json and hash columns added
"""
json = row.to_json()
row["Json"] = json
row["Hash"] = xxhash.xxh64_hexdigest(json.encode("utf-8"))
return row
def write_data(self,
data: str,
*,
remote_operation: bool = False) -> bytes:
"""verifies correctness of array data and performs write operation.
Parameters
----------
data: str
data to write to group.
remote_operation : optional, kwarg only, bool
If this is a remote process which is adding data, any necessary
hdf5 dataset files will be created in the remote data dir instead
of the stage directory. (default is False, which is for a regular
access process)
Returns
-------
bytes
string identifying the collection dataset and collection dim-0 index
which the array can be accessed at.
"""
encoded_data = data.encode()
checksum = xxh64_hexdigest(encoded_data)
if self.w_uid in self.wFp:
try:
row_idx = next(self.row_idx)
except StopIteration:
self._create_schema(remote_operation=remote_operation)
return self.write_data(data, remote_operation=remote_operation)
else:
self._create_schema(remote_operation=remote_operation)
return self.write_data(data, remote_operation=remote_operation)
encoded_row_idx = row_idx.encode()
try:
with self.wFp[self.w_uid].begin(write=True) as txn:
txn.put(encoded_row_idx, encoded_data, append=True)
except lmdb.MapFullError:
self._create_schema(remote_operation=remote_operation)
return self.write_data(data, remote_operation=remote_operation)
return lmdb_30_encode(self.w_uid, row_idx, checksum)
def post(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('workflow-name',
type=str,
required=True,
help='workflow name')
parser.add_argument('project-id',
type=str,
required=True,
help='workflow path')
args = parser.parse_args()
args['workflow-id'] = xxh64_hexdigest(args['workflow-name'])
args['status'] = 0
args['environment'] = 'local'
args['schedule'] = None
args['execution'] = None
db = get_plasma_db()
workflow_collection = db.get_collection('workflows')
workflow_collection.insert(dict(args))
update_project_statistics(args['project-id'])
response = generate_response(201)
return response
def test_xxh64_overflow(self):
s = 'I want an unsigned 64-bit seed!'
a = xxhash.xxh64(s, seed=0)
b = xxhash.xxh64(s, seed=2**64)
self.assertEqual(a.seed, b.seed)
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(), xxhash.xxh64_intdigest(s, seed=0))
self.assertEqual(a.intdigest(), xxhash.xxh64_intdigest(s, seed=2**64))
self.assertEqual(a.digest(), xxhash.xxh64_digest(s, seed=0))
self.assertEqual(a.digest(), xxhash.xxh64_digest(s, seed=2**64))
self.assertEqual(a.hexdigest(), xxhash.xxh64_hexdigest(s, seed=0))
self.assertEqual(a.hexdigest(), xxhash.xxh64_hexdigest(s, seed=2**64))
a = xxhash.xxh64(s, seed=1)
b = xxhash.xxh64(s, seed=2**64 + 1)
self.assertEqual(a.seed, b.seed)
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(), xxhash.xxh64_intdigest(s, seed=1))
self.assertEqual(a.intdigest(),
xxhash.xxh64_intdigest(s, seed=2**64 + 1))
self.assertEqual(a.digest(), xxhash.xxh64_digest(s, seed=1))
self.assertEqual(a.digest(), xxhash.xxh64_digest(s, seed=2**64 + 1))
self.assertEqual(a.hexdigest(), xxhash.xxh64_hexdigest(s, seed=1))
self.assertEqual(a.hexdigest(),
xxhash.xxh64_hexdigest(s, seed=2**64 + 1))
a = xxhash.xxh64(s, seed=2**65 - 1)
b = xxhash.xxh64(s, seed=2**66 - 1)
self.assertEqual(a.seed, b.seed)
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(),
xxhash.xxh64_intdigest(s, seed=2**65 - 1))
self.assertEqual(a.intdigest(),
xxhash.xxh64_intdigest(s, seed=2**66 - 1))
self.assertEqual(a.digest(), xxhash.xxh64_digest(s, seed=2**65 - 1))
self.assertEqual(a.digest(), xxhash.xxh64_digest(s, seed=2**66 - 1))
self.assertEqual(a.hexdigest(),
xxhash.xxh64_hexdigest(s, seed=2**65 - 1))
self.assertEqual(a.hexdigest(),
xxhash.xxh64_hexdigest(s, seed=2**66 - 1))
def read_data(self, hashVal: LMDB_30_DataHashSpec) -> str:
"""Read data from an hdf5 file handle at the specified locations
Parameters
----------
hashVal : LMDB_30_DataHashSpec
record specification parsed from its serialized store val in lmdb.
Returns
-------
str
requested data.
"""
try:
with self.Fp[hashVal.uid].begin(write=False) as txn:
res = txn.get(hashVal.row_idx.encode(), default=False)
if res is False:
raise RuntimeError(hashVal)
except AttributeError:
self.Fp[hashVal.uid] = self.Fp[hashVal.uid]()
return self.read_data(hashVal)
except KeyError:
process_dir = self.STAGEDIR if self.mode == 'a' else self.STOREDIR
if Path(process_dir, f'{hashVal.uid}.lmdbdir').is_file():
file_pth = self.DATADIR.joinpath(hashVal.uid)
self.rFp[hashVal.uid] = lmdb.open(str(file_pth),
readonly=True,
**LMDB_SETTINGS)
return self.read_data(hashVal)
else:
raise
out = res.decode()
if xxh64_hexdigest(res) != hashVal.checksum:
raise RuntimeError(
f'DATA CORRUPTION Checksum {xxh64_hexdigest(res)} != recorded {hashVal}'
)
return out
def post(self, workflow_id):
db = get_plasma_db()
execution_pass = {}
execution_pass['workflow-id'] = workflow_id
execution_pass['status'] = 1
execution_pass['started-at'] = time()
execution_pass['finished-at'] = None
execution_pass['execution-id'] = xxh64_hexdigest(workflow_id +
str(time()))
execution_collection = db.get_collection('executions')
workflow = db.workflows.find_one({"workflow-id": workflow_id})
project = db.projects.find_one({"project-id": workflow['project-id']})
payload = {}
payload['project-id'] = project['project-id']
payload['project-name'] = project['project-name']
payload['project-path'] = project['project-path']
payload['environment'] = workflow['environment']
payload['workflow-id'] = workflow_id
payload['workflow-name'] = workflow['workflow-name']
payload['execution-id'] = execution_pass['execution-id']
run_workflow(payload)
execution_collection.insert(execution_pass)
response = generate_response(200)
return response
def hash_numpy(x: numpy.ndarray) -> int:
"""Return a value that uniquely identifies a numpy array."""
return xxhash.xxh64_hexdigest(x.tobytes())
def perform_merge(self):
force = self._options.get("force", False)
slog_path = util.get_master_modpack_dir() / "logs" / "savedata.log"
new_entries = self.consolidate_diffs(self.get_all_diffs())
if not new_entries:
print("No savedata merging necessary.")
if slog_path.exists():
slog_path.unlink()
if (util.get_master_modpack_dir() / "logs" /
"savedata.sarc").exists():
(util.get_master_modpack_dir() / "logs" /
"savedata.sarc").unlink()
return
if slog_path.exists() and not force:
with slog_path.open("r") as l_file:
if xxhash.xxh64_hexdigest(str(new_entries)) == l_file.read():
print("No savedata merging necessary.")
return
savedata = get_stock_savedata()
save_files = sorted(savedata.get_files(), key=lambda f: f.name)[0:-2]
print("Merging changes...")
merged_entries = oead.byml.Array(
sorted(
{
entry["HashValue"].v: entry
for entry in [
*[
e for file in save_files for e in
oead.byml.from_binary(file.data)["file_list"][1]
],
*new_entries["add"],
] if entry not in new_entries["del"]
}.values(),
key=itemgetter("HashValue"),
))
print("Creating and injecting new savedataformat.sarc...")
new_savedata = oead.SarcWriter(
endian=oead.Endianness.Big if util.get_settings("wiiu") else oead.
Endianness.Little)
num_files = ceil(len(merged_entries) / 8192)
for i in range(num_files):
end_pos = (i + 1) * 8192
if end_pos > len(merged_entries):
end_pos = len(merged_entries)
data = oead.byml.to_binary(
oead.byml.Hash({
"file_list":
oead.byml.Array([
{
"IsCommon": False,
"IsCommonAtSameAccount": False,
"IsSaveSecureCode": True,
"file_name": "game_data.sav",
},
oead.byml.Array(merged_entries[i * 8192:end_pos]),
]),
"save_info":
oead.byml.Array([{
"directory_num": oead.S32(8),
"is_build_machine": True,
"revision": oead.S32(18203),
}]),
}),
big_endian=util.get_settings("wiiu"),
)
new_savedata.files[f"/saveformat_{i}.bgsvdata"] = data
new_savedata.files[f"/saveformat_{num_files}.bgsvdata"] = oead.Bytes(
savedata.get_file("/saveformat_6.bgsvdata").data)
new_savedata.files[
f"/saveformat_{num_files + 1}.bgsvdata"] = oead.Bytes(
savedata.get_file("/saveformat_7.bgsvdata").data)
del savedata
new_save_bytes = new_savedata.write()[1]
del new_savedata
util.inject_file_into_sarc(
"GameData/savedataformat.ssarc",
util.compress(new_save_bytes),
"Pack/Bootup.pack",
create_sarc=True,
)
(util.get_master_modpack_dir() / "logs").mkdir(parents=True,
exist_ok=True)
((util.get_master_modpack_dir() / "logs" /
"savedata.sarc").write_bytes(new_save_bytes))
print("Updating RSTB...")
rstable.set_size(
"GameData/savedataformat.sarc",
rstable.calculate_size("GameData/savedataformat.sarc",
new_save_bytes),
)
del new_save_bytes
slog_path.parent.mkdir(parents=True, exist_ok=True)
with slog_path.open("w", encoding="utf-8") as l_file:
l_file.write(xxhash.xxh64_hexdigest(str(new_entries)))
def perform_merge(self):
force = self._options.get("force", False)
glog_path = util.get_master_modpack_dir() / "logs" / "gamedata.log"
modded_entries = self.consolidate_diffs(self.get_all_diffs())
util.vprint("All gamedata diffs:")
util.vprint(modded_entries)
if not modded_entries:
print("No gamedata merging necessary.")
if glog_path.exists():
glog_path.unlink()
if (util.get_master_modpack_dir() / "logs" /
"gamedata.sarc").exists():
(util.get_master_modpack_dir() / "logs" /
"gamedata.sarc").unlink()
return
if glog_path.exists() and not force:
with glog_path.open("r") as l_file:
if xxhash.xxh64_hexdigest(
str(modded_entries)) == l_file.read():
print("No gamedata merging necessary.")
return
print("Loading stock gamedata...")
gamedata = consolidate_gamedata(get_stock_gamedata())
merged_entries = {
data_type:
oead.byml.Hash({entry["DataName"]: entry
for entry in entries})
for data_type, entries in gamedata.items()
}
del gamedata
print("Merging changes...")
for data_type in {d for d in merged_entries if d in modded_entries}:
util.dict_merge(
merged_entries[data_type],
modded_entries[data_type]["add"],
shallow=True,
)
for entry in modded_entries[data_type]["del"]:
try:
del merged_entries[data_type][entry]
except KeyError:
continue
merged_entries = oead.byml.Hash({
data_type: oead.byml.Array({value
for _, value in entries.items()})
for data_type, entries in merged_entries.items()
})
print("Creating and injecting new gamedata.sarc...")
new_gamedata = oead.SarcWriter(
endian=oead.Endianness.Big if util.get_settings("wiiu") else oead.
Endianness.Little)
for data_type in merged_entries:
num_files = ceil(len(merged_entries[data_type]) / 4096)
for i in range(num_files):
end_pos = (i + 1) * 4096
if end_pos > len(merged_entries[data_type]):
end_pos = len(merged_entries[data_type])
new_gamedata.files[
f"/{data_type}_{i}.bgdata"] = oead.byml.to_binary(
oead.byml.Hash({
data_type:
merged_entries[data_type][i * 4096:end_pos]
}),
big_endian=util.get_settings("wiiu"),
)
new_gamedata_bytes = new_gamedata.write()[1]
del new_gamedata
util.inject_file_into_sarc(
"GameData/gamedata.ssarc",
util.compress(new_gamedata_bytes),
"Pack/Bootup.pack",
create_sarc=True,
)
(util.get_master_modpack_dir() / "logs").mkdir(parents=True,
exist_ok=True)
(util.get_master_modpack_dir() / "logs" /
"gamedata.sarc").write_bytes(new_gamedata_bytes)
print("Updating RSTB...")
rstable.set_size(
"GameData/gamedata.sarc",
rstable.calculate_size("GameData/gamedata.sarc",
new_gamedata_bytes),
)
del new_gamedata_bytes
glog_path.parent.mkdir(parents=True, exist_ok=True)
with glog_path.open("w", encoding="utf-8") as l_file:
l_file.write(xxhash.xxh64_hexdigest(str(modded_entries)))
def compute_file_hash(file):
return xxh64_hexdigest(file.read())
def compute_hash_hex64(value):
return xxh64_hexdigest(value)
def hashit(dn, bloom_hashes, bloom_bits):
a = int(xxhash.xxh64_hexdigest(dn, seed=0), 16)
b = int(xxhash.xxh64_hexdigest(dn, seed=a), 16)
idx = [(a + b * k) % bloom_bits
for k in range(bloom_hashes)] # get int for bit setting
return idx
def get_volname_hash(volname):
"""XXHash based on Volume name"""
return xxhash.xxh64_hexdigest(volname)
def cachefile(self, path):
return os.path.join(self.cache_path, xxhash.xxh64_hexdigest(path))