def _feature_predict_hash(self):
params = "".join([
"{}{}".format(k, v)
for k, v in self.parent.config._asdict().items()
if k not in ["features", "predict"]
])
return secure_hash("".join([params] + self.features),
algorithm="sha384")
def model_folder_path(self):
_to_hash = self.features + [
self.config.predict.name,
str(len(self.cids)),
self.config.model_path,
]
model_name = secure_hash("".join(_to_hash), algorithm="sha384")
model_folder_path = self.base_path / model_name
if not model_folder_path.exists():
model_folder_path.mkdir(parents=True, exist_ok=True)
return model_folder_path
def _feature_predict_hash(self):
params = sorted([
"{}{}".format(k, v) for k, v in self.config._asdict().items()
if k not in [
"features",
"predict",
"vwcmd",
"class_cost",
"importance",
"tag",
"base",
]
])
params = "".join(params)
return secure_hash("".join([params] + self.features),
algorithm="sha384")
async def test_idx3(self, filename):
feature_name = "image"
async with IDX3Source(
IDX3SourceConfig(filename=str(filename), feature=feature_name)
) as source:
async with source() as sctx:
records = [record async for record in sctx.records()]
self.assertEqual(len(records), 60000)
self.assertIn(feature_name, records[0].features())
for i in range(-1, 1):
with self.subTest(index=i):
is_hash = secure_hash(
json.dumps(records[i].feature(feature_name)),
algorithm="sha384",
)
self.assertEqual(is_hash, IDX3_FIRST_LAST[i])
def _model_dir_path(self):
if self.parent.config.directory is None:
return None
_to_hash = self.features + [
self.classification,
str(len(self.cids)),
self.parent.config.model_path,
]
model = secure_hash("".join(_to_hash), algorithm="sha384")
# Needed to save updated model
if not os.path.isdir(self.parent.config.directory):
raise NotADirectoryError("%s is not a directory" %
(self.parent.config.directory))
os.makedirs(os.path.join(self.parent.config.directory, model),
exist_ok=True)
return os.path.join(self.parent.config.directory, model)
async def test_dumpb_loadb(self):
async with PNGConfigLoader.withconfig({}) as configloader:
async with configloader() as ctx:
image_bytes = (
pathlib.Path(__file__).parent
/ ".."
/ ".."
/ ".."
/ "examples"
/ "MNIST"
/ "image1.png"
).read_bytes()
original = await ctx.loadb(image_bytes)
hash_original = secure_hash(
json.dumps(original.flatten().tolist()), algorithm="sha384"
)
self.assertEqual(original.shape, (280, 280, 3))
self.assertEqual(hash_original, IMAGE1_HASH)
def _saved_file_path(self):
saved_file_name = (
secure_hash(self.config.predict.name, algorithm="sha384") +
".json")
return self._base_path / saved_file_name
def _filename(self):
return os.path.join(
self.config.directory,
secure_hash(self.config.predict.name, algorithm="sha384")
+ ".json",
)
