def test_window_gen_data(self, tmp_path):
zero_file = tmp_path / "zero.bin"
rep = 1000
with open(zero_file, 'wb') as file:
file.write(bytes([1] * rep))
w = wg.WindowGenerator(24, 1, 0, 10, zero_file)
assert all(w._data == [[0], [0], [0], [0], [0], [0], [0], [1]] * rep)
def test_split(self, tmp_path):
one_file = tmp_path / "one.bin"
test_vector = [[1], [2], [3], [4], [5], [6], [7]]
with open(one_file, 'wb') as file:
file.write(bytes([1] * 1000))
w = wg.WindowGenerator(self._input_size, self._label_size, 0,
self._batch_size, one_file)
inputs, labels = w._split_window(
tf.constant([test_vector] * self._batch_size))
assert inputs.shape == [self._batch_size, self._input_size, 1]
assert labels.shape == [self._batch_size, self._label_size, 1]
for i, l in zip(inputs, labels):
assert all(i == test_vector[:self._input_size])
assert all(l == test_vector[self._input_size:])
def test_dataset(self, tmp_path):
one_file = tmp_path / "one.bin"
test_vector = np.tile(
np.array([[random.randint(0, 255)] for _ in range(10)],
dtype=np.uint8), (self._batch_size, 1))
bit_windows = self._bytesarray2window(test_vector)
with open(one_file, 'wb') as file:
file.write(bytes(test_vector.ravel()))
w = wg.WindowGenerator(self._input_size, self._label_size, 0,
self._batch_size, one_file)
ds = list(
w._make_dataset(0,
len(test_vector) * 8 - 1).as_numpy_iterator())
assert len(ds) == len(bit_windows)
for d, v in zip(ds, bit_windows):
assert len(d) == 2
assert all((d[0].ravel() == v[:self._input_size]))
assert all((d[1].ravel() == v[self._input_size:]))
def test_all_ds(self, tmp_path):
one_file = tmp_path / "one.bin"
test_vector = np.tile(
np.array([[random.randint(0, 255)] for _ in range(100)],
dtype=np.uint8), (self._batch_size, 1))
bit_windows = self._bytesarray2window(test_vector)
with open(one_file, 'wb') as file:
file.write(bytes(test_vector.ravel()))
w = wg.WindowGenerator(self._input_size, self._label_size, 0,
self._batch_size, one_file)
train = list(list(w.train.as_numpy_iterator()))
val = list(list(w.val.as_numpy_iterator()))
test = list(w.test.as_numpy_iterator())
assert len(train) < len(bit_windows)
assert len(val) == len(bit_windows[len(train):len(train) + len(val)])
assert len(test) == len(bit_windows[len(train) + len(val):])
for d, v in zip(train + val + test, bit_windows):
assert len(d) == 2
assert all(d[0].ravel() == v[:self._input_size])
assert all(d[1].ravel() == v[self._input_size:])
def main():
# Make TF deterministic
tf.random.set_seed(42)
logging.basicConfig(level=logging.INFO)
args = eval_args()
if args is not None:
if args.Verbose is not True:
logging.getLogger("tensorflow").setLevel(logging.ERROR)
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Prevent of full memory allocation on the GPU handy for shred resources
phys_devs = tf.config.experimental.list_physical_devices('GPU')
for dev in phys_devs:
tf.config.experimental.set_memory_growth(dev, True)
tbCallBack = None
if args.Verbose is True:
save_dir = Path(args.path_save) / (
'linear_log/' + (str(datetime.datetime.now()).replace(
' ', '-').replace(':', '').replace('.', '')) + "-" +
str(args.path.stem) + "-" + str(args.CELLTYPE))
save_dir.mkdir(parents=True, exist_ok=True)
tbCallBack = [
tf.keras.callbacks.TensorBoard(log_dir=save_dir,
histogram_freq=1,
write_graph=True,
write_images=True)
]
multi_window = wg.WindowGenerator(100, 1, 0, args.BATCH_SIZE,
args.path)
data = learn.learn(args, 100, 1, multi_window.get_mean_error())
model = data.make_model()
model.compile(optimizer=tf.keras.optimizers.Adam(args.LEARN_RATE),
loss=data.custom_loss,
metrics=['mae']) # mean absolute error
model.fit(multi_window.train,
epochs=args.EPOCH,
steps_per_epoch=args.BATCH_SIZE,
callbacks=tbCallBack,
shuffle=False,
validation_data=multi_window.val,
validation_steps=1000,
batch_size=args.BATCH_SIZE)
logging.info("Predict random values...")
learnd, labels = data.prediction(multi_window, model)
logging.info("Eval values with dieharder...")
learnd_res, label_res, improved_res = dieharder.dieharder_eval(
learnd, labels)
logging.info("Analysis of the input data")
x = 0
for a, b, c in label_res:
if (c == "PASSED"):
x = x + 1
logging.info("Diehader fails on", x, "from", str(len(label_res)),
"tests")
if x / len(label_res) >= 0.95:
logging.info("dieharder: PASSED")
else:
logging.info("dieharder: FAILED")
logging.info("Analysis of the learned values")
x = 0
for a, b, c in learnd_res:
if (c == "PASSED"):
x = x + 1
if (x / learnd_res < 0.05):
logging.error("The too few training the AI learnd nothing")
exit()
logging.info("Analysis of xor input and learned")
x = 0
for a1, b1, c1, a2, b2, c2 in zip(label_res, improved_res):
logging.debug(a1, "|", b1, "->", b2, "|", c1, "->", c2)
if (c2 == "PASSED"):
x = x + 1
if (x / len(improved_res) >= 0.95):
logging.info("RNN_TEST: PASSED")
else:
logging.info("RNN_TEST: FAILED")
def test_window_gen_me_max(self, tmp_path):
one_file = tmp_path / "one.bin"
with open(one_file, 'wb') as file:
file.write(bytes([255] * 1000))
w = wg.WindowGenerator(24, 1, 0, 10, one_file)
assert w.get_mean_error() == 1
def test_window_gen_me_zero(self, tmp_path):
zero_file = tmp_path / "zero.bin"
with open(zero_file, 'wb') as file:
file.write(bytes([0] * 1000))
w = wg.WindowGenerator(24, 1, 0, 10, zero_file)
assert w.get_mean_error() == 0
def test_window_gen_real(self, tmp_path):
sys_rand_file = tmp_path / "sys_rand.bin"
with open(sys_rand_file, 'wb') as file:
file.write(os.urandom(50000000))
w = wg.WindowGenerator(24, 1, 0, 10, sys_rand_file)
assert w is not None