def get_all_shuffled():
from steves_utils.ORACLE.shuffled_dataset_accessor import Shuffled_Dataset_Factory
from steves_utils import utils
RANGE = len(ALL_SERIAL_NUMBERS)
path = os.path.join(utils.get_datasets_base_path(), "all_shuffled", "output")
print(utils.get_datasets_base_path())
print(path)
datasets = Shuffled_Dataset_Factory(path, train_val_test_splits=(0.6, 0.2, 0.2))
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
train_ds = train_ds.map(
lambda x: (x["IQ"],tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True
)
val_ds = val_ds.map(
lambda x: (x["IQ"],tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True
)
test_ds = test_ds.map(
lambda x: (x["IQ"],tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True
)
return train_ds, val_ds, test_ds
def test_for_duplicates(self):
datasets = Shuffled_Dataset_Factory(
self.output_path, train_val_test_splits=self.train_val_test_splits)
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
all_ds = train_ds.concatenate(val_ds).concatenate(test_ds)
train_hashes = []
for e in train_ds.unbatch():
train_hashes.append(
hash((
int(e["serial_number_id"].numpy()),
int(e["distance_feet"].numpy()),
int(e["run"].numpy()),
int(e["index_in_file"].numpy()),
)))
val_hashes = []
for e in val_ds.unbatch():
val_hashes.append(
hash((
int(e["serial_number_id"].numpy()),
int(e["distance_feet"].numpy()),
int(e["run"].numpy()),
int(e["index_in_file"].numpy()),
)))
test_hashes = []
for e in test_ds.unbatch():
test_hashes.append(
hash((
int(e["serial_number_id"].numpy()),
int(e["distance_feet"].numpy()),
int(e["run"].numpy()),
int(e["index_in_file"].numpy()),
)))
all_hashes = []
for e in all_ds.unbatch():
all_hashes.append(
hash((
int(e["serial_number_id"].numpy()),
int(e["distance_feet"].numpy()),
int(e["run"].numpy()),
int(e["index_in_file"].numpy()),
)))
self.assertTrue(
len(all_hashes) == len(train_hashes + val_hashes + test_hashes))
self.assertTrue(
len(train_hashes + val_hashes + test_hashes) == len(
set(train_hashes + val_hashes + test_hashes)))
def test_cardinality(self):
# I believe because we are working with discrete files, some of them are being dropped. We allow up to 1% of data to be lost
acceptable_cardinality_delta_percent = 0.01
datasets = Shuffled_Dataset_Factory(
self.output_path, train_val_test_splits=self.train_val_test_splits)
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
train_count = 0
for e in train_ds:
train_count += e["index_in_file"].shape[0]
val_count = 0
for e in val_ds:
val_count += e["index_in_file"].shape[0]
test_count = 0
for e in test_ds:
test_count += e["index_in_file"].shape[0]
expected_cardinality = self.cardinality
expected_train_count = expected_cardinality * self.train_val_test_splits[
0]
expected_val_count = expected_cardinality * self.train_val_test_splits[
1]
expected_test_count = expected_cardinality * self.train_val_test_splits[
2]
self.assertAlmostEqual(expected_cardinality,
train_count + val_count + test_count,
delta=expected_cardinality *
acceptable_cardinality_delta_percent)
self.assertAlmostEqual(train_count,
expected_train_count,
delta=expected_train_count *
acceptable_cardinality_delta_percent)
self.assertAlmostEqual(val_count,
expected_val_count,
delta=expected_val_count *
acceptable_cardinality_delta_percent)
self.assertAlmostEqual(test_count,
expected_test_count,
delta=expected_test_count *
acceptable_cardinality_delta_percent)
def test_compare_chunks_to_original(self):
datasets = Shuffled_Dataset_Factory(
self.output_path, train_val_test_splits=self.train_val_test_splits)
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
all_ds = train_ds.concatenate(val_ds).concatenate(test_ds)
for e in all_ds.unbatch():
original_iq = get_chunk_of_IQ_based_on_metadata_and_index(
serial_number_id=e["serial_number_id"].numpy(),
distance_feet=e["distance_feet"].numpy(),
run=e["run"].numpy(),
index=e["index_in_file"].numpy(),
num_samps_in_chunk=ORIGINAL_PAPER_SAMPLES_PER_CHUNK)
self.assertTrue(np.array_equal(e["IQ"].numpy(), original_iq))
def get_all_shuffled():
global RANGE
from steves_utils.ORACLE.shuffled_dataset_accessor import Shuffled_Dataset_Factory
from steves_utils import utils
BATCH = 256
path = os.path.join(utils.get_datasets_base_path(), "all_shuffled",
"output")
print(utils.get_datasets_base_path())
print(path)
datasets = Shuffled_Dataset_Factory(path,
train_val_test_splits=(0.6, 0.2, 0.2),
reshuffle_train_each_iteration=False)
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
train_ds = train_ds.map(
lambda x: (x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
val_ds = val_ds.map(lambda x:
(x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
test_ds = test_ds.map(lambda x:
(x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
train_ds = train_ds.unbatch().take(200000 *
len(ALL_SERIAL_NUMBERS)).batch(BATCH)
val_ds = val_ds.unbatch().take(10000 *
len(ALL_SERIAL_NUMBERS)).batch(BATCH)
test_ds = test_ds.unbatch().take(50000 *
len(ALL_SERIAL_NUMBERS)).batch(BATCH)
return train_ds, val_ds, test_ds
def test_shuffling(self):
"""
This one is a bit hard. How do you check for randomness?
What I ended up doing is taking the 'index_in_file' metadata field, sorting it, and comparing it to the original.
If they aren't the same then we should be good.
"""
datasets = Shuffled_Dataset_Factory(
self.output_path, train_val_test_splits=self.train_val_test_splits)
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
for ds in (train_ds, val_ds, test_ds):
indices = []
for e in ds:
indices.extend(e["index_in_file"].numpy())
sorted_indices = copy.deepcopy(indices)
sorted_indices.sort()
self.assertFalse(np.array_equal(indices, sorted_indices))
def get_all_shuffled_windowed():
global RANGE
from steves_utils.ORACLE.shuffled_dataset_accessor import Shuffled_Dataset_Factory
from steves_utils import utils
DATASET_BATCH_SIZE = 100
BATCH = 256
chunk_size = 4 * ORIGINAL_PAPER_SAMPLES_PER_CHUNK
STRIDE_SIZE = 1
NUM_REPEATS = math.floor(
(chunk_size - ORIGINAL_PAPER_SAMPLES_PER_CHUNK) / STRIDE_SIZE) + 1
path = os.path.join(utils.get_datasets_base_path(),
"all_shuffled_chunk-512", "output")
print(utils.get_datasets_base_path())
print(path)
datasets = Shuffled_Dataset_Factory(path,
train_val_test_splits=(0.6, 0.2, 0.2),
reshuffle_train_each_iteration=False)
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
train_ds = train_ds.unbatch().take(200000 * len(ALL_SERIAL_NUMBERS))
val_ds = val_ds.unbatch().take(10000 * len(ALL_SERIAL_NUMBERS))
test_ds = test_ds.unbatch().take(50000 * len(ALL_SERIAL_NUMBERS))
train_ds = train_ds.map(
lambda x: (x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
val_ds = val_ds.map(lambda x:
(x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
test_ds = test_ds.map(lambda x:
(x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
train_ds = train_ds.map(lambda x, y: (tf.transpose(
tf.signal.frame(x, ORIGINAL_PAPER_SAMPLES_PER_CHUNK, STRIDE_SIZE), [
1, 0, 2
]), tf.repeat(tf.reshape(y, (1, RANGE)), repeats=NUM_REPEATS, axis=0)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
# We aren't really windowing the val and test data, we are just splitting them into 128 sample chunks so that they are
# the same shape as the train data
val_ds = val_ds.map(
lambda x, y: (
tf.transpose(
# See, stride == length, meaning we are just splitting the chunks, not really windowing
tf.signal.frame(x, ORIGINAL_PAPER_SAMPLES_PER_CHUNK,
ORIGINAL_PAPER_SAMPLES_PER_CHUNK),
[1, 0, 2]),
tf.repeat(tf.reshape(y, (1, RANGE)),
repeats=math.floor(chunk_size /
ORIGINAL_PAPER_SAMPLES_PER_CHUNK),
axis=0)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
test_ds = test_ds.map(
lambda x, y: (
tf.transpose(
# See, stride == length, meaning we are just splitting the chunks, not really windowing
tf.signal.frame(x, ORIGINAL_PAPER_SAMPLES_PER_CHUNK,
ORIGINAL_PAPER_SAMPLES_PER_CHUNK),
[1, 0, 2]),
tf.repeat(tf.reshape(y, (1, RANGE)),
repeats=math.floor(chunk_size /
ORIGINAL_PAPER_SAMPLES_PER_CHUNK),
axis=0)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
train_ds = train_ds.unbatch().take(200000 * len(ALL_SERIAL_NUMBERS))
val_ds = val_ds.unbatch().take(10000 * len(ALL_SERIAL_NUMBERS))
test_ds = test_ds.unbatch().take(50000 * len(ALL_SERIAL_NUMBERS))
train_ds = train_ds.shuffle(DATASET_BATCH_SIZE * NUM_REPEATS * 3,
reshuffle_each_iteration=True)
train_ds = train_ds.batch(BATCH)
val_ds = val_ds.batch(BATCH)
test_ds = test_ds.batch(BATCH)
train_ds = train_ds.prefetch(100)
val_ds = val_ds.prefetch(100)
test_ds = test_ds.prefetch(100)
return train_ds, val_ds, test_ds
def get_windowed_foxtrot_shuffled():
from steves_utils.ORACLE.shuffled_dataset_accessor import Shuffled_Dataset_Factory
from steves_utils import utils
path = os.path.join(utils.get_datasets_base_path(), "foxtrot", "output")
datasets = Shuffled_Dataset_Factory(path,
train_val_test_splits=(0.6, 0.2, 0.2))
train_ds = datasets["train_ds"]
val_ds = datasets["val_ds"]
test_ds = datasets["test_ds"]
# count = 0
# for e in train_ds.concatenate(val_ds).concatenate(test_ds):
# count += e["IQ"].shape[0]
# print(count)
# sys.exit(1)
train_ds = train_ds.unbatch()
val_ds = val_ds.unbatch()
test_ds = test_ds.unbatch()
# Chunk size and batch is determined by the shuffled dataset
chunk_size = 4 * ORIGINAL_PAPER_SAMPLES_PER_CHUNK
STRIDE_SIZE = 1
BATCH = 1000
REBATCH = 500
NUM_REPEATS = math.floor(
(chunk_size - ORIGINAL_PAPER_SAMPLES_PER_CHUNK) / STRIDE_SIZE) + 1
# print(RANGE)
# sys.exit(1)
# serial_number_id ranges from [0,15]
# train_ds = train_ds.filter(lambda x: x["serial_number_id"] < 13 or x["serial_number_id"] > 13)
# val_ds = val_ds.filter(lambda x: x["serial_number_id"] < 13 or x["serial_number_id"] > 13)
# test_ds = test_ds.filter(lambda x: x["serial_number_id"] < 13 or x["serial_number_id"] > 13)
# train_ds = train_ds.filter(lambda x: x["serial_number_id"] != 13)
# val_ds = val_ds.filter(lambda x: x["serial_number_id"] != 13)
# test_ds = test_ds.filter(lambda x: x["serial_number_id"] != 13)
# train_ds = train_ds.filter(lambda x: x["serial_number_id"] < 15)
# val_ds = val_ds.filter(lambda x: x["serial_number_id"] < 15)
# test_ds = test_ds.filter(lambda x: x["serial_number_id"] < 15)
# val_ds = val_ds.filter(lambda x: x["serial_number_id"] in target_serials)
# test_ds = test_ds.filter(lambda x: x["serial_number_id"] in target_serials)
train_ds = train_ds.map(
lambda x: (x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
val_ds = val_ds.map(lambda x:
(x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
test_ds = test_ds.map(lambda x:
(x["IQ"], tf.one_hot(x["serial_number_id"], RANGE)),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
train_ds = train_ds.map(
lambda x, y: (
tf.transpose(
tf.signal.
frame(x, ORIGINAL_PAPER_SAMPLES_PER_CHUNK, STRIDE_SIZE
), # Somehow we get 9 frames from this
[1, 0, 2]),
tf.repeat(tf.reshape(y, (1, RANGE)), repeats=NUM_REPEATS, axis=0
) # Repeat our one hot tensor 9 times
),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
val_ds = val_ds.map(
lambda x, y: (
tf.transpose(
tf.signal.frame(x, ORIGINAL_PAPER_SAMPLES_PER_CHUNK,
ORIGINAL_PAPER_SAMPLES_PER_CHUNK
), # Somehow we get 9 frames from this
[1, 0, 2]),
tf.repeat(tf.reshape(y, (1, RANGE)),
repeats=math.floor(chunk_size /
ORIGINAL_PAPER_SAMPLES_PER_CHUNK),
axis=0) # Repeat our one hot tensor 9 times
),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
test_ds = test_ds.map(
lambda x, y: (
tf.transpose(
tf.signal.frame(x, ORIGINAL_PAPER_SAMPLES_PER_CHUNK,
ORIGINAL_PAPER_SAMPLES_PER_CHUNK
), # Somehow we get 9 frames from this
[1, 0, 2]),
tf.repeat(tf.reshape(y, (1, RANGE)),
repeats=math.floor(chunk_size /
ORIGINAL_PAPER_SAMPLES_PER_CHUNK),
axis=0) # Repeat our one hot tensor 9 times
),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
train_ds = train_ds.unbatch()
val_ds = val_ds.unbatch()
test_ds = test_ds.unbatch()
train_ds = train_ds.shuffle(BATCH * NUM_REPEATS * 4)
val_ds = val_ds.shuffle(BATCH * NUM_REPEATS * 4)
test_ds = test_ds.shuffle(BATCH * NUM_REPEATS * 4)
# for e in test_ds:
# print(e[1])
# sys.exit(1)
train_ds = train_ds.batch(REBATCH)
val_ds = val_ds.batch(REBATCH)
test_ds = test_ds.batch(REBATCH)
train_ds = train_ds.prefetch(100)
val_ds = val_ds.prefetch(100)
test_ds = test_ds.prefetch(100)
return train_ds, val_ds, test_ds
def __init__(
self,
input_shuffled_ds_dir,
input_shuffled_ds_num_samples_per_chunk,
output_batch_size,
output_max_file_size_MB,
seed,
num_windowed_examples_per_device,
num_val_examples_per_device,
num_test_examples_per_device,
output_window_size,
distances_to_filter_on,
serials_to_filter_on,
working_dir,
output_format_str,
stride_length,
# output_format_str="shuffled_batchSize-{batch_size}_part-{part}.tfrecord_ds",
) -> None:
self.serial_ids_to_filter_on = [serial_number_to_id(serial) for serial in serials_to_filter_on]
self.num_windowed_examples_per_device = num_windowed_examples_per_device
self.num_val_examples_per_device = num_val_examples_per_device
self.num_test_examples_per_device = num_test_examples_per_device
self.input_shuffled_ds_num_samples_per_chunk = input_shuffled_ds_num_samples_per_chunk
self.output_batch_size = output_batch_size
self.output_max_file_size_MB = output_max_file_size_MB
self.seed = seed
self.output_window_size = output_window_size
self.stride_length = stride_length
self.window_pile_dir = os.path.join(working_dir, "pile_train")
self.window_output_dir = os.path.join(working_dir, "train")
self.val_pile_dir = os.path.join(working_dir, "pile_val")
self.val_output_dir = os.path.join(working_dir, "val")
self.test_pile_dir = os.path.join(working_dir, "pile_test")
self.test_output_dir = os.path.join(working_dir, "test")
# If necessary we can customize these
self.window_output_format_str = output_format_str
self.val_output_format_str = output_format_str
self.test_output_format_str = output_format_str
self.num_devices = len(self.serial_ids_to_filter_on)
# Yeah it's pretty hacky since we don't really need to split the dataset into test and val, but
# it's already written and tested
datasets = Shuffled_Dataset_Factory(
input_shuffled_ds_dir, train_val_test_splits=(0.6, 0.2, 0.2), reshuffle_train_each_iteration=False
)
self.train_ds = datasets["train_ds"].unbatch()
self.val_ds = datasets["val_ds"].unbatch()
self.test_ds = datasets["test_ds"].unbatch()
self.og_datasets = {
"train_ds": self.train_ds,
"val_ds": self.val_ds,
"test_ds": self.test_ds
}
# Since we are windowing, the number of examples we take from the original dataset is smaller
# than the actual number of windows we want to generate
replication_factor = math.floor((input_shuffled_ds_num_samples_per_chunk - output_window_size)/stride_length + 1)
num_train_examples_to_get_per_device = math.ceil(num_windowed_examples_per_device/replication_factor)
# These are a little different. Since we are basically unchunking by striding as big as our output window
num_val_examples_per_device = math.ceil(num_val_examples_per_device / (input_shuffled_ds_num_samples_per_chunk/output_window_size))
num_test_examples_per_device = math.ceil(num_test_examples_per_device / (input_shuffled_ds_num_samples_per_chunk/output_window_size))
# print("Fetching {} Train Chunks Per Device".format(num_train_examples_to_get_per_device))
# print("Fetching {} Val Chunks Per Device".format(num_val_examples_per_device))
# print("Fetching {} Tess Chunks Per Device".format(num_test_examples_per_device))
# print("Replication Factor:", replication_factor)
self.train_ds = Windowed_Dataset_Shuffler.build_per_device_filtered_dataset(
distances_to_filter_on=distances_to_filter_on,
serial_ids_to_filter_on=self.serial_ids_to_filter_on,
num_examples_per_serial_id=num_train_examples_to_get_per_device,
ds=self.train_ds,
)
self.val_ds = Windowed_Dataset_Shuffler.build_per_device_filtered_dataset(
distances_to_filter_on=distances_to_filter_on,
serial_ids_to_filter_on=self.serial_ids_to_filter_on,
num_examples_per_serial_id=num_val_examples_per_device,
ds=self.val_ds,
)
self.test_ds = Windowed_Dataset_Shuffler.build_per_device_filtered_dataset(
distances_to_filter_on=distances_to_filter_on,
serial_ids_to_filter_on=self.serial_ids_to_filter_on,
num_examples_per_serial_id=num_test_examples_per_device,
ds=self.test_ds,
)
# print("Train Length Before Windowing:", utils.get_iterator_cardinality(self.train_ds))
# print("Val Length Before Windowing:", utils.get_iterator_cardinality(self.val_ds))
# print("Test Length Before Windowing:", utils.get_iterator_cardinality(self.test_ds))
self.train_ds = self.window_ds(self.train_ds, self.stride_length)
self.val_ds = self.window_ds(self.val_ds, output_window_size)
self.test_ds = self.window_ds(self.test_ds, output_window_size)
# self.train_ds = self.train_ds.batch(1000)
# self.val_ds = self.val_ds.batch(1000)
# self.test_ds = self.test_ds.batch(1000)
# print("Train Length:", utils.get_iterator_cardinality(self.train_ds))
# print("Val Length:", utils.get_iterator_cardinality(self.val_ds))
# print("Test Length:", utils.get_iterator_cardinality(self.test_ds))
# raise Exception("Done")
# This is another straight up hack. The val and test aren't really shuffled, we're just using this to write the DS to file
self.train_shuffler = self.make_train_shuffler()
self.val_shuffler = self.make_val_shuffler()
self.test_shuffler = self.make_test_shuffler()