def main():
gains = [40, 50, 60]
tx_beams = np.arange(0, 24)
num_samples_tot_gain_tx_beam = 10000
# Order is gain *
indexes = np.arange(
0,
num_samples_tot_gain_tx_beam * len(tx_beams) * len(gains)
)
batch_size = 32
data_path = '/media/michele/rx-12-tx-tm-0-rx-tm-1.h5'
num_blocks_per_frame = 15
how_many_blocks_per_frame = 1
num_samples_per_block = 2048
num_tx_beams = len(tx_beams)
input_size = 1024
dg = DataGenerator(
indexes,
batch_size,
data_path,
num_tx_beams,
num_blocks_per_frame,
input_size,
num_samples_per_block,
how_many_blocks_per_frame,
shuffle=False,
is_2d=False
)
batch_gain_tx_beam = num_samples_tot_gain_tx_beam / batch_size
# for [i_g, val_g] in enumerate(gains):
# print("Gain: " + str(val_g))
# for [i_t, val_t] in enumerate(tx_beams):
# print("Beam idx: " + str(val_t))
# batch_index = (i_g * len(tx_beams) * batch_gain_tx_beam) + i_t * batch_gain_tx_beam
# print("Batch idx: " + str(batch_index))
# [batch, batch_y] = dg.__getitem__(batch_index)
# print("tx_beam %d y % s" % (val_t, batch_y[0]))
# # print(batch_y[0])
for i in range(dg.__len__()):
print("Batch idx: " + str(i))
[batch, batch_y] = dg.__getitem__(i)
print("tx_beam %s %s y %s %s" % (batch[0][0], batch[-1][0], batch_y[0], batch_y[-1]))
print("batch_x_size: %s, batch_y_size: %s" % (str(batch.shape), str(batch_y.shape)))
def test_data_generator():
hdf5_file = os.path.join("..", "..", "data", "76_79_80.hdf5")
params = {"batch_size": 32, "shuffle": True, "n_classes": 2}
generator = DataGenerator(hdf5_file, "train", **params)
print("Batch size: ", generator.batch_size)
print("Data dim: ", generator.dim)
print("Classes: ", generator.n_classes)
print("Data number: ", generator.n_data)
print("list_IDs length: ", len(generator.list_IDs))
print("Training steps: ", len(generator))
steps = len(generator)
# mini batch not at the end of the dataset
dataset, labels = generator.__getitem__(steps - 2)
print("Data 1 shape: ", dataset[0].shape)
print("Data 32 shape: ", dataset[31].shape)
print("Label 1: ", labels[0])
print("Label 32: ", labels[31])
print("Data:", dataset)
try:
dataset[32]
except IndexError:
print("Dataset size is correct")
print()
# raise
# the last mini batch, size usually is smaller than batch_size
dataset, labels = generator.__getitem__(steps - 1)
print("Batch size is:", params["batch_size"])
print("The size of the last mini batch is:", len(dataset))
l = len(dataset)
print("Data 1 shape:", dataset[0].shape)
print("Data {} (last data) shape:".format(l), dataset[l - 1].shape)
print("Label 1:", labels[0])
print("Label {}:".format(l), labels[l - 1])
try:
dataset[l]
except IndexError:
print("Dataset size is correct")
class TestDataGen(unittest.TestCase):
def setUp(self):
_, self.df = combine_all_wavs_and_trans_from_csvs(
"data_dir/sample_librivox-test-clean.csv")
self.dg = DataGenerator(self.df, batch_size=10, epoch_length=10)
def tearDown(self):
del self.dg
# Data generator
def test_extract_features_and_pad(self):
indexes = np.arange(5)
x_data_raw, y_data_raw, sr = load_audio(self.df,
indexes_in_batch=indexes)
x_data, input_length = self.dg.extract_features_and_pad(x_data_raw, sr)
self.assertEqual(x_data.shape, (5, 382, 26))
self.assertEqual(len(input_length), 5)
self.assertLessEqual(all(input_length), 382)
def test_get_seq_size(self):
x_data_raw, _, sr = load_audio(self.df, indexes_in_batch=[0])
size = self.dg.get_seq_size(x_data_raw[0], sr)
self.assertEqual(size, 256)
def test_get_item(self):
batch0, _ = self.dg.__getitem__(0)
batch1, _ = self.dg.__getitem__(1)
x_data0 = batch0.get("the_input")
x_data1 = batch1.get("the_input")
y_data0 = batch0.get("the_labels")
y_data1 = batch1.get("the_labels")
input_length = batch0.get("input_length")
label_length = batch0.get("label_length")
self.assertTupleEqual(x_data0.shape, (10, 494, 26))
self.assertTupleEqual(x_data1.shape, (9, 1514, 26))
self.assertEqual(y_data0.shape[0], 10)
self.assertEqual(y_data1.shape[0], 9)
self.assertEqual(input_length.shape[0], 10)
self.assertEqual(label_length.shape[0], 10)
# Feature generation utils
def test_load_audio(self):
indexes = np.arange(5)
x_data_raw, y_data_raw, sr = load_audio(self.df,
indexes_in_batch=indexes)
self.assertEqual(len(x_data_raw), 5)
self.assertEqual(len(y_data_raw), 5)
def test_extract_mfcc(self):
x_data_raw, _, sr = load_audio(self.df, indexes_in_batch=[0])
mfcc_padded, x_length = extract_mfcc_and_pad(x_data_raw[0],
sr=sr,
max_pad_length=500,
frame_length=320,
hop_length=160,
mfcc_features=26,
n_mels=40)
self.assertTupleEqual(mfcc_padded.shape, (500, 26))
self.assertEqual(x_length, 256)
def test_extract_mel_spec(self):
x_data_raw, _, sr = load_audio(self.df, indexes_in_batch=[0])
mel_spec, x_length = extract_mel_spectrogram_and_pad(
x_data_raw[0],
sr=sr,
max_pad_length=500,
frame_length=320,
hop_length=160,
n_mels=40)
self.assertTupleEqual(mel_spec.shape, (500, 40))
self.assertEqual(x_length, 256)
def test_convert_transcripts(self):
_, y_data_raw, sr = load_audio(self.df, indexes_in_batch=[0])
transcript, y_length = convert_and_pad_transcripts(y_data_raw)
exp = [
23., 5., 18., 5., 0., 9., 0., 2., 21., 20., 0., 1., 12., 18., 5.,
1., 4., 25., 0., 15., 14., 0., 20., 8., 5., 0., 3., 1., 18., 20.
]
list = transcript[0].tolist()
self.assertListEqual(list, exp)
self.assertEqual(y_length, 30)
validation_generator.on_epoch_end()
else:
validation_generator = DataGenerator(
directory="/home/helle246/data/pancreas/testing",
shape=(512, 512),
img_channels=1,
lbl_channels=2,
flat_labels=True,
batch_size=20,
tvl=False,
ds='psd')
validation_generator.on_epoch_end()
for i in range(iterations):
x, y = validation_generator.__getitem__(i)
eval_results = model.evaluate(x, y)
#print(model.metrics_names)
#print(eval_results)
dices.append(eval_results[3])
recalls.append(eval_results[2])
precisions.append(eval_results[1])
y_pred = model.predict(x)
# np.save("x.npy", x)
# np.save("pred.npy", np.greater(np.reshape(y_pred[:,:,0],(20,512,512,1)), 0.5))
# np.save("true.npy", np.reshape(y[:,:,[0]], (20,512,512,1)))
y = y.flatten()
from DataGenerator import DataGenerator
training_generator = DataGenerator('../Data/ModelNet30/train_files.txt',
batch_size=32,
dim=(30, 30, 30),
n_classes=31,
shuffle=True)
training_generator.__getitem__(0)
datadirs=paths,
is_label_to_categorical=False,
is_normalize_image_datatype=True,
is_apply_text_preprocessing=FLAGS.is_apply_text_preprocessing,
is_apply_sequence_preprocessing=FLAGS.is_apply_sequence_preprocessing)
n_clusters = FLAGS.n_classes
print("n_clusters")
print(n_clusters)
kmeans = KMeans(n_clusters=n_clusters, n_init=20,
batch_size=FLAGS.batch_size) #random_state=0, #, n_jobs=4)
y_pred_kmeans = kmeans
if True: #False: #ToDo temp
for bi in range(0, data.__len__()):
x, y = data.__getitem__(bi, True, is_return_only_x=False)
y_pred_kmeans = y_pred_kmeans.partial_fit(x[:, :])
##
print("y ", y.shape, y_pred_kmeans.labels_.shape)
#print( metrics.acc(y, y_pred_kmeans.labels_) )
else:
x, y = data.__getitem__(0, True, is_return_only_x=False)
#to speed up moved out side loop, will it going to change anything with efficiency of the algorithm
#print( metrics.acc(y, y_pred_kmeans.labels_) ) #padd entire y
#dims = [x.shape[-1], 500, 500, 2000, 10]
#dims = [x.shape[-1], 500, 500, 600, 10]
dims = [x.shape[-1], 500, 500, 600, 234]
init = VarianceScaling(scale=1. / 3., mode='fan_in', distribution='uniform')
# Shuffle IDs
from sklearn.utils import shuffle
list_IDs_shu, labels_shu = shuffle(list_IDs, labels, random_state=None)
from DataGenerator import DataGenerator
params = {
'dim': (128, 128),
'batch_size': len(labels),
'n_classes': 91,
'n_channels': 1,
'shuffle': True
}
generator = DataGenerator(list_IDs_shu, labels_shu, **params)
x, y = generator.__getitem__(0)
pred = model.predict(x)
pred_en = encoder.predict(x)
pred_255 = pred * 255
x_255 = x * 255
from PIL import Image
for i in range(0, 10):
inp = np.zeros((128, 128, 3), dtype=np.uint8)
inp[:, :, 0] = pred_255[i][:, :, 0]
inp[:, :, 1] = pred_255[i][:, :, 0]
inp[:, :, 2] = pred_255[i][:, :, 0]
# print(np.shape(inp))
# print(inp)
img = Image.fromarray(inp)
output_feature_indecis = [0, 1]
data_generator = DataGenerator(path_name,
file_name,
list_IDs=train_IDs,
batch_size=batch_size,
hist_len=hist_len,
pred_len=pred_len,
input_feature_indecis=input_feature_indecis,
output_feature_indecis=output_feature_indecis,
shuffle=True,
return_output_data=False,
return_label=True)
data_generator.on_epoch_end()
input_data, label = data_generator.__getitem__(0)
input_data.shape
num_input_timesteps = input_data.shape[1]
num_input_features = input_data.shape[2]
# setup logger
FORMAT = '%(asctime)s %(process)d %(message)s'
logging.basicConfig(format=FORMAT, stream=sys.stdout, level=logging.INFO)
logger = logging.info
model_dir = 'md_categorical_' + model_name + \
'_neighbors' + str(neighbors) + \
'_hist_len' + str(hist_len) + \
'_pred_len' + str(pred_len) + \
'_layers' + str(num_layers) + \