def main(_):
uscLogger = USCLogger()
uscData = USCData(uscLogger.logger)
uscData.findListOfYoutubeDataFiles()
with tf.Session() as session:
session.run(tf.global_variables_initializer())
uscAutoEncoder = USCAutoEncoder(session, uscLogger, uscData)
if not uscAutoEncoder.isAlreadyTrained():
for trainingIterationNo in range(
uscData.youtube_data_max_category_data_file_count * 2):
uscLogger.logger.info("AutoEncoder Training Iteration No: " +
str(trainingIterationNo))
current_youtube_data_as_list = uscData.loadNextYoutubeData()
uscLogger.logAutoEncoderStepStart(session, trainingIterationNo)
prepareDataTimes = []
trainingTimes = []
trainingLosses = []
for current_batch_counter in range(
math.floor(
len(current_youtube_data_as_list) /
uscData.mini_batch_size)):
batch_data = current_youtube_data_as_list[
current_batch_counter *
uscData.mini_batch_size:(current_batch_counter + 1) *
uscData.mini_batch_size]
#uscLogger.logger.info("batch_data.shape: "+str(batch_data.shape))
trainingTime, trainingLoss, prepareDataTime = uscAutoEncoder.train(
batch_data)
trainingTimes.append(trainingTime)
trainingLosses.append(trainingLoss)
prepareDataTimes.append(prepareDataTime)
uscLogger.logAutoEncoderStepEnd(session, prepareDataTimes,
trainingTimes, trainingLosses,
trainingIterationNo)
else:
uscLogger.logger.info(
"No need to train AutoEncoder, already trained ... ")
uscData.prepareData()
uscModel = USCModel(session, uscLogger, uscData, uscAutoEncoder)
for trainingIterationNo in range(uscModel.training_iterations):
uscLogger.logStepStart(session, trainingIterationNo)
prepareDataTimes = []
trainingTimes = []
trainingAccuracies = []
testTimes = []
testAccuracies = []
for fold in np.random.permutation(uscData.fold_dirs):
#uscLogger.logger.info(" Fold : "+str(fold))
current_fold_data = uscData.get_fold_data(fold)
for current_batch_counter in range(
int(current_fold_data.shape[0] /
uscData.mini_batch_size)):
if (
current_batch_counter + 1
) * uscData.mini_batch_size <= current_fold_data.shape[0]:
batch_data = current_fold_data[
current_batch_counter *
uscData.mini_batch_size:(current_batch_counter +
1) *
uscData.mini_batch_size, :]
else:
batch_data = current_fold_data[
current_batch_counter *
uscData.mini_batch_size:, :]
if fold == "fold10":
## FOLD10 is reserved for testing
#uscLogger.logger.info(" Testing Batch Counter : "+str(current_batch_counter))
testTime, testAccuracy = uscModel.test(batch_data)
testTimes.append(testTime)
testAccuracies.append(testAccuracy)
else:
#uscLogger.logger.info(" Training Batch Counter : "+str(current_batch_counter))
trainingTime, trainingAccuracy, prepareDataTime = uscModel.train(
batch_data)
trainingTimes.append(trainingTime)
trainingAccuracies.append(trainingAccuracy)
prepareDataTimes.append(prepareDataTime)
uscLogger.logStepEnd(session, prepareDataTimes, trainingTimes,
trainingAccuracies, testTimes, testAccuracies,
trainingIterationNo)
def main(_):
uscLogger = USCLogger()
uscData = USCData(uscLogger.logger)
uscModel = USCModel(uscLogger, uscData)
for trainingIterationNo in range(uscModel.training_iterations):
epoch_logs = uscLogger.getNewLogDictionary()
uscLogger.logStepStart(trainingIterationNo)
mode = 'Training'
### YOUTUBE DATA
#if trainingIterationNo % 5 == 0 and uscLogger.lastAccuracy > 0.6 :
if (trainingIterationNo + 1) % 20000 == 0:
while uscData.current_youtube_data is None:
uscLogger.logger.info(
'Waiting 5 seconds for youtube data loader thread ....')
time.sleep(5)
current_youtube_data_as_list = uscData.getNextYoutubeData()
for current_batch_counter in range(
math.floor(
len(current_youtube_data_as_list) /
uscData.mini_batch_size)):
stage_logs = uscLogger.getNewLogDictionary()
batch_data = np.random.permutation(
current_youtube_data_as_list[current_batch_counter *
uscData.mini_batch_size:
(current_batch_counter + 1) *
uscData.mini_batch_size])
if uscLogger.lastAccuracy > 1: ## disable this branch >1
batch_data = uscModel.setPredictedLabel(
batch_data, np.ones((uscData.mini_batch_size, 1, 1)))
logData = uscModel.train(
batch_data, np.ones((uscData.mini_batch_size, 1, 1)))
uscLogger.appendLogData(stage_logs[mode], logData, False,
mode)
uscLogger.appendLogData(epoch_logs[mode], logData, False,
mode)
else:
logData = uscModel.train(
batch_data, np.zeros((uscData.mini_batch_size, 1, 1)))
uscLogger.appendLogData(stage_logs[mode], logData, True,
mode)
uscLogger.appendLogData(epoch_logs[mode], logData, True,
mode)
if current_batch_counter % 50 == 0:
uscLogger.logStepEnd(
'YoutubeData-' + str(current_batch_counter), mode,
stage_logs, trainingIterationNo)
### YOUTUBE DATA
for fold in np.random.permutation(uscData.fold_dirs):
stage_logs = uscLogger.getNewLogDictionary()
current_fold_data = np.random.permutation(
uscData.get_fold_data(fold))
for current_batch_counter in range(
int(current_fold_data.shape[0] / uscData.mini_batch_size)):
if (current_batch_counter + 1
) * uscData.mini_batch_size <= current_fold_data.shape[0]:
batch_data = current_fold_data[
current_batch_counter *
uscData.mini_batch_size:(current_batch_counter + 1) *
uscData.mini_batch_size, :]
else:
batch_data = current_fold_data[current_batch_counter *
uscData.mini_batch_size:, :]
if fold == "fold10":
mode = 'Testing'
## FOLD10 is reserved for testing
logData = uscModel.test(
batch_data, np.ones((uscData.mini_batch_size, 1, 1)))
else:
mode = 'Training'
logData = uscModel.train(
batch_data, np.ones((uscData.mini_batch_size, 1, 1)))
uscLogger.appendLogData(stage_logs[mode], logData, False, mode)
uscLogger.appendLogData(epoch_logs[mode], logData, False, mode)
uscLogger.logStepEnd(fold, mode, stage_logs, trainingIterationNo)
uscLogger.logStepEnd('SUMMARY', mode, epoch_logs, trainingIterationNo)
print(uscLogger.lastAccuracy)
def main(_):
continue_training=""
if len(sys.argv) > 1 :
continue_training=sys.argv[1]
uscLogger=USCLogger()
uscData=USCData(uscLogger.logger)
uscData.findListOfYoutubeDataFiles()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as session:
session.run(tf.global_variables_initializer())
uscAutoEncoder=USCAutoEncoder(session,uscLogger,uscData)
if uscAutoEncoder.alreadyTrained and continue_training!="contiune_training":
uscLogger.logger.info("AutoEncoder is already trained by-passing this phase ... ")
else:
#youtube data also contains urban sound data , replicated 10 times
for trainingIterationNo in range(uscData.youtube_data_max_category_data_file_count*10):
#for trainingIterationNo in range(uscData.youtube_data_max_category_data_file_count):
uscLogger.logger.info("AutoEncoder Training Iteration No: "+str(trainingIterationNo))
current_youtube_data_as_list=np.random.permutation(uscData.loadNextYoutubeData())
uscLogger.logAutoEncoderStepStart(session,trainingIterationNo)
prepareDataTimes=[ ]
trainingTimes=[ ]
trainingLosses=[ ]
trainingAccuracies=[ ]
for current_batch_counter in range(math.floor(len(current_youtube_data_as_list)/uscData.mini_batch_size)) :
batch_data=current_youtube_data_as_list[current_batch_counter*uscData.mini_batch_size:(current_batch_counter+1)*uscData.mini_batch_size]
#uscLogger.logger.info("batch_data.shape: "+str(batch_data.shape))
trainingTime,trainingLoss,accuracy,prepareDataTime=uscAutoEncoder.train(batch_data)
trainingTimes.append(trainingTime)
trainingLosses.append(trainingLoss)
prepareDataTimes.append(prepareDataTime)
trainingAccuracies.append(accuracy)
uscLogger.logAutoEncoderStepEnd(session,prepareDataTimes,trainingTimes,trainingLosses,trainingAccuracies,trainingIterationNo)
### AUTO ENCODER RAINING FINISHED, STARTING TO TRAIN MODEL
uscData.prepareData()
uscModel=USCModel(session,uscLogger,uscData,uscAutoEncoder)
for trainingIterationNo in range(uscModel.training_iterations):
uscLogger.logStepStart(session,trainingIterationNo)
prepareDataTimes=[ ]
trainingTimes=[ ]
trainingAccuracies=[ ]
testTimes=[ ]
testAccuracies=[ ]
for fold in np.random.permutation(uscData.fold_dirs):
if fold == "fold10":
uscLogger.logger.info("Testing Fold : "+fold)
else :
uscLogger.logger.info("Training Fold : "+fold)
#uscLogger.logger.info(" Fold : "+str(fold))
current_fold_data=uscData.get_fold_data(fold)
for current_batch_counter in range(int(current_fold_data.shape[0]/uscData.mini_batch_size)) :
if (current_batch_counter+1)*uscData.mini_batch_size <= current_fold_data.shape[0] :
batch_data=current_fold_data[current_batch_counter*uscData.mini_batch_size:(current_batch_counter+1)*uscData.mini_batch_size,:]
else:
batch_data=current_fold_data[current_batch_counter*uscData.mini_batch_size:,:]
if fold == "fold10":
## FOLD10 is reserved for testing
#uscLogger.logger.info(" Testing Batch Counter : "+str(current_batch_counter))
testTime,testAccuracy=uscModel.test(batch_data)
testTimes.append(testTime)
testAccuracies.append(testAccuracy)
else:
#uscLogger.logger.info(" Training Batch Counter : "+str(current_batch_counter))
trainingTime,trainingAccuracy,prepareDataTime=uscModel.train(batch_data)
trainingTimes.append(trainingTime)
trainingAccuracies.append(trainingAccuracy)
prepareDataTimes.append(prepareDataTime)
uscLogger.logStepEnd(session,prepareDataTimes,trainingTimes,trainingAccuracies,testTimes,testAccuracies,trainingIterationNo)
def main(_):
uscLogger = USCLogger()
uscData = USCData(uscLogger.logger)
uscModel = USCModel(uscLogger, uscData)
uscData.prepareData()
uscData.findListOfYoutubeDataFiles()
for trainingIterationNo in range(uscModel.training_iterations):
epoch_logs = uscLogger.getNewLogDictionary()
## youtube dta also contain fold1-9 of urban sound data
current_youtube_data_as_list = np.random.permutation(
uscData.loadNextYoutubeData())
uscLogger.logStepStart(trainingIterationNo)
mode = 'Training'
for current_batch_counter in range(
math.floor(
len(current_youtube_data_as_list) /
uscData.mini_batch_size)):
stage_logs = uscLogger.getNewLogDictionary()
batch_data = np.random.permutation(
current_youtube_data_as_list[current_batch_counter *
uscData.mini_batch_size:
(current_batch_counter + 1) *
uscData.mini_batch_size])
logData = uscModel.train(batch_data)
uscLogger.appendLogData(stage_logs[mode], logData)
uscLogger.appendLogData(epoch_logs[mode], logData)
if current_batch_counter % 50 == 0:
uscLogger.logStepEnd(
'YoutubeData-' + str(current_batch_counter), mode,
stage_logs, trainingIterationNo)
for fold in np.random.permutation(uscData.fold_dirs):
stage_logs = uscLogger.getNewLogDictionary()
current_fold_data = np.random.permutation(
uscData.get_fold_data(fold))
for current_batch_counter in range(
int(current_fold_data.shape[0] / uscData.mini_batch_size)):
if (current_batch_counter + 1
) * uscData.mini_batch_size <= current_fold_data.shape[0]:
batch_data = current_fold_data[
current_batch_counter *
uscData.mini_batch_size:(current_batch_counter + 1) *
uscData.mini_batch_size, :]
else:
batch_data = current_fold_data[current_batch_counter *
uscData.mini_batch_size:, :]
if fold == "fold10":
mode = 'Testing'
## FOLD10 is reserved for testing
logData = uscModel.test(batch_data)
else:
mode = 'Training'
logData = uscModel.train(batch_data)
uscLogger.appendLogData(stage_logs[mode], logData)
uscLogger.appendLogData(epoch_logs[mode], logData)
uscLogger.logStepEnd(fold, mode, stage_logs, trainingIterationNo)
uscLogger.logStepEnd('SUMMARY', mode, epoch_logs, trainingIterationNo)
def main(_):
uscLogger = USCLogger()
uscData = USCData(uscLogger.logger)
uscCochlea = USCCochlea(uscLogger, uscData)
uscModel = USCModel(uscLogger, uscData, uscCochlea)
for trainingIterationNo in range(uscCochlea.training_iterations):
epoch_logs = uscLogger.getNewLogDictionary()
uscLogger.logStepStart(trainingIterationNo)
mode = 'Training'
### TRAIN COCHLEA WITH YOUTUBE DATA
if (trainingIterationNo + 1) % 10 == 0:
mode = 'Testing'
while uscData.current_youtube_data is None:
uscLogger.logger.info(
'Waiting 5 seconds for youtube data loader thread ....')
time.sleep(5)
current_youtube_data_as_list = uscData.getNextYoutubeData()
for current_batch_counter in range(
math.floor(
len(current_youtube_data_as_list) /
uscData.mini_batch_size)):
uscLogger.logger.info(
str(current_batch_counter) + "/" + str(
math.floor(
len(current_youtube_data_as_list) /
uscData.mini_batch_size)))
stage_logs = uscLogger.getNewLogDictionary()
batch_data = np.random.permutation(
current_youtube_data_as_list[current_batch_counter *
uscData.mini_batch_size:
(current_batch_counter + 1) *
uscData.mini_batch_size])
if mode == 'Testing':
logData = uscCochlea.test(batch_data)
uscLogger.appendLogData(stage_logs[mode], logData, mode)
uscLogger.appendLogData(epoch_logs[mode], logData, mode)
#uscLogger.logStepEnd('YoutubeData-'+str(current_batch_counter),mode,stage_logs,trainingIterationNo)
else:
logData = uscCochlea.train(batch_data)
uscLogger.appendLogData(stage_logs[mode], logData, mode)
uscLogger.appendLogData(epoch_logs[mode], logData, mode)
uscLogger.logStepEnd('YoutubeData-' + str(current_batch_counter),
mode, stage_logs, trainingIterationNo)
uscLogger.logStepEnd('SUMMARY', mode, epoch_logs, trainingIterationNo)
### YOUTUBE DATA
for trainingIterationNo in range(uscModel.training_iterations):
epoch_logs = uscLogger.getNewLogDictionary()
uscLogger.logStepStart(trainingIterationNo)
mode = 'Training'
for fold in np.random.permutation(uscData.fold_dirs):
stage_logs = uscLogger.getNewLogDictionary()
current_fold_data = np.random.permutation(
uscData.get_fold_data(fold))
for current_batch_counter in range(
int(current_fold_data.shape[0] / uscData.mini_batch_size)):
if (current_batch_counter + 1
) * uscData.mini_batch_size <= current_fold_data.shape[0]:
batch_data = current_fold_data[
current_batch_counter *
uscData.mini_batch_size:(current_batch_counter + 1) *
uscData.mini_batch_size, :]
else:
batch_data = current_fold_data[current_batch_counter *
uscData.mini_batch_size:, :]
#self.play(self.augment_echo(x_data[5],2.5))
#plt.plot(batch_data[9]*100)
#plt.show()
#uscData.play(batch_data[9]*100)
#sys.exit(0)
if fold == "fold10":
mode = 'Testing'
## FOLD10 is reserved for testing
logData = uscModel.test(batch_data)
else:
mode = 'Training'
logData = uscModel.train(batch_data)
uscLogger.appendLogData(stage_logs[mode], logData, False, mode)
uscLogger.appendLogData(epoch_logs[mode], logData, False, mode)
uscLogger.logStepEnd(fold, mode, stage_logs, trainingIterationNo)
uscLogger.logStepEnd('SUMMARY', mode, epoch_logs, trainingIterationNo)
print(uscLogger.lastAccuracy)
def main(_):
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
uscLogger = USCLogger()
uscData = USCData(uscLogger.logger)
uscModel = USCModel(uscLogger, uscData)
#config = tf.compat.v1.ConfigProto
#config.gpu_options.per_process_gpu_memory_fraction = 0.9
#tf.keras.backend.set_session(tf.Session(config=config));
for trainingIterationNo in range(uscModel.training_iterations):
epoch_logs = uscLogger.getNewLogDictionary()
uscLogger.logStepStart(trainingIterationNo)
mode = 'Training'
# ### YOUTUBE DATA
# #if trainingIterationNo % 5 == 0 and uscLogger.lastAccuracy > 0.6 :
# #if (trainingIterationNo+1) % 20 == 0 :
# if (trainingIterationNo+1) % 100000 == 0 : ## never use youtube data
# while uscData.current_youtube_data is None:
# uscLogger.logger.info('Waiting 5 seconds for youtube data loader thread ....')
# time.sleep(5)
# current_youtube_data_as_list=uscData.getNextYoutubeData()
#
# for current_batch_counter in range(math.floor(len(current_youtube_data_as_list)/uscData.mini_batch_size)) :
#
# stage_logs=uscLogger.getNewLogDictionary()
# batch_data=np.random.permutation(current_youtube_data_as_list[current_batch_counter*uscData.mini_batch_size:(current_batch_counter+1)*uscData.mini_batch_size])
#
# if uscLogger.lastAccuracy > 1 : ## disable this branch >1
# batch_data=uscModel.setPredictedLabel(batch_data,np.ones((uscData.mini_batch_size,1,1)))
# logData = uscModel.train(batch_data,np.ones((uscData.mini_batch_size,1,1)))
# uscLogger.appendLogData(stage_logs[mode],logData,False,mode)
# uscLogger.appendLogData(epoch_logs[mode],logData,False,mode)
# else :
# logData = uscModel.train(batch_data,np.zeros((uscData.mini_batch_size,1,1)))
# uscLogger.appendLogData(stage_logs[mode],logData,True,mode)
# uscLogger.appendLogData(epoch_logs[mode],logData,True,mode)
#
# if current_batch_counter % 50 == 0:
# uscLogger.logStepEnd('YoutubeData-'+str(current_batch_counter),mode,stage_logs,trainingIterationNo)
#
# ### YOUTUBE DATA
stage_logs = uscLogger.getNewLogDictionary()
current_fold_data = np.random.permutation(
uscData.get_fold_data("training"))
for current_batch_counter in range(
int(current_fold_data.shape[0] / uscData.mini_batch_size)):
if (current_batch_counter +
1) * uscData.mini_batch_size <= current_fold_data.shape[0]:
batch_data = current_fold_data[current_batch_counter *
uscData.mini_batch_size:
(current_batch_counter + 1) *
uscData.mini_batch_size, :]
else:
batch_data = current_fold_data[current_batch_counter *
uscData.mini_batch_size:, :]
#self.play(self.augment_echo(x_data[5],2.5))
#plt.plot(batch_data[9]*100)
#plt.show()
#uscData.play(batch_data[9]*100)
#sys.exit(0)
mode = 'Training'
logData = uscModel.train(batch_data)
uscLogger.appendLogData(stage_logs[mode], logData, False, mode)
uscLogger.appendLogData(epoch_logs[mode], logData, False, mode)
if current_batch_counter % 50 == 0:
uscLogger.logStepEnd("Training-" + str(current_batch_counter),
mode, stage_logs, trainingIterationNo)
current_fold_data = np.random.permutation(
uscData.get_fold_data("test"))
for current_batch_counter in range(
int(current_fold_data.shape[0] / uscData.mini_batch_size)):
if (current_batch_counter +
1) * uscData.mini_batch_size <= current_fold_data.shape[0]:
batch_data = current_fold_data[current_batch_counter *
uscData.mini_batch_size:
(current_batch_counter + 1) *
uscData.mini_batch_size, :]
else:
batch_data = current_fold_data[current_batch_counter *
uscData.mini_batch_size:, :]
#self.play(self.augment_echo(x_data[5],2.5))
#plt.plot(batch_data[9]*100)
#plt.show()
#uscData.play(batch_data[9]*100)
#sys.exit(0)
mode = 'Testing'
logData = uscModel.test(batch_data)
uscLogger.appendLogData(stage_logs[mode], logData, False, mode)
uscLogger.appendLogData(epoch_logs[mode], logData, False, mode)
#if current_batch_counter % 50 == 0:
# uscLogger.logStepEnd("Testing-"+str(current_batch_counter),mode,stage_logs,trainingIterationNo)
uscLogger.logStepEnd('SUMMARY', mode, epoch_logs, trainingIterationNo)
