def train_model(net, optimizer, criterion, scheduler, epochs):
minibatches_per_interval = 30
epochs_per_test_evaluation = 1
try:
net.load_state_dict(torch.load('wild_net.pth'))
except FileNotFoundError:
pass
# Loop over the dataset multiple times
for epoch in range(epochs):
running_loss = 0.0
for i, data in enumerate(trainloader):
# Get the inputs; data is a list of [inputs, labels]
inputs, labels = data
# Zero the parameter gradients
optimizer.zero_grad()
# Forward + backward + optimize
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
# Print training statistics and save model checkpoint every minibatches_per_interval
# loops, or at the end of the epoch
if (i + 1
) % minibatches_per_interval == 0 or i == len(trainloader) - 1:
if (i + 1) % minibatches_per_interval == 0:
print('[Epoch {0}, mini-batch {1}] loss: {2:.3f}'.format(
str(epoch + 1).rjust(2, ' '),
str(i + 1).rjust(5, ' '),
running_loss / minibatches_per_interval))
running_loss = 0.0
torch.save(net.state_dict(), 'wild_net.pth')
torch.save(trainloader, 'trainloader.pth')
# Print test accuracy (remove if just trying to train quickly)
if (epoch + 1) % epochs_per_test_evaluation == 0:
testModel.testModel()
#scheduler.step()
print('Finished Training')
def __init__(self):
# Objects
self.rw = comClient() #Initialise Modbus comms class
self.xls = xlsLogging(4) #Initialise excel data logging
self.pg = plotActiveGraph() #Initialise graphical plot
self.NNctrl = neuralNetwork() #Initialise Neural Network
if int(sys.argv[1]) == 1:
self.r = testModel("../../tests/") #Initialise simulated lab rig
# Variables
self.count = 0 #For 'heart beat' counter
self.sampleTime = 10
def __init__(self):
# Objects
self.ext = osTools() #Initialise data key press
self.xls = xlsLogging(6) #Initialise excel data logging
self.pg = plotActiveGraph() #Initialise graphical plot
self.ctrl = controller() #Initialise PID Controller
if int(sys.argv[1]) == 1:
self.r = testModel("../../tests/") #Initialise simulated lab rig
else:
self.rw = comClient() #Initialise Modbus comms class
# Variables
self.count = 0 #For 'heart beat' counter
def __init__(self):
# Objects
self.ext = osTools() #Initialise data key press
self.xls = xlsLogging(4) #Initialise excel data logging
self.pg = plotActiveGraph() #Initialise graphical plot
if int(sys.argv[1]) == 1:
self.r = testModel("../../tests/") #Initialise simulated lab rig
else:
self.rw = comClient() #Initialise Modbus comms class
# Variables
self.Interval = 5 #For Time Interval
self.count = 0 #'Heart beat' counter start number
def __init__(self):
# Objects
self.ext = osTools() #Initialise data key press
self.xls = xlsLogging(6) #Initialise excel data logging
self.pg = plotActiveGraph() #For graphical plot (PV,SP,OP)
self.rls = RLS() #Initialise Recursive Least Squares Object
if int(sys.argv[1]) == 1:
self.r = testModel("../../tests/") #Initialise simulated lab rig
else:
self.rw = comClient() #Initialise Modbus comms class
# Variables
self.count = 0 #For 'heart beat' counter
self.intDataPoints = 4 #Initial Data Points to be read
self.sampleTime = 40 #Calculation loop time
def main():
mdl = testModel("./")
graph = plotActiveGraph()
ext = osTools()
startTime = time.time()
Interval = 0.5
while(True):
mdl.readModel()
graph.dataUpdate((time.time() - startTime),\
mdl.getRegister(0),\
mdl.getRegister(2),\
mdl.getRegister(3))
if ext.kbdExit():
break
time.sleep(Interval) #Loop Interval
def main( argv ):
trainingDataPath = ''
testDataPath = ''
iterations = DEFAULT_ITERATIONS
rate = DEFAULT_RATE
regularization = DEFAULT_REGULARIZER
weightsPath = ""
global weightsFile
# Make sure the correct arguments have been passed
if len( argv ) < 3:
print( usage )
sys.exit( 2 )
try:
opts, args = getopt.getopt( argv, 't:s:i:r:l:w:h', ["training-data=","test-data=","iterations=","learning-rate=","regularizer=","save-weights="] )
except getopt.GetoptError:
print( usage )
sys.exit( 2 )
for opt, arg in opts:
if opt == '-h':
print( usage )
sys.exit()
elif opt in ( '-t', "training-data=" ):
trainingDataPath = arg
elif opt in ( '-s', "test-data=" ):
testDataPath = arg
elif opt in ( '-i', "iterations=" ):
iterations = int(arg)
elif opt in ( '-l', "rate=" ):
rate = float(arg)
elif opt in ( '-r', "regularizer=" ):
regularization = float(arg)
elif opt in ( '-w', "save-weights=" ):
weightsPath = str( arg )
#Make sure we got a test and training data path
if trainingDataPath == '' or testDataPath == '':
print( usage )
sys.exit( 2 )
if weightsPath != "":
weightsFile = open( weightsPath, 'a' )
#Get our test and training data
trainingDataMatrix = list();
trainingResponseVec = list();
testDataMatrix = list();
testResponseVec = list();
print( "Parsing training data..." )
getDataMatrix( trainingDataPath, trainingDataMatrix, trainingResponseVec );
print( "Parsing test data..." )
getDataMatrix( testDataPath, testDataMatrix, testResponseVec );
#Make sure all our points are normalized
for i in trainingDataMatrix:
for j in range( len( i ) ):
if i[j] < 0 or i[j] > 1:
print( "Training data point out of range [0,1] at index:",j,"with value:",i[j] )
sys.exit(2)
for i in testDataMatrix:
for j in range( len( i ) ):
if i[j] < 0 or i[j] > 1:
print( "Test data point out of range [0,1] at index:",j,"with value:",i[j] )
sys.exit(2)
print( "Running regression..." )
weights = trainModel( trainingDataMatrix, trainingResponseVec, rate, regularization, iterations, trainingCallback )
#Close weights file if it is open
if weightsFile != None:
weightsFile.close()
print( weights )
print( "Testing model..." )
testModel( testDataMatrix, testResponseVec, weights, testCallback )
PREPROCESSED_FOLDER_PATH = DATASETS_PATH + "UTKFace-Preprocessed-" + str(
MARGIN) + "/"
PREPROCESSED_IMAGES_PATH = PREPROCESSED_FOLDER_PATH + "Images/"
PREPROCESSED_CSV_PATH = PREPROCESSED_FOLDER_PATH + "CSVs/"
# preprocessDataset(MARGIN, DATASETS_PATH, PREPROCESSED_FOLDER_PATH, PREPROCESSED_IMAGES_PATH, PREPROCESSED_CSV_PATH)
# distributeDatset(PREPROCESSED_CSV_PATH, MIN_AGE, MAX_AGE, SOCIAL_MEDIA_SEGMENTS)
best_age_mae, best_ML = -1, (0, 0)
for M in M_array:
for L in L_array:
# Output Path
OUTPUT_FOLDER_NAME = "LabelDiversity-" + RESNET_SIZE + "-" + str(
M) + "X" + str(L)
OUT_PATH = "../TrainedModels/" + OUTPUT_FOLDER_NAME + "/"
validLoss = trainModel(PREPROCESSED_IMAGES_PATH,
PREPROCESSED_CSV_PATH, OUT_PATH,
RESNET_SIZE, MIN_AGE, MAX_AGE, M, L)
age_mae = testModel(PREPROCESSED_IMAGES_PATH,
PREPROCESSED_CSV_PATH, OUT_PATH,
SOCIAL_MEDIA_SEGMENTS, RESNET_SIZE, MIN_AGE,
MAX_AGE, M, L)
if age_mae < best_age_mae or best_age_mae == -1:
best_age_mae = age_mae
best_ML = (M, L)
print("Current ML:\t", (M, L), "\tCurrent MAE:\t", age_mae)
print("Best ML:\t", best_ML, "\tBest MAE:\t", best_age_mae)
def run_wav2(wav0):
conn = psycopg2.connect(
user="******",
password="******",
host="test-heroku.cs0ubczlcaiz.us-west-1.rds.amazonaws.com",
port="5432",
database="test-aws")
cur = conn.cursor()
print('\nProcessing Run Wave 2...\n')
# wav = os.path.join('data/new_data/fft/' + string2 + '.png')
# wav0 = os.listdir('data/new_data/')[0]
wav = wav0.strip('data/new_data/')
wav1 = wav.strip('.wav') + '.png'
# wav = os.listdir('data/new_data/')[0]
# wav = next(join('data/new_data/', f) for f in os.listdir('data/new_data/'))
# print('Wav0: \n', wav0)
# print('Wav: \n', wav)
# print('Wav1: \n', wav1)
word = wav.split('-')
# print('Word: \n', word)
mea_date = word[0]
# print('Measure Date: \n', mea_date)
disp_date = mea_date[0:4] + '-' + mea_date[4:6] + '-' + mea_date[
6:8] + ' ' + mea_date[8:10] + ':' + mea_date[10:12] + ':' + mea_date[
12:14]
# print('Display Date: \n', disp_date)
dev_id = word[1]
# print('Dev ID: \n', dev_id)
qr_code0 = word[3]
qr_code = qr_code0.strip('.wav')
# print('RQ Code: \n', qr_code)
loc = word[2]
# print('Location: \n', loc)
print('\nProcessing Prob...')
conf = testModel()
p # rint('Conf-2: ',conf)
# run_prob (dev_id)
print('\nFinished Prob.')
# path = '/Users/andy/Projects/flow-ez/data'
# print("Moving file:")
# print(os.listdir(path))
# source = '/Users/andy/Projects/flow-ez/data/new_data'
# destination = '/Users/andy/Projects/flow-ez/data/old_data'
# dest = shutil.move(source, destination, copy_function=copy2)
# print("Finished Moving file:")
# print(os.listdir(path))
# print("Destination path:", dest)
print('\nProcessing Pulse...')
plt.figure(dpi=2400)
my_pulse(wav0)
print('\nFinished Pulse.')
print('\nProcessing FFT...')
my_fft(wav0)
print('\nFinished FFT.')
if (conf > 50):
res = 'Abnormal'
else:
res = 'Normal'
first_name = names.get_first_name()
last_name = names.get_last_name()
shutil.copy('static/trend/20191101101100-003004802801-UP-00000001.png',
'static/trend/' + wav1)
cur.execute("BEGIN TRANSACTION;")
# cur.execute("Update data_table (mea_date,dev_id,qr_code,loc,pulse,fft,trend) VALUES (?,?,?,?,?,?,?)",(mea_date,dev_id,qr_code,loc,wav1,wav1,wav1) where (last_name='West'))
# cur.execute("UPDATE data_table set mea_date=?,disp_date=?,dev_id=?,qr_code=?,loc=?,pulse=?,fft=?,trend=?,res=?,prob=? WHERE dev_id=?", (mea_date,disp_date,dev_id,qr_code,loc,wav1,wav1,wav1,res,conf,dev_id,) )
cur.execute(
"INSERT INTO data_table (first_name,last_name,mea_date,disp_date,dev_id,qr_code,loc,pulse,fft,trend,res,prob) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(first_name, last_name, mea_date, disp_date, dev_id, qr_code, loc,
wav1, wav1, wav1, res, conf))
cur.execute("COMMIT TRANSACTION;")
cur.close
conn.close()
#source = os.listdir("/Users/andy/Projects/flow-ez/data/new_data/")
#destination = "/Users/andy/Projects/flow-ez/data/old_data/"
source = os.listdir("/AboveCare/Projects/flow-ez/data/new_data/")
destination = "/AboveCare/Projects/flow-ez/data/old_data/"
for files in source:
if files.endswith('.wav'):
shutil.move("/AboveCare/Projects/flow-ez/data/new_data/" + files,
destination)
print('\nWave Files Moved.')
# my_psp (wav0) ## Leadtek's data has "RuntimeWarning: divide by zero" issue
history = parallel_model.fit( X_train, y_train,
batch_size=batch_size,
epochs=epochs,
shuffle=True,
verbose=1,
initial_epoch=initial_epoch,
validation_data=(X_test, y_test),
callbacks=[tensorboard, checkpoint, early_stop, WeightsSaver(parallel_model, weight_backup_epoch), csv_logger, epoch_end_test])
else:
history = parallel_model.fit_generator( train_gen,
steps_per_epoch=steps_per_epoch_train,
validation_data=valid_gen,
validation_steps=steps_per_epoch_validation,
use_multiprocessing=True,
workers=4,
shuffle=True,
epochs=epochs,
verbose=1,
initial_epoch=initial_epoch,
callbacks=[tensorboard, checkpoint, early_stop, WeightsSaver(parallel_model, weight_backup_epoch), csv_logger])
# save model
parallel_model.save(MODEL_FILE_basename+'.h5')
print('Saved the trained model to : {}'.format(MODEL_FILE_basename+'.h5'))
# plot metrics
plot_metrics()
# test model
testModel(MODEL_FILE_basename+'.h5', dataset=dataset)
