def train_model(net, optimizer, criterion, scheduler, epochs): minibatches_per_interval = 30 epochs_per_test_evaluation = 1 try: net.load_state_dict(torch.load('wild_net.pth')) except FileNotFoundError: pass # Loop over the dataset multiple times for epoch in range(epochs): running_loss = 0.0 for i, data in enumerate(trainloader): # Get the inputs; data is a list of [inputs, labels] inputs, labels = data # Zero the parameter gradients optimizer.zero_grad() # Forward + backward + optimize outputs = net(inputs) loss = criterion(outputs, labels) loss.backward() optimizer.step() running_loss += loss.item() # Print training statistics and save model checkpoint every minibatches_per_interval # loops, or at the end of the epoch if (i + 1 ) % minibatches_per_interval == 0 or i == len(trainloader) - 1: if (i + 1) % minibatches_per_interval == 0: print('[Epoch {0}, mini-batch {1}] loss: {2:.3f}'.format( str(epoch + 1).rjust(2, ' '), str(i + 1).rjust(5, ' '), running_loss / minibatches_per_interval)) running_loss = 0.0 torch.save(net.state_dict(), 'wild_net.pth') torch.save(trainloader, 'trainloader.pth') # Print test accuracy (remove if just trying to train quickly) if (epoch + 1) % epochs_per_test_evaluation == 0: testModel.testModel() #scheduler.step() print('Finished Training')
def __init__(self): # Objects self.rw = comClient() #Initialise Modbus comms class self.xls = xlsLogging(4) #Initialise excel data logging self.pg = plotActiveGraph() #Initialise graphical plot self.NNctrl = neuralNetwork() #Initialise Neural Network if int(sys.argv[1]) == 1: self.r = testModel("../../tests/") #Initialise simulated lab rig # Variables self.count = 0 #For 'heart beat' counter self.sampleTime = 10
def __init__(self): # Objects self.ext = osTools() #Initialise data key press self.xls = xlsLogging(6) #Initialise excel data logging self.pg = plotActiveGraph() #Initialise graphical plot self.ctrl = controller() #Initialise PID Controller if int(sys.argv[1]) == 1: self.r = testModel("../../tests/") #Initialise simulated lab rig else: self.rw = comClient() #Initialise Modbus comms class # Variables self.count = 0 #For 'heart beat' counter
def __init__(self): # Objects self.ext = osTools() #Initialise data key press self.xls = xlsLogging(4) #Initialise excel data logging self.pg = plotActiveGraph() #Initialise graphical plot if int(sys.argv[1]) == 1: self.r = testModel("../../tests/") #Initialise simulated lab rig else: self.rw = comClient() #Initialise Modbus comms class # Variables self.Interval = 5 #For Time Interval self.count = 0 #'Heart beat' counter start number
def __init__(self): # Objects self.ext = osTools() #Initialise data key press self.xls = xlsLogging(6) #Initialise excel data logging self.pg = plotActiveGraph() #For graphical plot (PV,SP,OP) self.rls = RLS() #Initialise Recursive Least Squares Object if int(sys.argv[1]) == 1: self.r = testModel("../../tests/") #Initialise simulated lab rig else: self.rw = comClient() #Initialise Modbus comms class # Variables self.count = 0 #For 'heart beat' counter self.intDataPoints = 4 #Initial Data Points to be read self.sampleTime = 40 #Calculation loop time
def main(): mdl = testModel("./") graph = plotActiveGraph() ext = osTools() startTime = time.time() Interval = 0.5 while(True): mdl.readModel() graph.dataUpdate((time.time() - startTime),\ mdl.getRegister(0),\ mdl.getRegister(2),\ mdl.getRegister(3)) if ext.kbdExit(): break time.sleep(Interval) #Loop Interval
def main( argv ): trainingDataPath = '' testDataPath = '' iterations = DEFAULT_ITERATIONS rate = DEFAULT_RATE regularization = DEFAULT_REGULARIZER weightsPath = "" global weightsFile # Make sure the correct arguments have been passed if len( argv ) < 3: print( usage ) sys.exit( 2 ) try: opts, args = getopt.getopt( argv, 't:s:i:r:l:w:h', ["training-data=","test-data=","iterations=","learning-rate=","regularizer=","save-weights="] ) except getopt.GetoptError: print( usage ) sys.exit( 2 ) for opt, arg in opts: if opt == '-h': print( usage ) sys.exit() elif opt in ( '-t', "training-data=" ): trainingDataPath = arg elif opt in ( '-s', "test-data=" ): testDataPath = arg elif opt in ( '-i', "iterations=" ): iterations = int(arg) elif opt in ( '-l', "rate=" ): rate = float(arg) elif opt in ( '-r', "regularizer=" ): regularization = float(arg) elif opt in ( '-w', "save-weights=" ): weightsPath = str( arg ) #Make sure we got a test and training data path if trainingDataPath == '' or testDataPath == '': print( usage ) sys.exit( 2 ) if weightsPath != "": weightsFile = open( weightsPath, 'a' ) #Get our test and training data trainingDataMatrix = list(); trainingResponseVec = list(); testDataMatrix = list(); testResponseVec = list(); print( "Parsing training data..." ) getDataMatrix( trainingDataPath, trainingDataMatrix, trainingResponseVec ); print( "Parsing test data..." ) getDataMatrix( testDataPath, testDataMatrix, testResponseVec ); #Make sure all our points are normalized for i in trainingDataMatrix: for j in range( len( i ) ): if i[j] < 0 or i[j] > 1: print( "Training data point out of range [0,1] at index:",j,"with value:",i[j] ) sys.exit(2) for i in testDataMatrix: for j in range( len( i ) ): if i[j] < 0 or i[j] > 1: print( "Test data point out of range [0,1] at index:",j,"with value:",i[j] ) sys.exit(2) print( "Running regression..." ) weights = trainModel( trainingDataMatrix, trainingResponseVec, rate, regularization, iterations, trainingCallback ) #Close weights file if it is open if weightsFile != None: weightsFile.close() print( weights ) print( "Testing model..." ) testModel( testDataMatrix, testResponseVec, weights, testCallback )
PREPROCESSED_FOLDER_PATH = DATASETS_PATH + "UTKFace-Preprocessed-" + str( MARGIN) + "/" PREPROCESSED_IMAGES_PATH = PREPROCESSED_FOLDER_PATH + "Images/" PREPROCESSED_CSV_PATH = PREPROCESSED_FOLDER_PATH + "CSVs/" # preprocessDataset(MARGIN, DATASETS_PATH, PREPROCESSED_FOLDER_PATH, PREPROCESSED_IMAGES_PATH, PREPROCESSED_CSV_PATH) # distributeDatset(PREPROCESSED_CSV_PATH, MIN_AGE, MAX_AGE, SOCIAL_MEDIA_SEGMENTS) best_age_mae, best_ML = -1, (0, 0) for M in M_array: for L in L_array: # Output Path OUTPUT_FOLDER_NAME = "LabelDiversity-" + RESNET_SIZE + "-" + str( M) + "X" + str(L) OUT_PATH = "../TrainedModels/" + OUTPUT_FOLDER_NAME + "/" validLoss = trainModel(PREPROCESSED_IMAGES_PATH, PREPROCESSED_CSV_PATH, OUT_PATH, RESNET_SIZE, MIN_AGE, MAX_AGE, M, L) age_mae = testModel(PREPROCESSED_IMAGES_PATH, PREPROCESSED_CSV_PATH, OUT_PATH, SOCIAL_MEDIA_SEGMENTS, RESNET_SIZE, MIN_AGE, MAX_AGE, M, L) if age_mae < best_age_mae or best_age_mae == -1: best_age_mae = age_mae best_ML = (M, L) print("Current ML:\t", (M, L), "\tCurrent MAE:\t", age_mae) print("Best ML:\t", best_ML, "\tBest MAE:\t", best_age_mae)
def run_wav2(wav0): conn = psycopg2.connect( user="******", password="******", host="test-heroku.cs0ubczlcaiz.us-west-1.rds.amazonaws.com", port="5432", database="test-aws") cur = conn.cursor() print('\nProcessing Run Wave 2...\n') # wav = os.path.join('data/new_data/fft/' + string2 + '.png') # wav0 = os.listdir('data/new_data/')[0] wav = wav0.strip('data/new_data/') wav1 = wav.strip('.wav') + '.png' # wav = os.listdir('data/new_data/')[0] # wav = next(join('data/new_data/', f) for f in os.listdir('data/new_data/')) # print('Wav0: \n', wav0) # print('Wav: \n', wav) # print('Wav1: \n', wav1) word = wav.split('-') # print('Word: \n', word) mea_date = word[0] # print('Measure Date: \n', mea_date) disp_date = mea_date[0:4] + '-' + mea_date[4:6] + '-' + mea_date[ 6:8] + ' ' + mea_date[8:10] + ':' + mea_date[10:12] + ':' + mea_date[ 12:14] # print('Display Date: \n', disp_date) dev_id = word[1] # print('Dev ID: \n', dev_id) qr_code0 = word[3] qr_code = qr_code0.strip('.wav') # print('RQ Code: \n', qr_code) loc = word[2] # print('Location: \n', loc) print('\nProcessing Prob...') conf = testModel() p # rint('Conf-2: ',conf) # run_prob (dev_id) print('\nFinished Prob.') # path = '/Users/andy/Projects/flow-ez/data' # print("Moving file:") # print(os.listdir(path)) # source = '/Users/andy/Projects/flow-ez/data/new_data' # destination = '/Users/andy/Projects/flow-ez/data/old_data' # dest = shutil.move(source, destination, copy_function=copy2) # print("Finished Moving file:") # print(os.listdir(path)) # print("Destination path:", dest) print('\nProcessing Pulse...') plt.figure(dpi=2400) my_pulse(wav0) print('\nFinished Pulse.') print('\nProcessing FFT...') my_fft(wav0) print('\nFinished FFT.') if (conf > 50): res = 'Abnormal' else: res = 'Normal' first_name = names.get_first_name() last_name = names.get_last_name() shutil.copy('static/trend/20191101101100-003004802801-UP-00000001.png', 'static/trend/' + wav1) cur.execute("BEGIN TRANSACTION;") # cur.execute("Update data_table (mea_date,dev_id,qr_code,loc,pulse,fft,trend) VALUES (?,?,?,?,?,?,?)",(mea_date,dev_id,qr_code,loc,wav1,wav1,wav1) where (last_name='West')) # cur.execute("UPDATE data_table set mea_date=?,disp_date=?,dev_id=?,qr_code=?,loc=?,pulse=?,fft=?,trend=?,res=?,prob=? WHERE dev_id=?", (mea_date,disp_date,dev_id,qr_code,loc,wav1,wav1,wav1,res,conf,dev_id,) ) cur.execute( "INSERT INTO data_table (first_name,last_name,mea_date,disp_date,dev_id,qr_code,loc,pulse,fft,trend,res,prob) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", (first_name, last_name, mea_date, disp_date, dev_id, qr_code, loc, wav1, wav1, wav1, res, conf)) cur.execute("COMMIT TRANSACTION;") cur.close conn.close() #source = os.listdir("/Users/andy/Projects/flow-ez/data/new_data/") #destination = "/Users/andy/Projects/flow-ez/data/old_data/" source = os.listdir("/AboveCare/Projects/flow-ez/data/new_data/") destination = "/AboveCare/Projects/flow-ez/data/old_data/" for files in source: if files.endswith('.wav'): shutil.move("/AboveCare/Projects/flow-ez/data/new_data/" + files, destination) print('\nWave Files Moved.') # my_psp (wav0) ## Leadtek's data has "RuntimeWarning: divide by zero" issue
history = parallel_model.fit( X_train, y_train, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=1, initial_epoch=initial_epoch, validation_data=(X_test, y_test), callbacks=[tensorboard, checkpoint, early_stop, WeightsSaver(parallel_model, weight_backup_epoch), csv_logger, epoch_end_test]) else: history = parallel_model.fit_generator( train_gen, steps_per_epoch=steps_per_epoch_train, validation_data=valid_gen, validation_steps=steps_per_epoch_validation, use_multiprocessing=True, workers=4, shuffle=True, epochs=epochs, verbose=1, initial_epoch=initial_epoch, callbacks=[tensorboard, checkpoint, early_stop, WeightsSaver(parallel_model, weight_backup_epoch), csv_logger]) # save model parallel_model.save(MODEL_FILE_basename+'.h5') print('Saved the trained model to : {}'.format(MODEL_FILE_basename+'.h5')) # plot metrics plot_metrics() # test model testModel(MODEL_FILE_basename+'.h5', dataset=dataset)