def __init__(self, config_path, dev_type=DataLoader.DEVICE_CAM): """ dev_type is device type (Dataloader.py) """ self.config_path = config_path self.data = None self.config = None self.pictures = list() self.email = None self.contour_area_restrict = 15000 def make_config_obj(): try: with open(self.config_path) as config_file: self.config = json.load(config_file, object_hook=lambda d: collections.namedtuple('Config', d.keys())(*d.values())) except Exception as e: sys.stderr.write("Probably you do not have a proper json file or file is missing") raise(e) make_config_obj() self.data = DataLoader(dev_type, self.config.cam) self.deque = collections.deque(maxlen=self.config.opencv.frames.avarage) self.email = Email(self.config.email)
def test_dataset(data): batchSize = 2 y_test, x_test = data['B'], data['A'] x_path = data['A_paths'] # print(y_train.shape) dataSet = MyDataset3(x_test, y_test,x_path) loader = DataLoader() loader.initialize(dataSet, batchSize, shuffle=False) dataset = loader.load_data() return dataset
def train_logistic_regression(n, d, n_epoch, batch_size, b_init, l_rate): # Generate the data for a coefficient vector & init progress tracker! data_loader = DataLoader(n, d, batch_size, binary=True) b_hist, func_val_hist, param_error, acc_hist = [], [], [], [] # Get the coefficients as solution to optimized sklearn function logreg = LogisticRegression(penalty='none', solver='lbfgs', multi_class='multinomial') logreg.fit(data_loader.X, data_loader.y) norm_coeff = np.linalg.norm(logreg.coef_.ravel()) b_dual = DualTensor(b_init, None) for epoch in range(n_epoch): # Shuffle the batch identities at beginning of each epoch data_loader.batch_shuffle() for batch_id in range(data_loader.num_batches): # Clear the gradient b_dual.zero_grad() # Select the current batch & perform "mini-forward" pass X, y = data_loader.get_batch_idx(batch_id) y_pred_1, y_pred_2 = forward(X, b_dual) # Calculate the forward AD - real = func, dual = deriv current_dual, acc = binary_cross_entropy_dual( y, y_pred_1, y_pred_2) # Perform grad step & append results to the placeholder list b_dual.real -= l_rate * np.array(current_dual.dual).flatten() b_hist.append(b_dual.real) func_val_hist.append(current_dual.real) param_error.append( np.linalg.norm(logreg.coef_.ravel() - b_hist[-1]) / norm_coeff) acc_hist.append(acc) if np.abs(param_error[-1] - param_error[-2]) < 0.00001: break if epoch % 1 == 0: print( "Accuracy: {} | Euclidean Param Norm: {} | fct min: {}".format( acc, param_error[-1], current_dual.real)) return b_hist, func_val_hist, param_error, acc_hist
def test(model, index=''): g = torch.load(model) batchSize = 2 # data = load_images('./images/test', n_images=20) data = load_images('./images/test', -1) y_train, x_train = data['B'], data['A'] print(data['A_paths']) # print(y_train.shape) dataSet = MyDataset(x_train, y_train) loader = DataLoader() loader.initialize(dataSet, batchSize, shuffle=False) dataset = loader.load_data() for step, (x, y) in enumerate(dataset): img = g(x) # print(img.shape) # print(y.shape) res = np.concatenate((img.data.cpu().numpy(), x.data.cpu().numpy(), y.data.cpu().numpy()), axis=3) for i in range(x.shape[0]): # print(res[i].shape) save_image(res[i], './res/img/res{}_{}.png'.format(index, str(step * x.shape[0] + i))) print('./res/img/res{}_{}.png'.format(index, str(step * batchSize + i)))
t_test = ja_vocab.transform(ja_test_path, eos=True) def sort(x, t): lens = [len(i) for i in x] indices = sorted(range(len(lens)), key=lambda i: -lens[i]) x = [x[i] for i in indices] t = [t[i] for i in indices] return (x, t) (x_train, t_train) = sort(x_train, t_train) (x_val, t_val) = sort(x_val, t_val) (x_test, t_test) = sort(x_test, t_test) train_dataloader = DataLoader((x_train, t_train), batch_first=False, device=device) val_dataloader = DataLoader((x_val, t_val), batch_first=False, device=device) test_dataloader = DataLoader((x_test, t_test), batch_size=1, batch_first=False, device=device) ''' 2. モデルの構築 ''' depth_x = len(en_vocab.i2w) depth_t = len(ja_vocab.i2w) input_dim = depth_x
from Dataloader import DataLoader from sklearn import preprocessing from sklearn.svm import SVC import pandas as pd import numpy as np def countAnamoly(y_pred): count = 0 for i in y_pred: if i == True: count += 1 # Number of predicted anamoly print(count) DataLoader = DataLoader() data = DataLoader.get_dataset() # Using SVM to create models with several trails # Normalize the data X = data.drop('result', axis=1) y = data['result'] min_max_scaler = preprocessing.MinMaxScaler() x_scaled = min_max_scaler.fit_transform(X) X = pd.DataFrame(x_scaled) inputs = [0.01, 0.1, 1, 10, 100] for i in inputs: svclassifier = SVC(kernel = 'rbf', C = i, class_weight="balanced") svclassifier.fit(X, y) y_pred = svclassifier.predict(X)
def train_multiple_outputs(continue_train=False, Gmodel=None, Dmodel=None, index='00', id=0): data = load_images('./images/train', n_images=1000) y_train, x_train = data['B'], data['A'] print(y_train.shape[0]) dataSet = MyDataset(x_train, y_train) loader = DataLoader() loader.initialize(dataSet, BATCH_SIZE) dataset = loader.load_data() testData = load_images('./images/test', -1) testDataset = test_dataset(testData) print('continue_train: ', continue_train) if continue_train: if Gmodel == None: g = Gnet().double() print('creat new Gnet') else: g = torch.load(Gmodel) print('using ', Gmodel) if DataLoader == None: d = NLayerDiscriminator().double() print('creat new Dnet') else: d = torch.load(Dmodel) print('using ', Dmodel) # dOnG = torch.load('DOnGnet') else: g = Gnet().double() print('creat new Gnet') d = NLayerDiscriminator().double() print('creat new Dnet') # dOnG = generator_containing_discriminator_multiple_outputs(Gnet=g, Dnet=d) g.cuda() d.cuda() # dOnG.cuda() optimizerG = torch.optim.Adam(g.parameters(), lr=LR) optimizerD = torch.optim.Adam(d.parameters(), lr=LR) loss_function = nn.MSELoss().cuda(device='0') l1Loss = nn.L1Loss().cuda(device='0') perceptauLoss = PerceptualLoss_v2().cuda(device='0') perceptauLoss.initialize(l1Loss) wassesrsteinLoss = WassesrsteinLoss().cuda(device='0') now = datetime.datetime.now() start = time.time() for i in range(8): GLoss = [] DLoss = [] for step, (x, y) in enumerate(dataset): x_f = g(x) for _ in range(2): dReal = d(y) dFalse = d(x_f) dLossReal = wassesrsteinLoss(dReal, outputTrueBatch) dLossFalse = wassesrsteinLoss(dFalse, outputFalseBatch) dLoss = 0.5 * (dLossReal + dLossFalse) optimizerD.zero_grad() dLoss.backward(retain_graph=True) optimizerD.step() DLoss.append(dLoss.data.cpu().numpy()) # # dFalse = d(x_f) dLoss = loss_function(dFalse, outputTrueBatch) # pLoss = perceptauLoss(x_f, y) gLoss = dLoss optimizerG.zero_grad() gLoss.backward() optimizerG.step() GLoss.append(gLoss.data.cpu().numpy()) # print('gloss: ', gLoss.data.cpu().numpy()) if step % 10 == 0: # now = datetime.datetime.now() print('-' * 10, ' ', id, ': Saving model\trun time: ', int((time.time() - start) / 60), 'm', int((time.time() - start) % 60), 's ', '-' * 10) start = time.time() save_dir = os.path.join(BASE_DIR, '{}{}_{}'.format(now.month, now.day, index)) if not os.path.exists(save_dir): os.makedirs(save_dir) print('dloss: ', np.mean(DLoss), '\tploss: ', np.mean(GLoss)) torch.save(g, os.path.join(save_dir, '{}_Gnet_{}'.format(index, id))) torch.save(d, os.path.join(save_dir, '{}_Dnet_{}'.format(index, id))) with open(os.path.join(save_dir, '{}_log.txt'.format(index)), 'a+') as f: f.write('{}\t{}\t{}\r\n'.format(id, np.mean(DLoss), np.mean(GLoss))) # id = id + 1 test_img(g, id, save_dir, testDataset) id = id + 1
def train_multiple_outputs(continue_train=False, Gmodel=None, Dmodel=None, index='00', id=0): data = load_images('./images/train', n_images=1000) y_train, x_train = data['B'], data['A'] print(y_train.shape[0]) dataSet = MyDataset(x_train, y_train) loader = DataLoader() loader.initialize(dataSet, BATCH_SIZE) dataset = loader.load_data() print('continue_train: ', continue_train) testData = load_images('./images/test', -1) testDataset = test_dataset(testData) LOSS = 1000 print(LR) if continue_train: if Gmodel == None: g = Gnet().double() print('creat new Gnet') else: g = torch.load(Gmodel) print('using ', Gmodel) if DataLoader == None: d = NLayerDiscriminator().double() print('creat new Dnet') else: d = torch.load(Dmodel) print('using ', Dmodel) # dOnG = torch.load('DOnGnet') else: g = Gnet().double() print('creat new Gnet') d = NLayerDiscriminator().double() print('creat new Dnet') # dOnG = generator_containing_discriminator_multiple_outputs(Gnet=g, Dnet=d) g.cuda() d.cuda() # dOnG.cuda() optimizerG = torch.optim.Adam(g.parameters(), lr=LR, betas=(0.9, 0.999), eps=1e-8) optimizerD = torch.optim.Adam(d.parameters(), lr=LR, betas=(0.9, 0.999), eps=1e-8) # optimizerDOnG = torch.optim.Adam(dOnG.parameters(), lr=LR) loss_function = nn.MSELoss().cuda(device='0') l1Loss = nn.L1Loss().cuda(device='0') perceptauLoss = PerceptualLoss_v2().cuda(device='0') perceptauLoss.initialize(loss_function) wassesrsteinLoss = WassesrsteinLoss().cuda(device='0') now = datetime.datetime.now() start = time.time() for i in range(8): # i = 1 GLoss = [] DLoss = [] # if i % 2 == 0: # lr = LR * (0.1 ** int(i/2)) # print('LR: ', lr) # for para_group in optimizerG.param_groups: # para_group['lr'] = lr # for para_group in optimizerD.param_groups: # para_group['lr'] = lr for step, (x, y) in enumerate(dataset): x_f = g(x) for _ in range(5): dReal = d(y) dFalse = d(x_f) # print(dReal.shape) # print(type(outputTrueBatch)) dLossReal = wassesrsteinLoss(dReal, outputTrueBatch) # dLossReal = wassesrsteinLoss(dReal, 1) # print(dLossReal) dLossFalse = wassesrsteinLoss(dFalse, outputFalseBatch) # dLossFalse = wassesrsteinLoss(dFalse, -1) dLoss = 0.5 * (dLossReal + dLossFalse) optimizerD.zero_grad() dLoss.backward(retain_graph=True) optimizerD.step() DLoss.append(dLoss.data.cpu().numpy()) # # dFalse = d(x_f) dLoss = wassesrsteinLoss(dFalse, outputTrueBatch) pLoss = perceptauLoss(x_f, y) gLoss = 100 * pLoss + dLoss # gLoss = 100 * perceptauLoss(x_f, y) + wassesrsteinLoss(x_f, y) optimizerG.zero_grad() gLoss.backward() optimizerG.step() LOSS_NOW = 100 * pLoss.data.cpu().numpy() # print('step', step, ': ', lossG.data.cpu().numpy()) GLoss.append(LOSS_NOW) # print('Dloss: ', np.mean(DLoss), 'dloss: ', dLoss.data.cpu().numpy(), '\tgloss: ', gLoss.data.cpu().numpy(), '\tperceptauLoss: ', pLoss.data.cpu().numpy()) # print('dloss: ', dLoss.data.cpu().numpy(), '\tgloss: ', LOSS_NOW, # '\tperceptauLoss: ', pLoss.data.cpu().numpy()) if gLoss.data.cpu().numpy() < LOSS: # now = datetime.datetime.now() save_dir = os.path.join(BASE_DIR, '{}{}_{}'.format(now.month, now.day, index)) if not os.path.exists(save_dir): os.makedirs(save_dir) torch.save(g, os.path.join(save_dir, '{}_Gnet_{}_{}'.format(index, 'bestGloss', int(LOSS_NOW)))) LOSS = LOSS_NOW print('best model saved!!\t', save_dir, '{}_Gnet_{}_{}'.format(index, 'bestGloss', LOSS_NOW)) save_dir = os.path.join(BASE_DIR, '{}{}_{}'.format(now.month, now.day, index)) if not os.path.exists(save_dir): os.makedirs(save_dir) test_img(g, '{}_{}'.format('bestGloss', LOSS_NOW), save_dir, testDataset) if step % 10 == 0: # now = datetime.datetime.now() print('-' * 10, ' ', id, ': Saving model\trun time: ', int((time.time() - start) / 60), 'm', int((time.time() - start) % 60), 's ', '-' * 10) start = time.time() currentLoss = int(np.mean(GLoss)) save_dir = os.path.join(BASE_DIR, '{}{}_{}'.format(now.month, now.day, index)) if not os.path.exists(save_dir): os.makedirs(save_dir) print('dloss: ', np.mean(DLoss), '\tploss: ', np.mean(GLoss)) torch.save(g, os.path.join(save_dir, '{}_Gnet_{}_{}'.format(index, id, currentLoss))) torch.save(d, os.path.join(save_dir, '{}_Dnet_{}'.format(index, id))) with open(os.path.join(save_dir, '{}_log.txt'.format(index)), 'a+') as f: f.write('{}\t{}\t{}\r\n'.format(id, np.mean(DLoss), np.mean(GLoss))) # id = id + 1 test_img(g, id, save_dir, testDataset) id = id + 1
def val_dataloader(self): test_loader = DataLoader(self.test_dataset, batch_size=self.arg.batch_size, shuffle=False) return test_loader
def train_dataloader(self): train_loader = DataLoader(self.train_dataset, batch_size=self.arg.batch_size, shuffle=True) return train_loader
class Surveillance(object): def __init__(self, config_path, dev_type=DataLoader.DEVICE_CAM): """ dev_type is device type (Dataloader.py) """ self.config_path = config_path self.data = None self.config = None self.pictures = list() self.email = None self.contour_area_restrict = 15000 def make_config_obj(): try: with open(self.config_path) as config_file: self.config = json.load(config_file, object_hook=lambda d: collections.namedtuple('Config', d.keys())(*d.values())) except Exception as e: sys.stderr.write("Probably you do not have a proper json file or file is missing") raise(e) make_config_obj() self.data = DataLoader(dev_type, self.config.cam) self.deque = collections.deque(maxlen=self.config.opencv.frames.avarage) self.email = Email(self.config.email) def run(self): avarage_frame = 0 avarage_frame_init = False snapshot_time = datetime.datetime.now() reference_time = datetime.datetime.now() for dframe in self.data.get_data_frame(): #cv2.imshow('Original', dframe) gray_scaled_frame = cv2.cvtColor(dframe, cv2.COLOR_BGR2GRAY) gaussian_blur_frame = (cv2.GaussianBlur(gray_scaled_frame, (self.config.opencv.filter.GaussianBlur.mask.row, self.config.opencv.filter.GaussianBlur.mask.column), 0)) #cv2.imshow('Gaussian blur', gaussian_blur_frame) _, binary_frame = cv2.threshold(gaussian_blur_frame, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) #cv2.imshow('Binary frame', binary_frame) #Initialization if self.deque.maxlen > len(self.deque): print("Waiting for another frame") self.deque.append(binary_frame.astype('uint16')) continue elif self.deque.maxlen == len(self.deque) and not avarage_frame_init: avarage_frame = sum(self.deque)/self.deque.maxlen avarage_frame_init = True continue avarage_frame = sum((self.deque))/self.deque.maxlen without_backgroung_frame = avarage_frame - binary_frame #cv2.imshow('Pict without background', without_backgroung_frame) erode_frame = cv2.erode(without_backgroung_frame.astype(np.uint8), np.ones((11, 11), np.uint8), iterations=1) dilate_frame = cv2.dilate(erode_frame, np.ones((15, 15), np.uint8), iterations=5) #cv2.imshow('Erode', erode_frame) #cv2.imshow('Dilate', dilate_frame) img, contours,hierarchy = cv2.findContours(dilate_frame, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) #cv2.imshow('Output with contours',cv2.drawContours(dframe.copy() , contours, -1, (0, 255, 0), 3)) snapshot_time = datetime.datetime.now() for contour in contours: if cv2.contourArea(contour) < self.contour_area_restrict: continue cnt = np.array(contour).reshape((-1,1,2)).astype(np.int32) cv2.drawContours(dframe, cnt, -1, (0, 255,255), 2) cv2.imshow("omg2", dframe) img_file = BytesIO(cv2.imencode('.png', dframe)[1]) self.pictures.append(img_file) if (snapshot_time - reference_time).seconds > int(self.config.email.user.timeout) and len(self.pictures): #send email self.email.send(self.pictures) reference_time = snapshot_time self.pictures = [] self.deque.append(binary_frame.astype('uint16')) avarage_frame = sum(self.deque)/self.deque.maxlen
def main(arg): # Set random seeds torch.manual_seed(42) torch.cuda.manual_seed(42) np.random.seed(42) torch.backends.cudnn.deterministic = True torch.manual_seed(42) rng = np.random.RandomState(42) # Get args bn = arg.batch_size mode = arg.mode name = arg.name load_from_ckpt = arg.load_from_ckpt lr = arg.lr L_inv_scal = arg.L_inv_scal epochs = arg.epochs device = torch.device( 'cuda:' + str(arg.gpu[0]) if torch.cuda.is_available() else 'cpu') arg.device = device # Load Datasets and DataLoader dataset = get_dataset(arg.dataset) if arg.dataset == 'pennaction': init_dataset = dataset(size=arg.reconstr_dim, action_req=[ "tennis_serve", "tennis_forehand", "baseball_pitch", "baseball_swing", "jumping_jacks", "golf_swing" ]) splits = [ int(len(init_dataset) * 0.8), len(init_dataset) - int(len(init_dataset) * 0.8) ] train_dataset, test_dataset = torch.utils.data.random_split( init_dataset, splits) else: train_dataset = dataset(size=arg.reconstr_dim, train=True) test_dataset = dataset(size=arg.reconstr_dim, train=False) train_loader = DataLoader(train_dataset, batch_size=bn, shuffle=True, num_workers=4) test_loader = DataLoader(test_dataset, batch_size=bn, shuffle=False, num_workers=4) if mode == 'train': # Make new directory model_save_dir = '../results/' + name if not os.path.exists(model_save_dir): os.makedirs(model_save_dir) os.makedirs(model_save_dir + '/summary') # Save Hyperparameters write_hyperparameters(arg.toDict(), model_save_dir) # Define Model model = Model(arg).to(device) if load_from_ckpt: model = load_model(model, model_save_dir, device).to(device) print(f'Number of Parameters: {count_parameters(model)}') # Definde Optimizer optimizer = torch.optim.Adam(model.parameters(), lr=lr) # Log with wandb wandb.init(project='Disentanglement', config=arg, name=arg.name) wandb.watch(model, log='all') # Make Training with torch.autograd.set_detect_anomaly(False): for epoch in range(epochs + 1): # Train on Train Set model.train() model.mode = 'train' for step, (original, keypoints) in enumerate(train_loader): if epoch != 0: model.L_sep = 0. original, keypoints = original.to(device), keypoints.to( device) image_rec, reconstruct_same_id, loss, rec_loss, transform_loss, precision_loss, mu, L_inv, mu_original = model( original) mu_norm = torch.mean(torch.norm( mu, p=1, dim=2)).cpu().detach().numpy() L_inv_norm = torch.mean( torch.linalg.norm(L_inv, ord='fro', dim=[2, 3])).cpu().detach().numpy() # Track Mean and Precision Matrix wandb.log({"Part Means": mu_norm}) wandb.log({"Precision Matrix": L_inv_norm}) # Zero out gradients optimizer.zero_grad() loss.backward() optimizer.step() # Track Loss wandb.log({"Training Loss": loss}) # Track Metric score = keypoint_metric(mu_original, keypoints) wandb.log({"Metric Train": score}) # Evaluate on Test Set model.eval() val_score = torch.zeros(1) val_loss = torch.zeros(1) for step, (original, keypoints) in enumerate(test_loader): with torch.no_grad(): original, keypoints = original.to( device), keypoints.to(device) image_rec, reconstruct_same_id, loss, rec_loss, transform_loss, precision_loss, mu, L_inv, mu_original = model( original) # Track Loss and Metric score = keypoint_metric(mu_original, keypoints) val_score += score.cpu() val_loss += loss.cpu() val_loss = val_loss / (step + 1) val_score = val_score / (step + 1) wandb.log({"Evaluation Loss": val_loss}) wandb.log({"Metric Validation": val_score}) # Track Progress & Visualization for step, (original, keypoints) in enumerate(test_loader): with torch.no_grad(): model.mode = 'predict' original, keypoints = original.to( device), keypoints.to(device) original_part_maps, mu_original, image_rec, part_maps, part_maps, reconstruction = model( original) # img = visualize_predictions(original, original_part_maps, keypoints, reconstruction, image_rec[:original.shape[0]], # image_rec[original.shape[0]:], part_maps[original.shape[0]:], part_maps[:original.shape[0]], # # L_inv_scal, model_save_dir + '/summary/', epoch, device, show_labels=False) # if epoch % 5 == 0: # wandb.log({"Summary_" + str(epoch): [wandb.Image(img)]}) save_model(model, model_save_dir) if step == 0: break # Decrements # model.L_sep = arg.sig_decr * model.L_sep elif mode == 'predict': # Make Directory for Predictions model_save_dir = '../results/' + arg.dataset + '/' + name # Dont use Transformations arg.tps_scal = 0. arg.rot_scal = 0. arg.off_scal = 0. arg.scal_var = 0. arg.augm_scal = 1. arg.contrast = 0. arg.brightness = 0. arg.saturation = 0. arg.hue = 0. # Load Model and Dataset model = Model(arg).to(device) model = load_model(model, model_save_dir, device) model.eval() # Log with wandb # wandb.init(project='Disentanglement', config=arg, name=arg.name) # wandb.watch(model, log='all') # Predict on Dataset val_score = torch.zeros(1) for step, (original, keypoints) in enumerate(test_loader): with torch.no_grad(): original, keypoints = original.to(device), keypoints.to(device) ground_truth_images, img_reconstr, mu, L_inv, part_map_norm, heat_map, heat_map_norm, total_loss = model( original) score, mu_new, L_inv, part_map_norm_new, heat_map_new = keypoint_metric( mu, keypoints, L_inv, part_map_norm, heat_map, arg.reconstr_dim) if step == 0: img = visualize_predictions(original, img_reconstr, mu_new, part_map_norm_new, heat_map_new, mu, part_map_norm, heat_map, model_save_dir) # wandb.log({"Prediction": [wandb.Image(img)]}) val_score += score.cpu() val_score = val_score / (step + 1) print("Validation Score: ", val_score)
# 4. Print the results for accuracy and other performance parameters. # 5. Repeat steps 2 to 4 for othe data models. # **************************************************************************************************************** from sklearn.naive_bayes import MultinomialNB from sklearn.feature_extraction.text import TfidfVectorizer from Dataloader import DataLoader from Preparedata import DataPreparer from Classification import Classification from sklearn.feature_extraction.text import CountVectorizer # Name of dataset file to be placed in the working directory of the script inputFile = "tweet_poll_event_dataset.xlsx" # Step 1 data_loader = DataLoader(inputFile).loadData() # Step 2 prepare_data_obama = DataPreparer(data_loader.sheet_obama) prepare_data_obama.clean_data() prepare_data_romney = DataPreparer(data_loader.sheet_romney) prepare_data_romney.clean_data() # Processing Obama data #Vectorize the tweets to create sparse matrix for the words as features # Experiments shows that using unigram features increases the accuracy for TF-IDF Vectorizer print ":::: Obama Data ::::" vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5) prepare_data_obama.createTrain_TestSet() prepare_data_obama.vectorizeData(vectorizer)
def main(arg): # Set random seeds torch.manual_seed(42) torch.cuda.manual_seed(42) np.random.seed(42) torch.backends.cudnn.deterministic = True torch.manual_seed(42) rng = np.random.RandomState(42) # Get args bn = arg.batch_size mode = arg.mode name = arg.name load_from_ckpt = arg.load_from_ckpt lr = arg.lr epochs = arg.epochs device = torch.device( 'cuda:' + str(arg.gpu[0]) if torch.cuda.is_available() else 'cpu') arg.device = device # Load Datasets and DataLoader if arg.dataset != "mix": dataset = get_dataset(arg.dataset) if arg.dataset == 'pennaction': # init_dataset = dataset(size=arg.reconstr_dim, action_req=["tennis_serve", "tennis_forehand", "baseball_pitch", # "baseball_swing", "jumping_jacks", "golf_swing"]) init_dataset = dataset(size=arg.reconstr_dim) splits = [ int(len(init_dataset) * 0.8), len(init_dataset) - int(len(init_dataset) * 0.8) ] train_dataset, test_dataset = random_split( init_dataset, splits, generator=torch.Generator().manual_seed(42)) elif arg.dataset == 'deepfashion': train_dataset = dataset(size=arg.reconstr_dim, train=True) test_dataset = dataset(size=arg.reconstr_dim, train=False) elif arg.dataset == 'human36': init_dataset = dataset(size=arg.reconstr_dim) splits = [ int(len(init_dataset) * 0.8), len(init_dataset) - int(len(init_dataset) * 0.8) ] train_dataset, test_dataset = random_split( init_dataset, splits, generator=torch.Generator().manual_seed(42)) elif arg.dataset == 'mix': # add pennaction dataset_pa = get_dataset("pennaction") init_dataset_pa = dataset_pa(size=arg.reconstr_dim, action_req=[ "tennis_serve", "tennis_forehand", "baseball_pitch", "baseball_swing", "jumping_jacks", "golf_swing" ], mix=True) splits_pa = [ int(len(init_dataset_pa) * 0.8), len(init_dataset_pa) - int(len(init_dataset_pa) * 0.8) ] train_dataset_pa, test_dataset_pa = random_split( init_dataset_pa, splits_pa, generator=torch.Generator().manual_seed(42)) # add deepfashion dataset_df = get_dataset("deepfashion") train_dataset_df = dataset_df(size=arg.reconstr_dim, train=True, mix=True) test_dataset_df = dataset_df(size=arg.reconstr_dim, train=False, mix=True) # add human36 dataset_h36 = get_dataset("human36") init_dataset_h36 = dataset_h36(size=arg.reconstr_dim, mix=True) splits_h36 = [ int(len(init_dataset_h36) * 0.8), len(init_dataset_h36) - int(len(init_dataset_h36) * 0.8) ] train_dataset_h36, test_dataset_h36 = random_split( init_dataset_h36, splits_h36, generator=torch.Generator().manual_seed(42)) # Concatinate all train_datasets = [train_dataset_df, train_dataset_h36] test_datasets = [test_dataset_df, test_dataset_h36] train_dataset = ConcatDataset(train_datasets) test_dataset = ConcatDataset(test_datasets) train_loader = DataLoader(train_dataset, batch_size=bn, shuffle=True, num_workers=4) test_loader = DataLoader(test_dataset, batch_size=bn, shuffle=True, num_workers=4) if mode == 'train': # Make new directory model_save_dir = '../results/' + arg.dataset + '/' + name if not os.path.exists(model_save_dir): os.makedirs(model_save_dir) os.makedirs(model_save_dir + '/summary') # Save Hyperparameters write_hyperparameters(arg.toDict(), model_save_dir) # Define Model model = Model(arg) if len(arg.gpu) > 1: model = torch.nn.DataParallel(model, device_ids=arg.gpu) model.to(device) if load_from_ckpt: model = load_model(model, model_save_dir, device).to(device) # Dataparallel print(arg.gpu) print(f'Number of Parameters: {count_parameters(model)}') # Definde Optimizer optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=arg.weight_decay) scheduler = ReduceLROnPlateau(optimizer, factor=0.2, threshold=1e-4, patience=6) # Log with wandb wandb.init(project='Disentanglement', config=arg, name=arg.name) wandb.watch(model, log='all') # Make Training with torch.autograd.set_detect_anomaly(False): for epoch in range(epochs + 1): # Train on Train Set model.train() # model.mode = 'train' for step, (original, keypoints) in enumerate(train_loader): bn = original.shape[0] original, keypoints = original.to(device), keypoints.to( device) # Forward Pass ground_truth_images, img_reconstr, mu, L_inv, part_map_norm, heat_map, heat_map_norm, total_loss = model( original) # Track Mean and Precision Matrix mu_norm = torch.mean(torch.norm( mu[:bn], p=1, dim=2)).cpu().detach().numpy() L_inv_norm = torch.mean( torch.linalg.norm(L_inv[:bn], ord='fro', dim=[2, 3])).cpu().detach().numpy() wandb.log({"Part Means": mu_norm}) wandb.log({"Precision Matrix": L_inv_norm}) # Zero out gradients optimizer.zero_grad() total_loss.backward() optimizer.step() # Track Loss wandb.log({"Training Loss": total_loss.cpu()}) # Track Metric score, mu, L_inv, part_map_norm, heat_map = keypoint_metric( mu, keypoints, L_inv, part_map_norm, heat_map, arg.reconstr_dim) wandb.log({"Metric Train": score}) # Track progress if step % 10000 == 0 and bn >= 4: for step_, (original, keypoints) in enumerate(test_loader): with torch.no_grad(): original, keypoints = original.to( device), keypoints.to(device) ground_truth_images, img_reconstr, mu, L_inv, part_map_norm,\ heat_map, heat_map_norm, total_loss = model(original) # Visualize Results score, mu, L_inv, part_map_norm, heat_map = keypoint_metric( mu, keypoints, L_inv, part_map_norm, heat_map, arg.reconstr_dim) img = visualize_results( ground_truth_images, img_reconstr, mu, L_inv, part_map_norm, heat_map, keypoints, model_save_dir + '/summary/', epoch, arg.background) wandb.log({ "Summary at step" + str(step): [wandb.Image(img)] }) save_model(model, model_save_dir) if step_ == 0: break # Evaluate on Test Set model.eval() val_score = torch.zeros(1) val_loss = torch.zeros(1) for step, (original, keypoints) in enumerate(test_loader): with torch.no_grad(): original, keypoints = original.to( device), keypoints.to(device) ground_truth_images, img_reconstr, mu, L_inv, part_map_norm, heat_map, heat_map_norm, total_loss = model( original) # Track Loss and Metric score, mu, L_inv, part_map_norm, heat_map = keypoint_metric( mu, keypoints, L_inv, part_map_norm, heat_map, arg.reconstr_dim) val_score += score.cpu() val_loss += total_loss.cpu() val_loss = val_loss / (step + 1) val_score = val_score / (step + 1) if epoch == 0: best_score = val_score if val_score <= best_score: best_score = val_score save_model(model, model_save_dir) scheduler.step(val_score) wandb.log({"Evaluation Loss": val_loss}) wandb.log({"Metric Validation": val_score}) # Track Progress & Visualization for step, (original, keypoints) in enumerate(test_loader): with torch.no_grad(): original, keypoints = original.to( device), keypoints.to(device) ground_truth_images, img_reconstr, mu, L_inv, part_map_norm, heat_map, heat_map_norm, total_loss = model( original) score, mu, L_inv, part_map_norm, heat_map = keypoint_metric( mu, keypoints, L_inv, part_map_norm, heat_map, arg.reconstr_dim) img = visualize_results(ground_truth_images, img_reconstr, mu, L_inv, part_map_norm, heat_map, keypoints, model_save_dir + '/summary/', epoch, arg.background) wandb.log( {"Summary_" + str(epoch): [wandb.Image(img)]}) if step == 0: break elif mode == 'predict': # Make Directory for Predictions model_save_dir = '../results/' + arg.dataset + '/' + name # Dont use Transformations arg.tps_scal = 0. arg.rot_scal = 0. arg.off_scal = 0. arg.scal_var = 0. arg.augm_scal = 1. arg.contrast = 0. arg.brightness = 0. arg.saturation = 0. arg.hue = 0. # Load Model and Dataset model = Model(arg).to(device) model = load_model(model, model_save_dir, device) model.eval() # Log with wandb # wandb.init(project='Disentanglement', config=arg, name=arg.name) # wandb.watch(model, log='all') # Predict on Dataset val_score = torch.zeros(1) for step, (original, keypoints) in enumerate(test_loader): with torch.no_grad(): original, keypoints = original.to(device), keypoints.to(device) ground_truth_images, img_reconstr, mu, L_inv, part_map_norm, heat_map, heat_map_norm, total_loss = model( original) score, mu_new, L_inv, part_map_norm_new, heat_map_new = keypoint_metric( mu, keypoints, L_inv, part_map_norm, heat_map, arg.reconstr_dim) if step == 0: img = visualize_predictions(original, img_reconstr, mu_new, part_map_norm_new, heat_map_new, mu, part_map_norm, heat_map, model_save_dir) # wandb.log({"Prediction": [wandb.Image(img)]}) val_score += score.cpu() val_score = val_score / (step + 1) print("Validation Score: ", val_score)
def main(arg): # Set random seeds torch.manual_seed(7) torch.cuda.manual_seed(7) np.random.seed(7) # Get args bn = arg.bn mode = arg.mode name = arg.name load_from_ckpt = arg.load_from_ckpt lr = arg.lr epochs = arg.epochs device = torch.device('cuda:' + str(arg.gpu) if torch.cuda.is_available() else 'cpu') arg.device = device if mode == 'train': # Make new directory model_save_dir = '../results/' + name if not os.path.exists(model_save_dir): os.makedirs(model_save_dir) os.makedirs(model_save_dir + '/summary') # Save Hyperparameters write_hyperparameters(arg.toDict(), model_save_dir) # Define Model & Optimizer model = Model(arg).to(device) if load_from_ckpt: model = load_model(model, model_save_dir).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=lr) # Log with wandb wandb.init(project='Disentanglement', config=arg, name=arg.name) wandb.watch(model, log='all') # Load Datasets and DataLoader train_data, test_data = load_deep_fashion_dataset() train_dataset = ImageDataset(np.array(train_data)) test_dataset = ImageDataset(np.array(test_data)) train_loader = DataLoader(train_dataset, batch_size=bn, shuffle=True, num_workers=4) test_loader = DataLoader(test_dataset, batch_size=bn, num_workers=4) # Make Training with torch.autograd.set_detect_anomaly(False): for epoch in range(epochs + 1): # Train on Train Set model.train() model.mode = 'train' for step, original in enumerate(train_loader): original = original.to(device) # Make transformations tps_param_dic = tps_parameters(original.shape[0], arg.scal, arg.tps_scal, arg.rot_scal, arg.off_scal, arg.scal_var, arg.augm_scal) coord, vector = make_input_tps_param(tps_param_dic) coord, vector = coord.to(device), vector.to(device) image_spatial_t, _ = ThinPlateSpline( original, coord, vector, original.shape[3], device) image_appearance_t = K.ColorJitter(arg.brightness, arg.contrast, arg.saturation, arg.hue)(original) image_spatial_t, image_appearance_t = normalize( image_spatial_t), normalize(image_appearance_t) reconstruction, loss, rec_loss, equiv_loss, mu, L_inv = model( original, image_spatial_t, image_appearance_t, coord, vector) mu_norm = torch.mean(torch.norm( mu, p=1, dim=2)).cpu().detach().numpy() L_inv_norm = torch.mean( torch.linalg.norm(L_inv, ord='fro', dim=[2, 3])).cpu().detach().numpy() wandb.log({"Part Means": mu_norm}) wandb.log({"Precision Matrix": L_inv_norm}) # Zero out gradients optimizer.zero_grad() loss.backward() optimizer.step() # Track Loss if step == 0: loss_log = torch.tensor([loss]) rec_loss_log = torch.tensor([rec_loss]) else: loss_log = torch.cat([loss_log, torch.tensor([loss])]) rec_loss_log = torch.cat( [rec_loss_log, torch.tensor([rec_loss])]) training_loss = torch.mean(loss_log) training_rec_loss = torch.mean(rec_loss_log) wandb.log({"Training Loss": training_loss}) wandb.log({"Training Rec Loss": training_rec_loss}) print(f'Epoch: {epoch}, Train Loss: {training_loss}') # Evaluate on Test Set model.eval() for step, original in enumerate(test_loader): with torch.no_grad(): original = original.to(device) tps_param_dic = tps_parameters(original.shape[0], arg.scal, arg.tps_scal, arg.rot_scal, arg.off_scal, arg.scal_var, arg.augm_scal) coord, vector = make_input_tps_param(tps_param_dic) coord, vector = coord.to(device), vector.to(device) image_spatial_t, _ = ThinPlateSpline( original, coord, vector, original.shape[3], device) image_appearance_t = K.ColorJitter( arg.brightness, arg.contrast, arg.saturation, arg.hue)(original) image_spatial_t, image_appearance_t = normalize( image_spatial_t), normalize(image_appearance_t) reconstruction, loss, rec_loss, equiv_loss, mu, L_inv = model( original, image_spatial_t, image_appearance_t, coord, vector) if step == 0: loss_log = torch.tensor([loss]) else: loss_log = torch.cat( [loss_log, torch.tensor([loss])]) evaluation_loss = torch.mean(loss_log) wandb.log({"Evaluation Loss": evaluation_loss}) print(f'Epoch: {epoch}, Test Loss: {evaluation_loss}') # Track Progress if True: model.mode = 'predict' original, fmap_shape, fmap_app, reconstruction = model( original, image_spatial_t, image_appearance_t, coord, vector) make_visualization(original, reconstruction, image_spatial_t, image_appearance_t, fmap_shape, fmap_app, model_save_dir, epoch, device) save_model(model, model_save_dir) elif mode == 'predict': # Make Directory for Predictions model_save_dir = '../results/' + name if not os.path.exists(model_save_dir + '/predictions'): os.makedirs(model_save_dir + '/predictions') # Load Model and Dataset model = Model(arg).to(device) model = load_model(model, model_save_dir).to(device) data = load_deep_fashion_dataset() test_data = np.array(data[-4:]) test_dataset = ImageDataset(test_data) test_loader = DataLoader(test_dataset, batch_size=bn) model.mode = 'predict' model.eval() # Predict on Dataset for step, original in enumerate(test_loader): with torch.no_grad(): original = original.to(device) tps_param_dic = tps_parameters(original.shape[0], arg.scal, arg.tps_scal, arg.rot_scal, arg.off_scal, arg.scal_var, arg.augm_scal) coord, vector = make_input_tps_param(tps_param_dic) coord, vector = coord.to(device), vector.to(device) image_spatial_t, _ = ThinPlateSpline(original, coord, vector, original.shape[3], device) image_appearance_t = K.ColorJitter(arg.brightness, arg.contrast, arg.saturation, arg.hue)(original) image, reconstruction, mu, shape_stream_parts, heat_map = model( original, image_spatial_t, image_appearance_t, coord, vector)
def main2(arg): # Get args bn = arg.bn mode = arg.mode name = arg.name load_from_ckpt = arg.load_from_ckpt lr = arg.lr weight_decay = arg.weight_decay epochs = arg.epochs device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') arg.device = device if mode == 'train': # Make new directory model_save_dir = name + "/training2" if not os.path.exists(model_save_dir): os.makedirs(model_save_dir) os.makedirs(model_save_dir + '/image') os.makedirs(model_save_dir + '/reconstruction') os.makedirs(model_save_dir + '/mu') os.makedirs(model_save_dir + '/parts') # Load Datasets train_data = load_images_from_folder()[:100] train_dataset = ImageDataset2(train_data, arg) test_data = load_images_from_folder()[-1000:] test_dataset = ImageDataset2(test_data, arg) # Prepare Dataloader & Instances train_loader = DataLoader(train_dataset, batch_size=bn, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=bn) model = Model2(arg).to(device) if load_from_ckpt == True: model = load_model(model, model_save_dir).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) # Make Training for epoch in range(epochs): # Train on Train Set model.train() model.mode = 'train' for step, original in enumerate(train_loader): original = original.to(device, dtype=torch.float) optimizer.zero_grad() # plot_tensor(original[0]) # plot_tensor(spat[0]) # plot_tensor(app[0]) # plot_tensor(original[1]) # plot_tensor(spat[1]) # plot_tensor(app[1]) # print(coord, vec) prediction, loss = model(original) loss.backward() optimizer.step() if epoch % 2 == 0 and step == 0: print(f'Epoch: {epoch}, Train Loss: {loss}') # Evaluate on Test Set model.eval() for step, original in enumerate(test_loader): with torch.no_grad(): original = original.to(device, dtype=torch.float) prediction, loss = model(original) if epoch % 2 == 0 and step == 0: print(f'Epoch: {epoch}, Test Loss: {loss}') # Track Progress if epoch % 5 == 0: model.mode = 'predict' image, reconstruction, mu, shape_stream_parts, heat_map = model( original) for i in range(len(image)): save_image( image[i], model_save_dir + '/image/' + str(i) + '_' + str(epoch) + '.png') save_image( reconstruction[i], model_save_dir + '/image/' + str(i) + '_' + str(epoch) + '.png') #save_image(mu[i], model_save_dir + '/image/' + str(epoch) + '.png') #save_image(shape_stream_parts[i], model_save_dir + '/image/' + str(epoch) + '.png') # Save the current Model if epoch % 50 == 0: save_model(model, model_save_dir) elif arg.mode == 'predict': model_save_dir = arg.name + "/prediction" if not os.path.exists(model_save_dir): os.makedirs(model_save_dir)