class Trainer: def __init__(self, isInjector=True): self.isInjector = isInjector # Input shape cube_shape = config['cube_shape'] self.img_rows = config['cube_shape'][1] self.img_cols = config['cube_shape'][2] self.img_depth = config['cube_shape'][0] self.channels = 1 self.num_classes = 5 self.img_shape = (self.img_rows, self.img_cols, self.img_depth, self.channels) # Configure data loader if self.isInjector: self.dataset_path = config['unhealthy_samples'] self.modelpath = config['modelpath_inject'] else: self.dataset_path = config['healthy_samples'] self.modelpath = config['modelpath_remove'] self.dataloader = DataLoader(dataset_path=self.dataset_path, normdata_path=self.modelpath, img_res=(self.img_rows, self.img_cols, self.img_depth)) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 100 self.df = 100 optimizer = Adam(0.0002, 0.5) optimizer_G = Adam(0.000001, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.summary() self.discriminator.compile(loss='mse', optimizer=optimizer_G, metrics=['accuracy']) # ------------------------- # Construct Computational # Graph of Generator # ------------------------- # Build the generator self.generator = self.build_generator() self.generator.summary() # Input images and their conditioning images img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # By conditioning on B generate a fake version of A fake_A = self.generator([img_B]) # For the combined model we will only train the generator self.discriminator.trainable = False # Discriminators determines validity of translated images / condition pairs valid = self.discriminator([fake_A, img_B]) self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A]) self.combined.compile(loss=['mse', 'mae'], loss_weights=[1, 100], optimizer=optimizer) def build_generator(self): """U-Net Generator""" def get_crop_shape(target, refer): # depth, the 4rth dimension cd = (target.get_shape()[3] - refer.get_shape()[3]).value assert (cd >= 0) if cd % 2 != 0: cd1, cd2 = int(cd / 2), int(cd / 2) + 1 else: cd1, cd2 = int(cd / 2), int(cd / 2) # width, the 3rd dimension cw = (target.get_shape()[2] - refer.get_shape()[2]).value assert (cw >= 0) if cw % 2 != 0: cw1, cw2 = int(cw / 2), int(cw / 2) + 1 else: cw1, cw2 = int(cw / 2), int(cw / 2) # height, the 2nd dimension ch = (target.get_shape()[1] - refer.get_shape()[1]).value assert (ch >= 0) if ch % 2 != 0: ch1, ch2 = int(ch / 2), int(ch / 2) + 1 else: ch1, ch2 = int(ch / 2), int(ch / 2) return (ch1, ch2), (cw1, cw2), (cd1, cd2) def conv3d(layer_input, filters, f_size=4, bn=True): """Layers used during downsampling""" d = Conv3D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d def deconv3d(layer_input, skip_input, filters, f_size=4, dropout_rate=0.5): """Layers used during upsampling""" u = UpSampling3D(size=2)(layer_input) u = Conv3D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u) if dropout_rate: u = Dropout(dropout_rate)(u) u = BatchNormalization(momentum=0.8)(u) # u = Concatenate()([u, skip_input]) ch, cw, cd = get_crop_shape(u, skip_input) crop_conv4 = Cropping3D(cropping=(ch, cw, cd), data_format="channels_last")(u) u = Concatenate()([crop_conv4, skip_input]) return u # Image input d0 = Input(shape=self.img_shape, name="input_image") # Downsampling d1 = conv3d(d0, self.gf, bn=False) d2 = conv3d(d1, self.gf * 2) d3 = conv3d(d2, self.gf * 4) d4 = conv3d(d3, self.gf * 8) d5 = conv3d(d4, self.gf * 8) u3 = deconv3d(d5, d4, self.gf * 8) u4 = deconv3d(u3, d3, self.gf * 4) u5 = deconv3d(u4, d2, self.gf * 2) u6 = deconv3d(u5, d1, self.gf) u7 = UpSampling3D(size=2)(u6) output_img = Conv3D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u7) return Model(inputs=[d0], outputs=[output_img]) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, bn=True): """Discriminator layer""" d = Conv3D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # Concatenate image and conditioning image by channels to produce input model_input = Concatenate(axis=-1)([img_A, img_B]) d1 = d_layer(model_input, self.df, bn=False) d2 = d_layer(d1, self.df * 2) d3 = d_layer(d2, self.df * 4) d4 = d_layer(d3, self.df * 8) validity = Conv3D(1, kernel_size=4, strides=1, padding='same')(d4) return Model([img_A, img_B], validity) def train(self, epochs, batch_size=1, sample_interval=50): start_time = datetime.datetime.now() # Adversarial loss ground truths valid = np.zeros((batch_size, ) + self.disc_patch) fake = np.ones((batch_size, ) + self.disc_patch) for epoch in range(epochs): # save model if epoch > 0: print("Saving Models...") self.generator.save(os.path.join( self.modelpath, "G_model.h5")) # creates a HDF5 file self.discriminator.save( os.path.join( self.modelpath, "D_model.h5")) # creates a HDF5 file 'my_model.h5' for batch_i, (imgs_A, imgs_B) in enumerate( self.dataloader.load_batch(batch_size)): # --------------------- # Train Discriminator # --------------------- # Condition on B and generate a translated version fake_A = self.generator.predict([imgs_B]) # Train the discriminators (original images = real / generated = Fake) d_loss_real = self.discriminator.train_on_batch( [imgs_A, imgs_B], valid) d_loss_fake = self.discriminator.train_on_batch( [fake_A, imgs_B], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # ----------------- # Train Generator # ----------------- # Train the generators g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, imgs_A]) elapsed_time = datetime.datetime.now() - start_time # Plot the progress print( "[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s" % (epoch, epochs, batch_i, self.dataloader.n_batches, d_loss[0], 100 * d_loss[1], g_loss[0], elapsed_time)) # If at save interval => save generated image samples if batch_i % sample_interval == 0: self.show_progress(epoch, batch_i) def show_progress(self, epoch, batch_i): filename = "%d_%d.png" % (epoch, batch_i) if self.isInjector: savepath = os.path.join(config['progress'], "injector") else: savepath = os.path.join(config['progress'], "remover") os.makedirs(savepath, exist_ok=True) r, c = 3, 3 imgs_A, imgs_B = self.dataloader.load_data(batch_size=3, is_testing=True) fake_A = self.generator.predict([imgs_B]) gen_imgs = np.concatenate([imgs_B, fake_A, imgs_A]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 titles = ['Condition', 'Generated', 'Original'] fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(gen_imgs[cnt].reshape( (self.img_rows, self.img_cols, self.img_depth))[16, :, :]) axs[i, j].set_title(titles[i]) axs[i, j].axis('off') cnt += 1 fig.savefig(os.path.join(savepath, filename)) plt.close()
from utils.dataloader import DataLoader import torch from model import model_utils from optimizer.optimizer import NoamOpt from train.trainer import Trainer hidden_size = 256 num_encoder = 6 num_decoder = 6 n_head = 8 pf_dim = 1024 drop_out = 0.5 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') device = 'cpu' dataloader = DataLoader(device) train_iterator, valid_iterator, test_iterator = dataloader.load_data(64) model = model_utils.create_model(dataloader.src_vocab_size(), dataloader.trg_vocab_size(), hidden_size, num_encoder, num_decoder, n_head, pf_dim, drop_out, dataloader.get_pad_idx(), device) print(model_utils.count_parameters(model)) model_utils.init(model) optimizer = NoamOpt(hidden_size , 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) trainer = Trainer(train_iterator, valid_iterator, model, optimizer, dataloader.get_pad_idx(), device) trainer.train(5) # for i, batch in enumerate(train_iterator): # src = batch.src.permute(1, 0).to(device) # trg = batch.trg.permute(1, 0).to(device)
def to_df(csv: str) -> pd.DataFrame: data = pd.read_csv(csv) loader = DataLoader() loader.fit(data) return loader.load_data()
#prediction1 = Predictor().predict(X_test) #loaded_model = pickle.load(open('models/KNN.pickle', 'rb')) #print(loaded_model.score(test_set[x_columns].values, test_set[y_column].values)) #print(accuracy_score(y_test,prediction)) #print(accuracy_score(y,prediction1)) #print(X_test) PREDICT_ROUTE = "http://127.0.0.1:8000/predict" info = specifications['description'] x_columns, y_column, metrics = info['X'], info['y'], info['metrics'] train_set = pd.read_csv(TRAIN_CSV, header=0) test_set = pd.read_csv(VAL_CSV, header=0) train_x, train_y = train_set[x_columns], train_set[y_column] test_x, test_y = test_set[x_columns], test_set[y_column] loader = DataLoader() loader.fit(train_x) train_processed = loader.load_data() loader = DataLoader() loader.fit(test_x) test_processed = loader.load_data() trained = Estimator.fit(train_processed, train_y) trained_predict = Estimator.predict(trained, test_processed) trained_score = round(eval(metrics)(test_y, trained_predict), 2) req_data = {'data': json.dumps(test_x.to_dict())} response = requests.get(PREDICT_ROUTE, data=req_data) api_predict = response.json()['prediction'] api_score = round(eval(metrics)(test_y, api_predict), 2) print(trained_score) print(api_score) assert trained_score == api_score
from utils.dataloader import DataLoader from settings.constants import TRAIN_CSV with open('settings/specifications.json') as f: specifications = json.load(f) raw_train = pd.read_csv(TRAIN_CSV) x_columns = specifications['description']['X'] y_column = specifications['description']['y'] x_raw = raw_train[x_columns] loader = DataLoader() loader.fit(x_raw) X = loader.load_data() y = raw_train.Response model = LogisticRegression(C=0.01, penalty='l1', solver='liblinear') model.fit(X, y) with open('models/log_reg.pickle', 'wb')as f: pickle.dump(model, f) import pickle import json import pandas as pd