def set_model(network, num_classes, include_top=False, weights='imagenet', input_shape=(224, 224, 3)): """ The function constructs the base model with """ if network == 'resnet50': base_model = resnet.ResNet50(include_top=include_top, weights=weights, input_shape=input_shape) elif network == 'resnet101': base_model = resnet.ResNet50(include_top=include_top, weights=weights, input_shape=input_shape) else: raise ValueError( "Oops, wrong network, {} doesn't exist".format(network)) x = base_model.output x = Flatten()(x) x = Dense(1024, activation='relu', name='fc1')(x) x = Dropout(0.7, name='dropout1')(x) x = Dense(128, activation='relu', name='fc2')(x) x = Dropout(0.5, name='dropout2')(x) x = Dense(num_classes, activation='sigmoid')(x) model = Model(inputs=base_model.input, outputs=x) return model
def __init__(self): super().__init__() self.bottom_up = r.ResNet50() # Top layer self.toplayer = nn.Conv2d(2048, 256, kernel_size=1, stride=1, padding=0) # Smooth layers self.smooth1 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1) self.smooth2 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1) self.smooth3 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1) # Lateral layers self.latlayer1 = nn.Conv2d(1024, 256, kernel_size=1, stride=1, padding=0) self.latlayer2 = nn.Conv2d(512, 256, kernel_size=1, stride=1, padding=0) self.latlayer3 = nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0)
def model_STN(cfg, idx_char_dict): if K.image_data_format() == 'channels_first': # input_shape = (cfg.nb_channels, cfg.height, cfg.width) input_shape = (cfg.nb_channels, cfg.height, None) else: # input_shape = (cfg.height, cfg.width, cfg.nb_channels) input_shape = (cfg.height, None, cfg.nb_channels) inputs_data = Input(name='the_input', shape=input_shape, dtype='float32') # if cfg.stn: # if K.image_data_format() == 'channels_first': # x = STN(inputs_data, sampling_size=input_shape[1:]) # else: # x = STN(inputs_data, sampling_size=input_shape[:2]) # else: x = inputs_data y_pred_1 = resnet.ResNet50(x, len(idx_char_dict) + 1) # H = GraphConvolution(16, support, activation='relu', kernel_regularizer=l2(5e-4))([H] + G) prediction_model = Model(inputs=inputs_data, outputs=y_pred_1) # y_pred_2 = GraphConvolution() prediction_model.summary() # labels = Input(name='the_labels', shape=[cfg.label_len], dtype='float32') labels = Input(name='the_labels', shape=[None], dtype='float32') input_length = Input(name='input_length', shape=[1], dtype='int64') label_length = Input(name='label_length', shape=[1], dtype='int64') ctc_loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')( [y_pred_1, labels, input_length, label_length]) training_model = Model( inputs=[inputs_data, labels, input_length, label_length], outputs=[ctc_loss_out]) return training_model, prediction_model
def model_STN(cfg): if K.image_data_format() == 'channels_first': # input_shape = (cfg.nb_channels, cfg.height, cfg.width) input_shape = (cfg.nb_channels, cfg.height, None) else: # input_shape = (cfg.height, cfg.width, cfg.nb_channels) input_shape = (cfg.height, None, cfg.nb_channels) inputs_data = Input(name='the_input', shape=input_shape, dtype='float32') if cfg.stn: if K.image_data_format() == 'channels_first': x = STN(inputs_data, sampling_size=input_shape[1:]) else: x = STN(inputs_data, sampling_size=input_shape[:2]) y_pred_1 = resnet.ResNet50(x, len(cfg.characters)) prediction_model = Model(inputs=inputs_data, outputs=y_pred_1) y_pred_2 = GraphConvolution() prediction_model.summary() labels = Input(name='the_labels', shape=[cfg.label_len], dtype='float32') input_length = Input(name='input_length', shape=[1], dtype='int64') label_length = Input(name='label_length', shape=[1], dtype='int64') ctc_loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')( [y_pred_1, labels, input_length, label_length]) training_model = Model( inputs=[inputs_data, labels, input_length, label_length], outputs=[ctc_loss_out]) return training_model, prediction_model
def __init__(self, dropout_rate, feat_length=512, archi_type='resnet18'): super(CIFAR10FeatureLayer, self).__init__() self.archi_type = archi_type self.feat_length = feat_length if self.archi_type == 'default': self.add_module('conv1', nn.Conv2d(3, 32, kernel_size=3, padding=1)) self.add_module('bn1', nn.BatchNorm2d(32)) self.add_module('relu1', nn.ReLU()) self.add_module('pool1', nn.MaxPool2d(kernel_size=2)) #self.add_module('drop1', nn.Dropout(dropout_rate)) self.add_module('conv2', nn.Conv2d(32, 32, kernel_size=3, padding=1)) self.add_module('bn2', nn.BatchNorm2d(32)) self.add_module('relu2', nn.ReLU()) self.add_module('pool2', nn.MaxPool2d(kernel_size=2)) #self.add_module('drop2', nn.Dropout(dropout_rate)) self.add_module('conv3', nn.Conv2d(32, 64, kernel_size=3, padding=1)) self.add_module('bn3', nn.BatchNorm2d(64)) self.add_module('relu3', nn.ReLU()) self.add_module('pool3', nn.MaxPool2d(kernel_size=2)) #self.add_module('drop3', nn.Dropout(dropout_rate)) elif self.archi_type == 'resnet18': self.add_module('resnet18', resnet.ResNet18(feat_length)) elif self.archi_type == 'resnet50': self.add_module('resnet50', resnet.ResNet50(feat_length)) elif self.archi_type == 'resnet152': self.add_module('resnet152', resnet.ResNet152(feat_length)) else: raise NotImplementedError
def get_model(args): if args.model == 'resnet18': model = resnet.ResNet18(num_classes=10) elif args.model == 'resnet50': model = resnet.ResNet50(num_classes=10) elif args.model == 'densenet40': model = densenet.DenseNet3(depth=40, num_classes=10) elif args.model == 'vgg16': model = vgg.VGG('VGG16') return model
def res_down(): inputs = Input(shape=(None, None, 3)) rn50 = resnet.ResNet50(input_tensor=inputs) p0 = rn50.get_layer('conv1_relu').output p1 = rn50.get_layer('conv2_block3_out').output p2 = rn50.get_layer('conv3_block4_out').output p3 = rn50.get_layer('conv4_block6_out').output feature = Conv2D(256, kernel_size=1, use_bias=False)(p3) feature = BatchNormalization()(feature) model = Model(inputs=inputs, outputs=[p0, p1, p2, feature]) return model
def get_model(training, img_h, nclass): input_shape = (None, img_h, 1) # (128, 64, 1) # Make Networkw inputs = Input(name='the_input', shape=input_shape, dtype='float32') # (None, 128, 64, 1) inner = resnet.ResNet50(include_top=False, weights = None, input_tensor = inputs) # Convolution layer (VGG) # CNN to RNN #inner = Reshape(target_shape=((32, 2048)), name='reshape')(inner) # (None, 32, 2048) inner = TimeDistributed(Flatten(), name='flatten')(inner) #inner = Dense(64, activation='relu', kernel_initializer='he_normal', name='dense1')(inner) # (None, 32, 64) lstm_unit_num = 256 # RNN layer lstm_1 = LSTM(lstm_unit_num, return_sequences=True, kernel_initializer='he_normal', name='lstm1')(inner) # (None, 32, 512) lstm_1b = LSTM(lstm_unit_num, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='lstm1_b')(inner) lstm1_merged = add([lstm_1, lstm_1b]) # (None, 32, 512) lstm1_merged = BatchNormalization()(lstm1_merged) #lstm1_merged = Dropout(0.1)(lstm1_merged) lstm_2 = LSTM(lstm_unit_num, return_sequences=True, kernel_initializer='he_normal', name='lstm2')(lstm1_merged) lstm_2b = LSTM(lstm_unit_num, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='lstm2_b')(lstm1_merged) lstm2_merged = concatenate([lstm_2, lstm_2b]) # (None, 32, 1024) lstm_merged = BatchNormalization()(lstm2_merged) #lstm_merged = Dropout(0.1)(lstm_merged) # transforms RNN output to character activations: inner = Dense(nclass, kernel_initializer='he_normal',name='dense2')(lstm2_merged) #(None, 32, 63) y_pred = Activation('softmax', name='softmax')(inner) labels = Input(name='the_labels', shape=[None], dtype='float32') # (None ,8) input_length = Input(name='input_length', shape=[1], dtype='int64') # (None, 1) label_length = Input(name='label_length', shape=[1], dtype='int64') # (None, 1) # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([y_pred, labels, input_length, label_length]) #(None, 1) model = None if training: model = Model(inputs=[inputs, labels, input_length, label_length], outputs=loss_out) else: model = Model(inputs=inputs, outputs=y_pred) return model, model model.summary() multi_model = multi_gpu_model(model, gpus=GPU_NUM) save_model = model ada = Adadelta() #multi_model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer='adam', metrics=['accuracy']) multi_model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=ada, metrics=['accuracy']) return save_model, multi_model
def build_backbone(backbone, output_stride, BatchNorm, detach=False): if backbone == 'resnet50': return resnet.ResNet50(output_stride, BatchNorm, detach=detach) elif backbone == 'resnet101': return resnet.ResNet101(output_stride, BatchNorm, detach=detach) # elif backbone == 'xception': # return xception.AlignedXception(output_stride, BatchNorm) # elif backbone == 'drn': # return drn.drn_d_54(BatchNorm) # elif backbone == 'mobilenet': # return mobilenet.MobileNetV2(output_stride, BatchNorm) else: raise NotImplementedError
def __init__(self, n_classes, init_lr=0.1, momentum=0.9, weight_decay=5e-4, device='cuda', log_dir='', ckpt_file='', model='resnet-10', multi_gpu=True): super().__init__(n_classes, init_lr, momentum, weight_decay, device) self.n_planes = [64, 128, 256, 512] if model == 'resnet-10': self.net = resnet.ResNet10(n_classes=self.n_classes, n_output_planes=self.n_planes) if model == 'resnet-18': self.net = resnet.ResNet18(n_classes=self.n_classes, n_output_planes=self.n_planes) elif model == 'resnet-34': self.net = resnet.ResNet34(n_classes=self.n_classes, n_output_planes=self.n_planes) elif model == 'resnet-50': self.net = resnet.ResNet50(n_classes=self.n_classes, n_output_planes=self.n_planes) self.net.to(self.device) if ckpt_file: print('loading pretrained classifier checkpoint') if device == 'cpu': ckpt = torch.load(ckpt_file, map_location=lambda storage, loc: storage) else: ckpt = torch.load(ckpt_file) self.net.load_state_dict(ckpt['clf']) if multi_gpu and self.device == 'cuda': print('replicating model on multiple gpus ... ') self.net = torch.nn.DataParallel(self.net) self.optim = torch.optim.SGD(self.net.parameters(), self.init_lr, momentum=self.momentum, weight_decay=self.weight_decay) self.criterion = torch.nn.CrossEntropyLoss().to(self.device) print('Number of dnn parameters: {}'.format( sum([p.data.nelement() for p in self.net.parameters()]))) if log_dir: utils.save_model_desc(self.net, join(log_dir, 'classifier_desc.txt'))
def get_compile_model(): # get model res_output, img_input = re.ResNet50(params["img_shape"], 5) model = Model(inputs=img_input, outputs=[res_output]) model.summary() '''Compile model''' Adam = adam(lr=params["learning_rate"], beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer=Adam, loss="categorical_crossentropy", metrics=["accuracy", fmeasure]) return model
def get_model(model_name, parameters): device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') if model_name == 'resnet18': net = resnet.ResNet18(parameters, num_classes=10).to(device) elif model_name == 'resnet34': net = resnet.ResNet34(parameters, num_classes=10).to(device) elif model_name == 'resnet50': net = resnet.ResNet50(parameters, num_classes=10).to(device) elif model_name == 'resnet101': net = resnet.ResNet101(parameters, num_classes=10).to(device) elif model_name == 'resnet152': net = resnet.ResNet152(parameters, num_classes=10).to(device) elif model_name == 'vgg16': net = 0 else: print("Entered student model is not compatibale currently!\n") net = -1 return net
def get_model(device): """ :param device: instance of torch.device :return: An instance of torch.nn.Module """ num_classes = 2 if config["dataset"] == "Cifar100": num_classes = 100 elif config["dataset"] == "Cifar10": num_classes = 10 elif config["dataset"] == "15-Scene": num_classes = 15 elif config["dataset"] == "MNIST": num_classes = 10 model = { "resnet10": lambda: resnet.ResNet10(num_classes=num_classes), "resnet18": lambda: resnet.ResNet18(num_classes=num_classes), "resnet34": lambda: resnet.ResNet34(num_classes=num_classes), "resnet50": lambda: resnet.ResNet50(num_classes=num_classes), "resnet101": lambda: resnet.ResNet101(num_classes=num_classes), "resnet152": lambda: resnet.ResNet152(num_classes=num_classes), "bert": lambda: modeling_bert_appendix.BertImage(config, num_classes=num_classes), }[config["model"]]() model.to(device) if device == "cuda": # model = torch.nn.DataParallel(model) # multiple GPUs not available # for free on Google Colab -EU torch.backends.cudnn.benchmark = True return model
transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])) data_test = MNIST('data', train=False, download=True, transform=transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])) data_train_loader = DataLoader(data_train, batch_size=256, shuffle=True, num_workers=8) data_test_loader = DataLoader(data_test, batch_size=1024, num_workers=8) net = resnet.ResNet50(num_classes=10).cuda() criterion = torch.nn.CrossEntropyLoss().cuda() optimizer = torch.optim.Adam(net.parameters(), lr=0.001) if args.dataset == 'cifar10': transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
def main(): hvd.init() gpus = tf.config.experimental.list_physical_devices('GPU') for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) if gpus: tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()], 'GPU') tf.config.threading.intra_op_parallelism_threads = 1 # Avoid pool of Eigen threads tf.config.threading.inter_op_parallelism_threads = max( 2, 40 // hvd.size() - 2) os.environ['TF_CUDNN_DETERMINISTIC'] = '1' cmdline = add_cli_args() FLAGS, unknown_args = cmdline.parse_known_args() if not FLAGS.xla_off: tf.config.optimizer.set_jit(True) if not FLAGS.fp32: tf.config.optimizer.set_experimental_options( {"auto_mixed_precision": True}) data = create_dataset(FLAGS.train_data_dir, FLAGS.batch_size, validation=False) validation_data = create_dataset(FLAGS.validation_data_dir, FLAGS.batch_size, validation=True) if FLAGS.model == 'resnet50': if not FLAGS.fine_tune: model = resnet.ResNet50(weights=None, weight_decay=FLAGS.l2_weight_decay, pooling='avg', classes=FLAGS.num_classes) else: model = resnet.ResNet50(weights='imagenet', classes=FLAGS.num_classes) elif FLAGS.model == 'resnet50v2': if not FLAGS.fine_tune: model = resnet.ResNet50V2(weights=None, weight_decay=FLAGS.l2_weight_decay, pooling='avg', classes=FLAGS.num_classes) else: model = resnet.ResNet50V2(weights='imagenet', classes=FLAGS.num_classes) elif FLAGS.model == 'resnet50v2_evo': if not FLAGS.fine_tune: model = resnet_evo.ResNet50V2(weights=None, weight_decay=FLAGS.l2_weight_decay, pooling='avg', classes=FLAGS.num_classes) elif FLAGS.model == 'darknet53': model = darknet.Darknet(weight_decay=FLAGS.l2_weight_decay) model.summary() learning_rate = (FLAGS.learning_rate * hvd.size() * FLAGS.batch_size) / 256 steps_per_epoch = int( (FLAGS.train_dataset_size / (FLAGS.batch_size * hvd.size()))) scheduler = tf.keras.optimizers.schedules.PiecewiseConstantDecay( boundaries=[ steps_per_epoch * 25, steps_per_epoch * 55, steps_per_epoch * 75 ], # 5 epochs for warmup values=[ learning_rate, learning_rate * 0.1, learning_rate * 0.01, learning_rate * 0.001 ]) scheduler = WarmupScheduler(optimizer=scheduler, initial_learning_rate=learning_rate / hvd.size(), warmup_steps=steps_per_epoch * 5) opt = tf.keras.optimizers.SGD( learning_rate=scheduler, momentum=FLAGS.momentum, nesterov=True) # needs momentum correction term if not FLAGS.fp32: opt = tf.train.experimental.enable_mixed_precision_graph_rewrite( opt, loss_scale=128.) loss_func = tf.keras.losses.CategoricalCrossentropy( label_smoothing=FLAGS.label_smoothing, reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE) if hvd.rank() == 0: model_dir = os.path.join( FLAGS.model + datetime.datetime.now().strftime("_%Y-%m-%d_%H-%M-%S")) path_logs = os.path.join(os.getcwd(), model_dir, 'log.csv') os.mkdir(model_dir) logging.basicConfig( filename=path_logs, filemode='a', format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.DEBUG) logging.info("Training Logs") logger = logging.getLogger('logger') logger.info('Batch Size: %f, Learning Rate: %f, Momentum: %f' % \ (FLAGS.batch_size, FLAGS.learning_rate, FLAGS.momentum)) hvd.allreduce(tf.constant(0)) start_time = time() curr_step = tf.Variable(initial_value=0, dtype=tf.int32) best_validation_accuracy = 0.0 for epoch in range(FLAGS.num_epochs): if hvd.rank() == 0: print('Starting training Epoch %d/%d' % (epoch, FLAGS.num_epochs)) training_score = 0 for batch, (images, labels) in enumerate(tqdm(data)): # momentum correction (V2 SGD absorbs LR into the update term) prev_lr = opt._optimizer.learning_rate(curr_step - 1) curr_lr = opt._optimizer.learning_rate(curr_step) momentum_correction_factor = curr_lr / prev_lr opt._optimizer.momentum = opt._optimizer.momentum * momentum_correction_factor loss, score = train_step(model, opt, loss_func, images, labels, batch == 0 and epoch == 0, fp32=FLAGS.fp32) # restore momentum opt._optimizer.momentum = FLAGS.momentum training_score += score.numpy() curr_step.assign_add(1) training_accuracy = training_score / (FLAGS.batch_size * (batch + 1)) average_training_accuracy = hvd.allreduce( tf.constant(training_accuracy)) average_training_loss = hvd.allreduce(tf.constant(loss)) if hvd.rank() == 0: print('Starting validation Epoch %d/%d' % (epoch, FLAGS.num_epochs)) validation_score = 0 counter = 0 for images, labels in tqdm(validation_data): loss, score = validation_step(images, labels, model, loss_func) validation_score += score.numpy() counter += 1 validation_accuracy = validation_score / (FLAGS.batch_size * counter) average_validation_accuracy = hvd.allreduce( tf.constant(validation_accuracy)) average_validation_loss = hvd.allreduce(tf.constant(loss)) if hvd.rank() == 0: info_str = 'Epoch: %d, Train Accuracy: %f, Train Loss: %f, Validation Accuracy: %f, Validation Loss: %f LR:%f' % \ (epoch, average_training_accuracy, average_training_loss, average_validation_accuracy, average_validation_loss, scheduler(curr_step)) print(info_str) logger.info(info_str) if average_validation_accuracy > best_validation_accuracy: logger.info("Found new best accuracy, saving checkpoint ...") best_validation_accuracy = average_validation_accuracy model.save('{}-best/{}'.format(FLAGS.model_dir, FLAGS.model)) if hvd.rank() == 0: logger.info('Total Training Time: %f' % (time() - start_time))
def evaluate(MAX_STEP, tf_file, logs_test_dir, restore_dir, loss_dir, rgb_write_dir): filenames = tf.placeholder(tf.string, shape=[None]) validation_filenames = tf_file iterator = data.read_and_decode(filenames, BATCH_SIZE, False) sess = tf.Session() sess.run(iterator.initializer, feed_dict={filenames: validation_filenames}) test_imgs, test_labels = iterator.get_next() test_logits, c5 = resnet.ResNet50(test_imgs, False) test_p2, test_p3, test_p4, test_p5 = resnet.FPN(test_logits, False) test_est_labels2, test_fea_rgbs2= resnet.cal_loss(test_p2, False, 'p2', is_weight) test_est_labels3, test_fea_rgbs3= resnet.cal_loss(test_p3, False, 'p3', is_weight) test_est_labels4, test_fea_rgbs4= resnet.cal_loss(test_p4, False, 'p4', is_weight) test_est_labels5, test_fea_rgbs5= resnet.cal_loss(test_p5, False, 'p5', is_weight) test_est_labels = (test_est_labels2 + test_est_labels3 + test_est_labels4 + test_est_labels5)/4.0 # test_est_labels, test_fea_rgbs= resnet.cal_loss(test_logits, False, 'logits') test_losss2 = resnet.losses(test_est_labels2, test_labels) test_losss3 = resnet.losses(test_est_labels3, test_labels) test_losss4 = resnet.losses(test_est_labels4, test_labels) test_losss5 = resnet.losses(test_est_labels5, test_labels) visualization2 = utils.get_visualization(test_imgs, test_fea_rgbs2, test_est_labels2, test_labels, BATCH_SIZE) visualization3 = utils.get_visualization(test_imgs, test_fea_rgbs3, test_est_labels3, test_labels, BATCH_SIZE) visualization4 = utils.get_visualization(test_imgs, test_fea_rgbs4, test_est_labels4, test_labels, BATCH_SIZE) visualization5 = utils.get_visualization(test_imgs, test_fea_rgbs5, test_est_labels5, test_labels, BATCH_SIZE) test_losss = resnet.losses(test_est_labels, test_labels) summary_op = tf.summary.merge_all() train_writer = tf.summary.FileWriter(logs_test_dir, sess.graph) saver = tf.train.Saver() saver.restore(sess, restore_dir) try: for step in range(MAX_STEP): test_ill_label, test_est_label, test_loss, test_loss2, test_loss3, test_loss4, \ test_loss5, vis_img2, vis_img3, vis_img4, vis_img5, summary_str = sess.run( [test_labels, test_est_labels, test_losss, test_losss2, test_losss3, test_losss4, test_losss5, visualization2, visualization3, \ visualization4, visualization5, summary_op]) print('label:', test_ill_label) print('estimate:', test_est_label) print('Step %d, test loss = %.2f' % (step + 1, test_loss),\ 'loss2 = %.2f loss3 = %.2f loss4 = %.2f loss5 = %.2f' \ % (test_loss2, test_loss3, test_loss4, test_loss5)) list_loss.append(test_loss) with open(loss_dir, 'a') as f: f.write('%.2f ' % (test_loss)) f.write('%.2f ' % (test_loss2)) f.write('%.2f ' % (test_loss3)) f.write('%.2f ' % (test_loss4)) f.write('%.2f ' % (test_loss5)) f.write(str(test_ill_label)) f.write(str(test_est_label)) f.write("\n") # for k, merged in enumerate(vis_img2): # cv.imwrite(visualize.format(step + 1, k + 4, test_loss2).strip('\r'), # merged * 255) # for k, merged in enumerate(vis_img3): # cv.imwrite(visualize.format(step + 1, k + 3, test_loss3).strip('\r'), # merged * 255) # for k, merged in enumerate(vis_img4): # cv.imwrite(visualize.format(step + 1, k + 2, test_loss4).strip('\r'), # merged * 255) # for k, merged in enumerate(vis_img5): # cv.imwrite(visualize.format(step + 1, k + 1, test_loss5).strip('\r'), # merged * 255) train_writer.add_summary(summary_str, step) if step % 2000 == 0 or (step + 1) == MAX_STEP: checkpoint_path = os.path.join(logs_test_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=step) except tf.errors.OutOfRangeError: print('Done training -- epoch limit reached') sess.close() metric(list_loss)
dtload_train = DTloader(folder_loc + '/data/train', train=True, transform=transformTrain) DT_train = torch.utils.data.DataLoader(dtload_train, batch_size=batch_sizes, shuffle=True) dtload_val = DTloader(folder_loc + '/data/val', train=True, transform=transformVal) DT_val = torch.utils.data.DataLoader(dtload_val, batch_size=batch_sizes, shuffle=True) model = resnet.ResNet50().cuda() model = torch.nn.DataParallel(model).cuda() optimizer = optim.Adam(model.parameters(), lr=learning_rate, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-5) scheduler = ReduceLROnPlateau(optimizer, factor=0.1, patience=patience, mode='min') criterion = nn.L1Loss() lossMIN = 100000
def main(): global lr, best_prec1 epochs = 50 model = resnet.ResNet50(10) # model = nn.Sequential(nn.BatchNorm2d(num_features=3, affine=False), pretrained) # num_ftrs = model.fc.in_features # model.fc = nn.Linear(num_ftrs, 10) model.cuda() criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), lr, momentum=momentum, weight_decay=weight_decay) # lr_scheduler = torch.optim.lr_scheduler.MultiStepLR( # optimizer, milestones=[25, 50, 75], gamma=0.2) # train_loader, val_loader, class_names = datasets.get_train_test_dataset(dataset, batch_size, True) # dataiter = iter(train_loader) # images, labels = dataiter.next() transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) train_loader = torch.utils.data.DataLoader(datasets.CIFAR10( root='data/cifar10-data', train=True, download=True, transform=transform), batch_size=batch_size, shuffle=True) test_indices = list(range(100)) val_loader = torch.utils.data.DataLoader( datasets.CIFAR10(root='data/cifar10-data', train=False, download=True, transform=transform), sampler=torch.utils.data.sampler.SubsetRandomSampler(test_indices), batch_size=batch_size) for epoch in range(epochs): adjust_learning_rate(optimizer, epoch, lr) # train for one epoch train_epoch(train_loader, model, criterion, optimizer, epoch) # lr_scheduler.step() # evaluate on validation set prec1 = validate_epoch(val_loader, model, criterion) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best)
classes = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'd', 'e', 'f', 'g', 'h', 'n', 'q', 'r', 't',) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(device) if layers == 18: net = resnet.ResNet18(num_class=num_class, channels=channels).to(device) elif layers == 34: net = resnet.ResNet34(num_class=num_class, channels=channels).to(device) elif layers == 50: net = resnet.ResNet50(num_class=num_class, channels=channels).to(device) elif layers == 101: net = resnet.ResNet101(num_class=num_class, channels=channels).to(device) else: net = resnet.ResNet152(num_class=num_class, channels=channels).to(device) print(net) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) #scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[3, 7], gamma=0.1) best_score = 0.0 for epoch in range(num_epoch): # エポック数
def train(tf_file, checkpoint, visualize): restore_dir = checkpoint learning_rate = tf.Variable(0.0001, trainable=False) decay_lr = tf.assign(learning_rate, tf.maximum(learning_rate / 10.0, 1e-6)) filenames = tf.placeholder(tf.string, shape=[None]) training_filenames = tf_file with tf.device('/cpu:0'): iterator = data.read_and_decode(filenames, BATCH_SIZE, True) sess = tf.Session() sess.run(iterator.initializer, feed_dict={filenames: training_filenames}) tra_imgs, tra_labels = iterator.get_next() logits, tra_c5 = resnet.ResNet50(tra_imgs, True) p2, p3, p4, p5 = resnet.FPN(logits, True) tra_est_labels2, tra_fea_rgbs2= resnet.cal_loss(p2, True, 'p2', is_weight) tra_est_labels3, tra_fea_rgbs3= resnet.cal_loss(p3, True, 'p3', is_weight) tra_est_labels4, tra_fea_rgbs4= resnet.cal_loss(p4, True, 'p4', is_weight) tra_est_labels5, tra_fea_rgbs5= resnet.cal_loss(p5, True, 'p5', is_weight) tra_losss2 = resnet.losses(tra_est_labels2, tra_labels) tra_losss3 = resnet.losses(tra_est_labels3, tra_labels) tra_losss4 = resnet.losses(tra_est_labels4, tra_labels) tra_losss5 = resnet.losses(tra_est_labels5, tra_labels) tra_losss = (tra_losss2 + tra_losss3 + tra_losss4 + tra_losss5)/4.0 tra_est_labels = (tra_est_labels2 + tra_est_labels3 + tra_est_labels4 + tra_est_labels5) / 4.0 tra_fea_rgbs = tra_fea_rgbs2 # tra_est_labels, tra_fea_rgbs= resnet.cal_loss(logits, True, 'logits') # tra_losss = resnet.losses(tra_est_labels, tra_labels) saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'resnet_model')) train_op = resnet.trainning(tra_losss, learning_rate) visualization = utils.get_visualization(tra_imgs, tra_fea_rgbs, tra_est_labels, tra_labels, BATCH_SIZE) summary_op = tf.summary.merge_all() train_writer = tf.summary.FileWriter(checkpoint, sess.graph) try: load_checkpoint(restore_dir, sess) except: sess.run(tf.global_variables_initializer()) saver.restore(sess, './resnet_imagenet_v2/model.ckpt-225207') print("restore succed!") saver = tf.train.Saver() try: for epoch in range(sess.run(it_cnt), END_EPOCH): sess.run(update_cnt) batches_per_epoch = train_num // BATCH_SIZE start_time = time.time() for j in range(batches_per_epoch): _, lr_value, tra_loss, vis_img = sess.run( [train_op, learning_rate, tra_losss, visualization]) duration_time = time.time() - start_time if epoch % 10 == 0: # tra_weight.imshow() print('Step %d, train loss = %.2f' % (epoch, tra_loss)) print("{} images per secs, lr = {}".format(batches_per_epoch * BATCH_SIZE / duration_time, lr_value)) summary_str = sess.run(summary_op) train_writer.add_summary(summary_str, epoch) # if epoch % 200 == 0: # for k, merged in enumerate(vis_img): # if k % 8==0: # cv2.imwrite(visualize.format(epoch + 1, k + 1, tra_loss).strip('\r'), # merged * 255) if (epoch + 1) % 2000 == 0 or (epoch + 1) == END_EPOCH: print("start decay lr") sess.run(decay_lr) checkpoint_path = os.path.join(checkpoint, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=epoch) except tf.errors.OutOfRangeError: print('Done training -- epoch limit reached') sess.close()
def main(): global args, best_prec1 args = parser.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu if args.dataset == 'cifar10': mean = (0.4914, 0.4822, 0.4465) std = (0.2023, 0.1994, 0.2010) else: mean = (0.5071, 0.4867, 0.4408) std = (0.2675, 0.2565, 0.2761) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean, std), ]) # meanstd transformation transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std), ]) if (args.dataset == 'cifar10'): print("| Preparing CIFAR-10 dataset...") sys.stdout.write("| ") trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=False, transform=transform_test) num_classes = 10 elif (args.dataset == 'cifar100'): print("| Preparing CIFAR-100 dataset...") sys.stdout.write("| ") trainset = torchvision.datasets.CIFAR100(root='./data', train=True, download=True, transform=transform_train) testset = torchvision.datasets.CIFAR100(root='./data', train=False, download=False, transform=transform_test) num_classes = 100 # Creating data indices for training and validation splits: validation_split = 0.2 dataset_size = len(trainset) num_val = int(np.floor(validation_split * dataset_size)) trainset, valset = torch.utils.data.random_split( trainset, [dataset_size - num_val, num_val]) train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batchsize, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(valset, batch_size=100, shuffle=True, num_workers=args.workers, pin_memory=True) test_loader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=args.workers, pin_memory=True) # Check the save_dir exists or not if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) if args.arch == 'resnet50': model = resnet.ResNet50(num_classes) elif args.arch == 'resnet18': model = resnet.ResNet18(num_classes) model = torch.nn.DataParallel(model) model.cuda() # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})".format( args.evaluate, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # define loss function (criterion) and pptimizer if args.coeff == -1: criterion = nn.CrossEntropyLoss().cuda() else: criterion = BeliefMatchingLoss(args.coeff, args.prior) eval_criterion = nn.CrossEntropyLoss().cuda() if args.coeff == -1: name = '{}-softmax'.format(args.arch) else: name = '{}-bm-coeff{}-prior{}'.format(args.arch, args.coeff, args.prior) name += '-{}'.format(args.dataset) optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weightdecay) lr_scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer, milestones=[100, 150], last_epoch=args.start_epoch - 1) # Warmup for param_group in optimizer.param_groups: param_group['lr'] = args.lr * 0.1 if args.evaluate: validate(val_loader, model, criterion) return for epoch in range(args.start_epoch, args.epochs): if epoch < 6 and epoch > 0: param_group['lr'] = args.lr * 0.1 * (epoch * 2) # train for one epoch print('current lr {:.5e}'.format(optimizer.param_groups[0]['lr'])) train(train_loader, model, criterion, optimizer, epoch) lr_scheduler.step() # evaluate on validation set prec1 = validate(val_loader, model, criterion) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) if is_best: test_prec1 = validate(test_loader, model, criterion) save_checkpoint( { 'state_dict': model.state_dict(), 'best_prec1': test_prec1, 'epoch': epoch, }, is_best, filename=os.path.join(args.save_dir, 'checkpoint_{}.th'.format(name)))
def main(args): # Data Loader (Input Pipeline) model_str = args.dataset + '_%s_' % args.model_type + args.noise_type + '_' + str(args.noise_rate) + '_' + str(args.seed) txtfile = save_dir + "/" + model_str + ".txt" nowTime = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S') if os.path.exists(txtfile): os.system('mv %s %s' % (txtfile, txtfile + ".bak-%s" % nowTime)) # Data Loader (Input Pipeline) print('loading dataset...') train_dataset, val_dataset, test_dataset = load_data(args) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=True) val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) # Define models print('building model...') if args.dataset == 'mnist': clf1 = LeNet() optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1) elif args.dataset == 'fmnist': clf1 = resnet.ResNet50(input_channel=1, num_classes=10) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1) elif args.dataset == 'cifar10': clf1 = resnet.ResNet50(input_channel=3, num_classes=10) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1) elif args.dataset == 'cifar100': clf1 = resnet.ResNet50(input_channel=3, num_classes=100) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1) clf1.cuda() with open(txtfile, "a") as myfile: myfile.write('epoch train_acc1 val_acc1 test_acc1\n') epoch = 0 train_acc1 = 0 # evaluate models with random weights val_acc1 = evaluate(val_loader, clf1) print('Epoch [%d/%d] Val Accuracy on the %s val data: Model1 %.4f %%' % ( epoch + 1, args.n_epoch, len(val_dataset), val_acc1)) test_acc1 = evaluate(test_loader, clf1) print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %%' % ( epoch + 1, args.n_epoch, len(test_dataset), test_acc1)) # save results with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n") val_acc_list = [] test_acc_list = [] for epoch in range(0, args.n_epoch): scheduler1.step() print(optimizer1.state_dict()['param_groups'][0]['lr']) clf1.train() train_acc1 = train(train_loader, epoch, clf1, optimizer1, args) val_acc1 = evaluate(val_loader, clf1) val_acc_list.append(val_acc1) test_acc1 = evaluate(test_loader, clf1) test_acc_list.append(test_acc1) # save results print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %% ' % ( epoch + 1, args.n_epoch, len(test_dataset), test_acc1)) with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n") id = np.argmax(np.array(val_acc_list)) test_acc_max = test_acc_list[id] return test_acc_max
def replace_vgg16(model): for i in range(len(model.features)): print(model.features[i]) if isinstance(model.features[i], torch.nn.Conv2d): print("Replaced CONV") model.features[i] = PrunedConv(model.features[i]) for i in range(len(model.classifier)): if isinstance(model.classifier[i], torch.nn.Linear): print("Replaced Linear") model.classifier[i] = PrunedLinear(model.classifier[i]) return model device = 'cuda' if torch.cuda.is_available() else 'cpu' model = resnet.ResNet50() #model = resnet_in.resnet50() #model = vgg16.VGG16() #model = inception_v3_c10.inception_v3() model = model.to(device) replace_with_pruned(model, "model") #model = replace_vgg16(model) #if not torch.distributed.is_initialized(): # port = np.random.randint(10000, 65536) # torch.distributed.init_process_group(backend='nccl', init_method='tcp://127.0.0.1:%d'%port, rank=0, world_size=1) #model = torch.nn.parallel.DistributedDataParallel(model) model.load_state_dict(torch.load(args.path))
def load_weights(model): pre_init = False model.load_weights(os.path.join(params["save_path"], "weights.hdf5"), by_name=True, skip_mismatch=True) print("loading balanced model weights") pre_init = True if not pre_init: print("init random weights") '''train and save model''' # get model res_output, img_input = re.ResNet50(params["img_shape"], 5) model = Model(inputs=img_input, outputs=[res_output]) model.summary() '''Compile model''' sgd = keras.optimizers.SGD(lr=params["learning_rate"], momentum=0.99, decay=0.0, nesterov=False) Adam = adam(lr=params["learning_rate"], beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(optimizer=sgd,
default='vgg16', choices=[ 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', 'vgg16' ], help='Base student model for IKD') args = parser.parse_args() device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') generator = Generator(args.channels, args.img_size, args.latent_dim).to(device) criterion = nn.CrossEntropyLoss().to(device) original_params = [64, 64, 128, 256, 512] if args.dataset == 'MNIST': teacher = resnet.ResNet50(original_params, num_classes=10).to(device) teacher.load_state_dict(torch.load(args.teacher_dict)) teacher.eval() data_test = MNIST(args.data, train=False, transform=transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])) data_test_loader = DataLoader(data_test, batch_size=64, num_workers=1, shuffle=False)
def main(use_cuda): """ Advbox demo which demonstrate how to use advbox. """ class_dim = 1000 IMG_NAME = 'img' LABEL_NAME = 'label' #模型路径 pretrained_model = "models/resnet_50/115" with_memory_optimization = 1 image_shape = [3,224,224] image = fluid.layers.data(name=IMG_NAME, shape=image_shape, dtype='float32') # gradient should flow image.stop_gradient = False label = fluid.layers.data(name=LABEL_NAME, shape=[1], dtype='int64') # model definition model = resnet.ResNet50() out = model.net(input=image, class_dim=class_dim) #test_program = fluid.default_main_program().clone(for_test=True) if with_memory_optimization: fluid.memory_optimize(fluid.default_main_program()) # 根据配置选择使用CPU资源还是GPU资源 place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) #加载模型参数 if pretrained_model: def if_exist(var): return os.path.exists(os.path.join(pretrained_model, var.name)) fluid.io.load_vars(exe, pretrained_model, predicate=if_exist) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) # advbox demo m = PaddleModel( fluid.default_main_program(), IMG_NAME, LABEL_NAME, out.name, avg_cost.name, (-1, 1), channel_axis=3) attack = FGSM(m) attack_config = {"epsilons": 0.3} test_data = get_image("cat.jpg") # 猫对应的标签 test_label = 285 adversary = Adversary(test_data, test_label) # FGSM non-targeted attack adversary = attack(adversary, **attack_config) if adversary.is_successful(): print( 'attack success, original_label=%d, adversarial_label=%d' % (test_label, adversary.adversarial_label)) else: print('attack failed, original_label=%d, ' % (test_label)) print("fgsm attack done")
import resnet import numpy as np from keras.preprocessing.image import load_img, img_to_array from keras_applications.imagenet_utils import decode_predictions # ------------------------------------- # Load pre-trained models # ------------------------------------- resnet50 = resnet.ResNet50(weights='imagenet') resnet101 = resnet.ResNet101(weights='imagenet') resnet152 = resnet.ResNet152(weights='imagenet') # ------------------------------------- # Helper functions # ------------------------------------- def path_to_tensor(image_path, target_size): image = load_img(image_path, target_size=target_size) tensor = img_to_array(image) tensor = np.expand_dims(tensor, axis=0) return tensor # ------------------------------------- # Make predictions # ------------------------------------- image_path = 'images/dog.jpeg' image_tensor = path_to_tensor(image_path, (224, 224)) pred_resnet50 = resnet50.predict(image_tensor)
outputs = model(imgs) loss = loss_fn(outputs, labels) optimizer.zero_grad() loss.backward() optimizer.step() loss_train += loss.item() if epoch == 1 or epoch % 10 == 0: print('{} Epoch {}, Training loss {}'.format( datetime.datetime.now(), epoch, loss_train / len(train_loader))) torch.save( net.state_dict(), 'models/firstmodel_' + str(epoch) + '_' + str(loss_train / len(train_loader)) + '.pt') validate(model, train_loader, val_loader, epoch) net = resnet.ResNet50() net = net.to(device) # model = NetResDeep(n_chans1=32, n_blocks=100).to(device=device) optimizer = optim.SGD(net.parameters(), lr=3e-3) loss_fn = nn.CrossEntropyLoss() training_loop( n_epochs=1000, optimizer=optimizer, model=net, loss_fn=loss_fn, train_loader=train_loader, )