def _setup_app_singleton(self): if not self._app or not self._api: self._app = Flask(CONFIG['APP_NAME']) self._app.logger = create_logger(slack=True) self._app.config.from_object('src.config.ChatterConfig') self._api = Api(self._app)
def main(): log = utils.create_logger() if validate.config_validate(): log.info('Load and preprocess data...') df = utils.preprocess_df(filename=config.INI['DIRECTORY']['filename'], features=config.all_features, num_rows=int( config.INI['DIRECTORY']['num_rows'])) log.info('Checking features data type if same as expected...') utils.check_dtype(df, config.features_dtype_int, int) utils.check_dtype(df, config.features_dtype_list, list) log.info('Writing to TFRecords...') utils.write_to_tfrecords(df, config.all_features) log.info('Finished all')
def main(args): device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') makedirs(args.log_root) makedirs(args.model_folder) setattr(args, 'log_folder', args.log_root) setattr(args, 'model_folder', args.model_folder) logger = create_logger(args.log_root, args.todo, 'info') print_args(args, logger) if args.model == 'enet': model = EfficientNet.from_pretrained('efficientnet-b5', num_classes=2) elif args.model == 'xception': model, *_ = model_selection(modelname='xception', num_out_classes=2, init_checkpoint=args.init_load) else: raise NotImplementedError if args.load_checkpoint is not None: if device.type == 'cpu': checkpoint = torch.load(args.load_checkpoint, map_location=torch.device('cpu')) else: checkpoint = torch.load(args.load_checkpoint) model.load_state_dict(checkpoint) if torch.cuda.device_count() > 1: print('GPUs: ', torch.cuda.device_count()) model = nn.DataParallel(model) model = model.to(device) trainer = Trainer(args, logger) if args.todo == 'train': if args.array: transform = transforms.Normalize(mean=[0.5], std=[0.5]) def npy_loader(path): sample = torch.from_numpy(np.load(path)) return sample #BUILD TRAIN SET print("Initializing Dataset...") train_set = DatasetFolderWithPaths(root=args.data_root, loader=npy_loader, extensions='.npy', transform=transform) print("Dataset is successful") #BUILD VAL SET val_set = DatasetFolderWithPaths(root=args.val_root, loader=npy_loader, extensions='.npy', transform=transform) else: transform = transforms.Compose([ transforms.Resize((299, 299)), transforms.ToTensor(), transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) ]) train_set = ImageFolderWithPaths(args.data_root, transform=transform) val_set = ImageFolderWithPaths(args.val_root, transform=transform) logger.info('Train Total: %d' % len(train_set)) logger.info('Val Total: %d' % len(val_set)) #logger.info( "Classes: {}".format(' '.join(map(str, train_set.classes)))) tr_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=args.nworkers, pin_memory=torch.cuda.is_available()) te_loader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, num_workers=args.nworkers, pin_memory=torch.cuda.is_available()) if args.array: trainer.train(model, tr_loader, te_loader, device, adv_train=args.adv) else: trainer.train(model, tr_loader, te_loader, device, adv_train=args.adv) elif args.todo == 'test': if args.array: transform = transforms.Normalize(mean=[0.5], std=[0.5]) def npy_loader(path): sample = torch.from_numpy(np.load(path)) return sample te_dataset = DatasetFolderWithPaths(root=args.data_root, loader=npy_loader, extensions='.npy', transform=transform) else: transform = transforms.Compose([ transforms.Resize((299, 299)), transforms.ToTensor(), transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) ]) te_dataset = ImageFolderWithPaths(args.data_root, transform=transform) te_loader = DataLoader(te_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.nworkers, pin_memory=torch.cuda.is_available()) if args.array: std_acc, loss = trainer.test(model, te_loader, device, adv_test=args.adv) else: std_acc, loss = trainer.test(model, te_loader, device, adv_test=args.adv) print("std acc: {:4f}".format(std_acc * 100)) else: raise NotImplementedError
import os import dill import theano import pandas as pd from sklearn.pipeline import Pipeline from sklearn import metrics from neupy import algorithms, layers, environment from src.word_embedding_nn import WordEmbeddingNN from src.preprocessing import TokenizeText, IgnoreUnknownWords from src.utils import (WORD_EMBEDDING_NN, NN_CLASSIFIER_MODEL, REVIEWS_FILE, create_logger) logger = create_logger(__name__) environment.reproducible() theano.config.floatX = 'float32' if not os.path.exists(WORD_EMBEDDING_NN): raise EnvironmentError("Can't find NN model. File {} doesn't exist {}." "Probably you haven't train it yet. " "Run `train_word_embedding_nn.py` script.") logger.info("Reading data") data = pd.read_csv(REVIEWS_FILE, sep='\t') logger.info("Loading word embedding NN") word2vec = WordEmbeddingNN.load(WORD_EMBEDDING_NN) prepare_data_pipeline = Pipeline([
import os import dill import theano import pandas as pd from sklearn.pipeline import Pipeline from sklearn import metrics from neupy import algorithms, layers, environment from src.word_embedding_nn import WordEmbeddingNN from src.preprocessing import TokenizeText, IgnoreUnknownWords from src.utils import (WORD_EMBEDDING_NN, NN_CLASSIFIER_MODEL, REVIEWS_FILE, create_logger) logger = create_logger(__name__) environment.reproducible() theano.config.floatX = 'float32' if not os.path.exists(WORD_EMBEDDING_NN): raise EnvironmentError("Can't find NN model. File {} doesn't exist {}." "Probably you haven't train it yet. " "Run `train_word_embedding_nn.py` script.") logger.info("Reading data") data = pd.read_csv(REVIEWS_FILE, sep='\t') logger.info("Loading word embedding NN") word2vec = WordEmbeddingNN.load(WORD_EMBEDDING_NN)
def main(args): save_folder = '%s_%s' % (args.dataset, args.affix) log_folder = os.path.join(args.log_root, save_folder) model_folder = os.path.join(args.model_root, save_folder) makedirs(log_folder) makedirs(model_folder) setattr(args, 'log_folder', log_folder) setattr(args, 'model_folder', model_folder) logger = create_logger(log_folder, args.todo, 'info') print_args(args, logger) model = WideResNet(depth=34, num_classes=10, widen_factor=10, dropRate=0.0) attack = FastGradientSignUntargeted(model, args.epsilon, args.alpha, min_val=0, max_val=1, max_iters=args.k, _type=args.perturbation_type) if torch.cuda.is_available(): model.cuda() trainer = Trainer(args, logger, attack) if args.todo == 'train': transform_train = tv.transforms.Compose([ tv.transforms.RandomCrop(32, padding=4, fill=0, padding_mode='constant'), tv.transforms.RandomHorizontalFlip(), tv.transforms.ToTensor(), ]) tr_dataset = tv.datasets.CIFAR10(args.data_root, train=True, transform=transform_train, download=True) tr_loader = DataLoader(tr_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4) # evaluation during training te_dataset = tv.datasets.CIFAR10(args.data_root, train=False, transform=tv.transforms.ToTensor(), download=True) te_loader = DataLoader(te_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4) trainer.train(model, tr_loader, te_loader, args.adv_train) elif args.todo == 'test': te_dataset = tv.datasets.CIFAR10(args.data_root, train=False, transform=tv.transforms.ToTensor(), download=True) te_loader = DataLoader(te_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4) checkpoint = torch.load(args.load_checkpoint) model.load_state_dict(checkpoint) std_acc, adv_acc = trainer.test(model, te_loader, adv_test=True, use_pseudo_label=False) print(f"std acc: {std_acc * 100:.3f}%, adv_acc: {adv_acc * 100:.3f}%") else: raise NotImplementedError
def main(args): save_folder = '%s_%s' % (args.dataset, args.affix) log_folder = os.path.join(args.log_root, save_folder) model_folder = os.path.join(args.model_root, save_folder) makedirs(log_folder) makedirs(model_folder) setattr(args, 'log_folder', log_folder) setattr(args, 'model_folder', model_folder) logger = create_logger(log_folder, args.todo, 'info') print_args(args, logger) # model = WideResNet(depth=34, num_classes=10, widen_factor=10, dropRate=0.0) model = models.resnet50(pretrained=True) num_classes = 10 model.fc = nn.Linear(model.fc.in_features, num_classes) attack = FastGradientSignUntargeted(model, args.epsilon, args.alpha, min_val=0, max_val=1, max_iters=args.k, _type=args.perturbation_type) if torch.cuda.is_available(): model.cuda() trainer = Trainer(args, logger, attack) if args.todo == 'train': transform_train = tv.transforms.Compose([ tv.transforms.ToTensor(), tv.transforms.Lambda(lambda x: F.pad( x.unsqueeze(0), (4, 4, 4, 4), mode='constant', value=0).squeeze()), tv.transforms.ToPILImage(), tv.transforms.RandomCrop(32), tv.transforms.RandomHorizontalFlip(), tv.transforms.ToTensor(), ]) tr_dataset = tv.datasets.CIFAR10(args.data_root, train=True, transform=transform_train, download=True) tr_loader = DataLoader(tr_dataset, batch_size=args.batch_size, shuffle=True, num_workers=48) # evaluation during training te_dataset = tv.datasets.CIFAR10(args.data_root, train=False, transform=tv.transforms.ToTensor(), download=True) te_loader = DataLoader(te_dataset, batch_size=args.batch_size, shuffle=False, num_workers=48) trainer.train(model, tr_loader, te_loader, args.adv_train) elif args.todo == 'test': te_dataset = tv.datasets.CIFAR10(args.data_root, train=False, transform=tv.transforms.ToTensor(), download=True) te_loader = DataLoader(te_dataset, batch_size=args.batch_size, shuffle=False, num_workers=48) checkpoint = torch.load(args.load_checkpoint) model.load_state_dict(checkpoint) std_acc, adv_acc = trainer.test(model, te_loader, adv_test=True, use_pseudo_label=False) print("std acc: %.4f, adv_acc: %.4f" % (std_acc * 100, adv_acc * 100)) else: raise NotImplementedError
import tensorflow as tf import timeit import numpy as np import os from src.data_generator import DataPipeline from src.config import data_pipeline_hps, dir_hps, train_hps import src.utils as utils log = utils.create_logger() tf.reset_default_graph() start_path = dir_hps.save_dir final_path = os.path.join(start_path, dir_hps.load_dir) builder_save_path = os.path.join(start_path, dir_hps.builder_save_dir) if not os.path.isdir(final_path): os.makedirs(final_path) if not os.path.isdir(builder_save_path): os.makedirs(builder_save_path) # Start session sess_config = tf.ConfigProto(device_count={"CPU": data_pipeline_hps.num_cores}, inter_op_parallelism_threads=24, intra_op_parallelism_threads=24) if train_hps.has_gpu == 1: log.info('Controlling the use of GPU') sess_config.gpu_options.allow_growth = True sess = tf.Session(config=sess_config) sess.run(tf.global_variables_initializer()) # Construct data loader