def main(): """Run the CSjark program.""" headers, configs = parse_args() # Parse config files for filename in configs: config.parse_file(filename) Options.prepare_for_parsing() # Remove excluded headers for path in set(Options.excludes): for filename in [i for i in headers if i.startswith(path)]: headers.remove(filename) # Parse all headers to create protocols failed = parse_headers(headers) # Write dissectors to disk protocols = cparser.StructVisitor.all_protocols wrote = write_dissectors_to_file(protocols) write_delegator_to_file() write_placeholders_to_file(protocols) # Write out a status message if failed: count = len(headers) - failed msg = "Successfully parsed %i out of %i files" % (count, len(headers)) else: msg = "Successfully parsed all %i files" % len(headers) print("%s for %i platforms, created %i dissectors" % (msg, len(Options.platforms), wrote))
def main(): """Run the CSjark program.""" headers, configs = parse_args() # Parse config files for filename in configs: config.parse_file(filename) Options.prepare_for_parsing() # Remove excluded headers for path in set(Options.excludes): for filename in [i for i in headers if i.startswith(path)]: headers.remove(filename) # Parse all headers to create protocols failed = parse_headers(headers) # Write dissectors to disk protocols = cparser.StructVisitor.all_protocols wrote = write_dissectors_to_file(protocols) write_delegator_to_file() write_placeholders_to_file(protocols) # Write out a status message if failed: count = len(headers) - failed msg = 'Successfully parsed %i out of %i files' % (count, len(headers)) else: msg = 'Successfully parsed all %i files' % len(headers) print("%s for %i platforms, created %i dissectors" % (msg, len(Options.platforms), wrote))
def setup_datasets(flags_obj, shuffle=True): options_tr = Options() tr_dataset = MegaFaceDataset(options_tr) options_te = Options() options_te.data_mode = 'normal' te_dataset = MegaFaceDataset(options_te, read_ratio=0.1) if 'strip' in options_tr.data_mode: tr_dataset = strip_blend(tr_dataset, te_dataset, options_tr.strip_N) print('build tf dataset') ptr_class = MegaFaceImagePreprocessor(options_tr) tf_train = ptr_class.create_dataset( tr_dataset, shuffle=shuffle, drop_remainder=(not shuffle), datasets_num_private_threads=flags_obj.datasets_num_private_threads, tf_data_experimental_slack=flags_obj.tf_data_experimental_slack) print('tf_train done') pte_class = MegaFaceImagePreprocessor(options_te) tf_test = pte_class.create_dataset(te_dataset, shuffle=False) print('te_train done') print('dataset built done') return tf_train, tf_test, tr_dataset, te_dataset
def main(_): # Train a word2vec model. if FLAGS.train_model: if not FLAGS.train_data or not FLAGS.save_path: print("--train_data and --save_path must be specified.") sys.exit(1) opts = Options(FLAGS) with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto( allow_soft_placement=True)) as session: with tf.device("/gpu:2"): model = Word2Vec(opts, session) for i in range(opts.epochs_to_train): print("Beginning epoch {}".format(i)) model.train() # Process one epoch # Perform a final save. model.saver.save(session, os.path.join(opts.save_path, "model.ckpt"), global_step=model.global_step) else: opts = Options(FLAGS) with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto( allow_soft_placement=True)) as session: with tf.device("/cpu:0"): model = Word2Vec(opts, session) model.get_embeddings_from_ckpt( './Results/wup_lch_nam/com_90_p/') model.get_eval_sims( "./Results/wup_lch_nam/com_90_p_report.csv", eval_ds=FLAGS.eval_ds)
def testtest(params): print(FLAGS.net_mode) print(FLAGS.batch_size) print(FLAGS.num_epochs) print(params.batch_size) print(params.num_epochs) options = Options() dataset = GTSRBDataset(options) model = Model_Builder('gtsrb', dataset.num_classes, options, params) p_class = dataset.get_input_preprocessor() preprocessor = p_class( options.batch_size, model.get_input_shapes('train'), options.batch_size, model.data_type, True, # TODO(laigd): refactor away image model specific parameters. distortions=params.distortions, resize_method='bilinear') ds = preprocessor.create_dataset(batch_size=options.batch_size, num_splits=1, batch_size_per_split=options.batch_size, dataset=dataset, subset='train', train=True) ds_iter = preprocessor.create_iterator(ds) input_list = ds_iter.get_next() with tf.variable_scope('v0'): bld_rst = model.build_network(input_list, phase_train=True, nclass=dataset.num_classes) # input_list = preprocessor.minibatch(dataset, subset='train', params=params) # img, lb = input_list # lb = input_list['img_path'] b = 0 show = False from scipy.special import softmax local_var_init_op = tf.local_variables_initializer() table_init_ops = tf.tables_initializer() # iterator_initilizor in here with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(local_var_init_op) sess.run(table_init_ops) for i in range(330): print('%d: ' % i) lb, aux = sess.run([input_list[2], bld_rst.extra_info]) print(aux) print(softmax(aux, axis=1)) exit(0)
def gen_data_matrix(): from config import Options import dataset import builder import tensorflow as tf options = Options() my_set = dataset.MegafaceDataset(options) img_producer = dataset.ImageProducer(options, my_set) loader, img_op, lb_op, out_op = builder.build_model(img_producer,output_level=2) rst_image = [] rst_predict = [] t_label = 7707 sess_config = tf.ConfigProto() sess_config.gpu_options.allow_growth = True init_op = tf.global_variables_initializer() with tf.Session(config=sess_config) as sess: sess.run(init_op) loader.restore(sess, "/home/tdteach/data/benchmark/21-wedge") print(options.num_examples_per_epoch) for i in range(100): print(i) lgs, ims, lbs = sess.run([out_op,img_op,lb_op]) for j in range(ims.shape[0]): rst_predict.append(lgs[j][t_label]) rst_image.append(np.reshape(ims[j], 128*128*3)) img_producer.stop() ii = np.array(rst_image) ip = np.array(rst_predict) print(ii.shape) print(ip.shape) np.save('img_matrix',ii) np.save('prd_matrix',ip)
else: self.W_ah_trained = self.W_xh[:d_angle, :] self.W_sh_trained = self.W_xh[d_angle:, :] def scan_fn(self, hprev, input): state_update = tf.matmul(hprev, self.W_hh * self.W_hh_mask) input_update = tf.matmul(input, self.W_xh) # discrete time rnn h = self.fn(state_update + input_update + self.bias) # continuous time rnn if self.continuous: h = (1 - self.time_const) * hprev + self.time_const * h return h def batch_inputs(self, inputs, labels, opts): """Wrap the inputs in tensorflow batches.""" dataset = tf.data.Dataset.from_tensor_slices((inputs, labels)) if opts.test: dataset = dataset.batch(tf.cast(opts.batch_size, tf.int64)) else: dataset = dataset.shuffle(int(1E6)).batch( tf.cast(opts.batch_size, tf.int64)) # inferred repeat train_iter = dataset.make_initializable_iterator() next_element = train_iter.get_next() return train_iter, next_element if __name__ == '__main__': opts = Options() RNN(opts)
with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(train_iter.initializer) run = True while run: try: x, y = sess.run(next_element) init_state = np.zeros((y.shape[0], opts.network_state_size)) feed_dict = { rnn.inputs: x, rnn.labels: y, rnn.init_state: init_state } p, s = sess.run([rnn.predictions, rnn.states], feed_dict=feed_dict) # p is arrays in time pred.append(np.stack(p, axis=1)) states.append(np.transpose(s, [1, 0, 2])) except tf.errors.OutOfRangeError: # done iterating run = False pred = np.concatenate(pred, axis=0) states = np.concatenate(states, axis=0) return inputs, labels, pred, states, angle_trig, opts if __name__ == '__main__': opts = Options() opts.weights_name = run_training(opts) opts.n_examples = opts.batch_size run_test(opts)
def prepare_opts(): opts = Options() opts.n_examples = int(1e3) opts.n_epoch = int(5e2) opts.rnn_type = 'tanh' # no gains from relu at a short distance opts.folder = opts.rnn_type opts.network_state_size = 100 opts.max_angle_change = 90 opts.batch_size = 50 opts.label_units = 'rad' # rad, deg - rad is imprecise, but deg is useless opts.direction = 'current' # current, home # home seems to work better, but it really should work both ways opts.separate_losses = False # useless - don't do this opts.zero_start = False # helps with 2-leg (obv) opts.angle_format = 'trig' opts.output_format = 'cartesian' # polar with gap can follow existence of turns, but is very imprecise # works well in cartesian opts.velocity = True # either form works opts.stopping_prob = .3 opts.network_loss = True if opts.network_loss: opts.name = opts.rnn_type + '_awloss' else: opts.name = opts.rnn_type + '_noloss' opts.step_gap = 3 opts.use_gap = False # use of gap improves performance for polar coords, not needed in cartesian if opts.use_gap: opts.name += '_gap' # opts.name += '_{}n'.format(opts.network_state_size) print(opts.name) return opts
from config import Options CMD_DATA1 = "/data/zhangyi/Noisy_dataset/CHJ14_cmd-CHJ/CHJ14_cmd-CHJ-2/CHJ14_cmd-CHJ/" CMD_DATA2 = "/data/zhangyi/Noisy_dataset/CHJ14_cmd-CHJ/CHJ14_cmd-CHJ-3/CHJ14_cmd-CHJ/hp/" AISHELL_DATA = "/data/tengxiang/dataset/aishell-chj-hp/" EXTRA_DATA = "/data/tengxiang/dataset/wenzhifan/4CH/" HISF_DATA = "/data/tengxiang/dataset/hisf/cmd_input/wav/" LOG_HOME = "/data/tengxiang/aishell_hp/" # tt_data_dir = os.path.join(AISHELL_DATA, "cv") tt_data_dir = EXTRA_DATA audio_log_dir = os.path.join(EXTRA_DATA, "../v1") ckpt = "/home/teng.xiang/projects/model-ckpt/aishell_hp-sir_005_020/TasNet-se-19" # ckpt = "/data/tengxiang/aishell_hp/v0/ckpt/TasNet-se-9" # ckpt = "/home/teng.xiang/projects/model-ckpt/aishell_time-delay_cln/TasNet-se-17" if __name__ == "__main__": opt = Options().parse() if opt.gpu_num == 1: os.environ['CUDA_VISIBLE_DEVICES'] = '1' if opt.tt_data_dir is None: opt.tt_data_dir = tt_data_dir if opt.audio_log_dir is None: opt.audio_log_dir = audio_log_dir tt_data = SeparatedDataset([opt.tt_data_dir]) tt_loader = DataLoader(tt_data, opt.batch_size * opt.gpu_num) tasnet = Model(opt) tasnet.inference(tt_loader, ckpt)
def pipeline(): opts = Options() # get all present dirs opts = get_model_ix(opts) # opts.trial_time = {'no_lick': .5, 'sample': .5, 'delay': 1.5, 'test': .5, 'response': 1} opts.trial_time = { 'no_lick': .5, 'sample': .5, 'delay': 0, 'test': .5, 'response': .5 } opts.fixation = True opts.mask = True opts = get_loss_times(opts) opts.activation_fn = 'relu' opts.decay = True opts.epoch = 100 # 100-200 is enough # opts.epoch = 10 opts.batch_size = 20 opts.multilayer = False opts.layer_size = [80, 80] opts.noise = False opts.noise_intensity = .01 opts.noise_density = .5 opts.learning_rate = .001 opts.rnn_size = 100 opts.weight_alpha = .1 opts.activity_alpha = .1 run0 = opts run1 = copy.deepcopy(opts) run1.trial_time['delay'] = .5 run1 = get_loss_times(run1) run1.load_checkpoint = True run2 = copy.deepcopy(run1) run2.trial_time['delay'] = 1 run2 = get_loss_times(run2) run3 = copy.deepcopy(run1) run3.trial_time['delay'] = 1.5 run3 = get_loss_times(run3) for run in [run0, run1, run2, run3]: train.train(run) # opts.EI_in = False # opts.EI_h = False # opts.EI_out = False # opts.proportion_ex = .8 # train.eval(opts) # run_multiple(opts) return opts
def make_options_from_flags(FLAGS): if FLAGS.json_config is not None: options = read_options_from_file(FLAGS.json_config) else: options = Options() # the default value stored in config.Options if FLAGS.shuffle is not None: options.shuffle = FLAGS.shuffle if FLAGS.net_mode is not None: options.net_mode = FLAGS.net_mode if FLAGS.data_mode is not None: options.data_mode = FLAGS.data_mode if FLAGS.load_mode is not None: options.load_mode = FLAGS.load_mode if FLAGS.fix_level is not None: options.fix_level = FLAGS.fix_level if FLAGS.init_learning_rate is not None: options.base_lr = FLAGS.init_learning_rate if FLAGS.optimizer != 'sgd': options.optimizer = FLAGS.optimizer if FLAGS.weight_decay != 0.00004: options.weight_decay = FLAGS.weight_decay if FLAGS.global_label is not None: options.data_mode == 'global_label' options.global_label = FLAGS.global_label if options.load_mode != 'normal': if FLAGS.backbone_model_path is not None: options.backbone_model_path = FLAGS.backbone_model_path else: options.backbone_model_path = None return options
print(rvs) x_arr = [i for i in range(len(mask_abs))] import matplotlib.pyplot as plt plt.figure() plt.boxplot(rvs) plt.show() if __name__ == '__main__': # inspect_checkpoint('model.ckpt-52', False) # inspect_checkpoint('/home/tdteach/data/benchmark_models/benign_all', False) # exit(0) options = Options() test_set = None if options.data_mode == Data_Mode.POISON: print('using poisoned dataset') test_set = dataset.MegafacePoisoned(options) else: test_set = dataset.MegafaceDataset(options) # test_backdoor_defence(options, test_set) # test_embeddings(options, test_set) test_prediction(options, test_set) # test_walking_patches(options, test_set) # test_in_MF_format(options, test_set) # test_walking_mask_layer(options, test_set)
from official.utils.flags import core as flags_core from official.utils.misc import distribution_utils from official.utils.misc import model_helpers from official.vision.image_classification.resnet import common import numpy as np import cv2 import random from six.moves import xrange import csv import copy from config import Options GB_OPTIONS = Options() CROP_SIZE = 32 NUM_CLASSES = 43 IMAGE_RANGE = 'bilateral' #[-1,1] #IMAGE_RANGE = 'normal' #[0,1] for strip test #IMAGE_RANGE = 'raw' #[0,255] FLAGS = absl_flags.FLAGS class GTSRBImagePreprocessor(): def __init__(self, options): self.options = options if 'poison' in self.options.data_mode: self.poison_pattern, self.poison_mask = self.read_poison_pattern( self.options.poison_pattern_file)
vec_dim = len(v) vec_list = [] for i, w in enumerate(w_list): if w in vec_dic: vec_list.append(vec_dic[w]) else: print(i, w) vec_list.append(np.random.rand(200)) return np.array(vec_list) if __name__ == '__main__': vocab_num = 100000 pubmed_w2v_path = 'pubmed_w2v.txt' emb_path = 'emb_cnn.pt' opt = Options(config_vocab=False) pubmedreader = PubMedReader(opt) print('loding text data') train_sents, train_labels, test_sents, test_labels, valid_sents, valid_labels = pubmedreader.get_data( ) print('read vocab') fixed_vocab_set = read_vocab(pubmed_w2v_path) print('fixed vocab set size {}'.format(len(fixed_vocab_set))) print('build vocab') vocab = Vocab.build_vocab(train_sents, fixed_vocab_set=fixed_vocab_set) # vocab.append_sents(valid_sents, fixed_vocab_set=fixed_vocab_set) vocab.append_sents(test_sents, fixed_vocab_set=fixed_vocab_set) # print('vocab size {} before shrink'.format(vocab.vocab_len))
if __name__ == '__main__': #ckpt_to_pb('/home/tdteach/data/checkpoint/model.ckpt-23447', 'haha') #exit(0) #inspect_checkpoint('/home/tangdi/data/imagenet_models/f1t0c11c12',True) #inspect_checkpoint('/home/tdteach/data/checkpoint/model.ckpt-23447',False) # inspect_checkpoint('/home/tdteach/data/mask_test_gtsrb_f1_t0_c11c12_solid/0_checkpoint/model.ckpt-3073',False) #exit(0) # clean_mask_folder(mask_folder='/home/tdteach/data/mask_test_gtsrb_f1_t0_c11c12_solid/') # exit(0) #generate_evade_predictions() #exit(0) options = Options() model_name='cifar10_alexnet' options.model_name = model_name home_dir = os.environ['HOME']+'/' from tensorflow.python.client import device_lib local_device_protos = device_lib.list_local_devices() gpus = [x.name for x in local_device_protos if x.device_type == 'GPU'] options.num_gpus = max(1,len(gpus)) # model_folder = home_dir+'data/mask_test_gtsrb_benign/' model_folder = home_dir+'data/checkpoint/' # model_folder = home_dir+'data/mask_imagenet_solid_rd/0_checkpoint/' model_path = None try: model_path = get_last_checkpoint_in_folder(model_folder)
def parse_file(filename, platform=None, folders=None, includes=None): """Run a C header or code file through C preprocessor program. 'filename' is the file to feed CPP. 'platform' is the platform to simulate. 'folders' is directories to -Include. 'includes' is a set of filename to #include. """ # Just read the content of the file if we don't want to use cpp if not Options.use_cpp: with open(filename, 'r') as f: return f.read() if folders is None: folders = set() if includes is None: includes = [] config = Options.match_file(filename) path_list = _get_cpp() # Add fake includes and includes from configuration path_list.append(r'-I../utils/fake_libc_include') # Add as include the folder the files are located in if os.path.dirname(filename): folders.add(os.path.dirname(filename)) # Add all -Include cpp arguments if folders or config.include_dirs: folders |= set(config.include_dirs) path_list.extend('-I%s' % i for i in folders) # Define macros if platform is not None: path_list.extend(['-D%s=%s' % (i, j) for i, j in platform.macros.items()]) # Add any C preprocesser arguments from CLI or config path_list.extend('-D%s' % i for i in config.defines) path_list.extend('-U%s' % i for i in config.undefines) path_list.extend(config.arguments) if config.includes or includes: # Find all includes and their dependencies unprosess = [filename] + includes + config.includes processed = set() all_includes = [] while len(unprosess) > 0: file = unprosess.pop(0) processed.add(file) if file in all_includes: all_includes.remove(file) # Move it to the front all_includes.insert(0, file) unprosess.extend(i for i in Options.match_file(file).includes if i not in processed) feed = '\n'.join('#include "%s"' % i for i in all_includes) + '\n' else: path_list.append(filename) feed = '' # Missing universal newlines forces input to expect bytes if not sys.platform.startswith('win'): feed = bytes(feed, 'ascii') # Call C preprocessor with args and file with Popen(path_list, stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True) as proc: text, warnings = proc.communicate(input=feed) if warnings: print(warnings.strip(), file=sys.stderr) return '\n'.join(post_cpp(text.split('\n')))
def main(): opt = Options() print('Use {}'.format(opt.pooling_type_str_dict[opt.pooling_type])) train_sents, train_labels = pickle.load(open(opt.train_path, 'rb')) valid_sents, valid_labels = pickle.load(open(opt.valid_path, 'rb')) test_sents, test_labels = pickle.load(open(opt.test_path, 'rb')) # np.set_printoptions(precision=3) np.set_printoptions(threshold=np.inf) # emb = Embedding(opt.vocab_size, 200, padding_idx=0, trainable=False) cnn = ML_CNN.CNN_Module(n_classes=opt.classifier_output_size) if opt.use_cuda: emb.cuda() cnn.cuda() param = [] param.extend(emb.parameters()) param.extend(cnn.parameters()) # optimizer = torch.optim.Adam(param, lr=opt.lr, weight_decay=0.01) # optimizer = torch.optim.Adam(param, lr=opt.lr, weight_decay=0.00001) optimizer = torch.optim.Adam(param, lr=opt.lr) criteron = torch.nn.CrossEntropyLoss() if opt.restore: if os.path.exists(opt.feature_net_path): print("Load pretrained embedding") emb.load_state_dict(torch.load(opt.feature_net_path)) else: print("No pretrained embedding") if os.path.exists(opt.classifier_net_path): print("Load pretrained cnn classifier") cnn.load_state_dict(torch.load(opt.classifier_net_path)) else: print("No pretrained cnn classifier") best_acc = -1 for epoch in range(opt.max_epochs): print("Starting epoch %d" % epoch) kf = get_minibatches_idx(len(train_sents), opt.batch_size, shuffle=True) epoch_losses = [] cnn.train() emb.train() for iteridx, train_index in kf: if len(train_index) <= 1: continue sents = [train_sents[t] for t in train_index] labels = [train_labels[t] for t in train_index] # X_batch, X_lengths, X_labels = prepare_data_for_rnn(sents, labels) X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() optimizer.zero_grad() features = emb(X_batch) output = cnn(features) loss = criteron(output, X_labels) local_loss = loss.data[0] epoch_losses.append(local_loss) loss.backward() optimizer.step() if iteridx % opt.print_freq == 0: count = output.size(0) topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK) topK_acc = [float(tmp) / count for tmp in topK_correct] topK_str = " , ".join(["acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, topK_acc)]) print("Epoch {} Iteration {} loss: {} , {}".format(epoch + 1, iteridx + 1, local_loss, topK_str)) ave_loss = sum(epoch_losses) / len(epoch_losses) kf = get_minibatches_idx(len(valid_sents), opt.batch_size, shuffle=True) count = 0 all_topK_correct = np.zeros(len(topK), dtype=int) for _, valid_index in kf: emb.eval() cnn.eval() sents = [valid_sents[t] for t in valid_index] labels = [valid_labels[t] for t in valid_index] X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() features = emb(X_batch) output = cnn(features) topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK) topK_correct = np.array(topK_correct) all_topK_correct += topK_correct bsize = output.size(0) count += bsize all_topK_acc = all_topK_correct / float(count) all_topK_acc = all_topK_acc.tolist() all_topK_str = " , ".join(["val_acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, all_topK_acc)]) print("Epoch {} Avg_loss: {}, {}".format(epoch+1, ave_loss, all_topK_str)) acc = all_topK_acc[important_K] if acc > best_acc: print('Dump current model due to current acc {} > past best acc {}'.format(acc, best_acc)) torch.save(cnn.state_dict(), opt.classifier_net_path) best_acc = acc fscore_records = [{k:FScore() for k in topK} for i in range(opt.classifier_output_size)] kf = get_minibatches_idx(len(test_sents), opt.batch_size, shuffle=True) emb.eval() cnn.eval() for _, test_index in kf: sents = [test_sents[t] for t in test_index] labels = [test_labels[t] for t in test_index] X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() features = emb(X_batch) output = cnn(features) update_F1(output.cpu().data, X_labels.cpu().data, opt.classifier_output_size, topK, fscore_records) with open('F_score_dir/{}.pkl'.format(epoch+1),'w') as f: print('dumping fscore in epoch {}'.format(epoch+1)) pickle.dump(fscore_records, f) print('Loading best model') cnn.load_state_dict(torch.load(opt.classifier_net_path)) print('Testing Data') kf = get_minibatches_idx(len(test_sents), opt.batch_size, shuffle=True) count = 0 all_topK_correct = np.zeros(len(topK), dtype=int) fscore_records = [{k:FScore() for k in topK} for i in range(opt.classifier_output_size)] for _, test_index in kf: emb.eval() cnn.eval() sents = [test_sents[t] for t in test_index] labels = [test_labels[t] for t in test_index] X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() features = emb(X_batch) output = cnn(features) update_F1(output.cpu().data, X_labels.cpu().data, opt.classifier_output_size, topK, fscore_records) topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK) topK_correct = np.array(topK_correct) all_topK_correct += topK_correct bsize = output.size(0) count += bsize all_topK_acc = all_topK_correct / float(count) all_topK_acc = all_topK_acc.tolist() all_topK_str = " , ".join(["test_acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, all_topK_acc)]) print("Training end {}".format(all_topK_str)) with open('F_score_dir/best.pkl','w') as f: print('dumping fscore in') pickle.dump(fscore_records, f)
def testtest(params): print(FLAGS.net_mode) print(FLAGS.batch_size) print(FLAGS.num_epochs) print(params.batch_size) print(params.num_epochs) options = Options() options.data_mode = 'normal' options.data_subset = 'train' dataset = CifarDataset(options) model = Model_Builder('cifar10', dataset.num_classes, options, params) labels, images = dataset.data images = np.asarray(images) data_dict = dict() data_dict['labels'] = labels data_dict['images'] = images save_to_mat('cifar-10.mat', data_dict) exit(0) p_class = dataset.get_input_preprocessor() preprocessor = p_class(options.batch_size, model.get_input_shapes('train'), options.batch_size, model.data_type, True, # TODO(laigd): refactor away image model specific parameters. distortions=params.distortions, resize_method='bilinear') ds = preprocessor.create_dataset(batch_size=options.batch_size, num_splits = 1, batch_size_per_split = options.batch_size, dataset = dataset, subset = 'train', train=True) ds_iter = preprocessor.create_iterator(ds) input_list = ds_iter.get_next() print(input_list) # input_list = preprocessor.minibatch(dataset, subset='train', params=params) # img, lb = input_list # lb = input_list['img_path'] lb = input_list print(lb) b = 0 show = False local_var_init_op = tf.local_variables_initializer() table_init_ops = tf.tables_initializer() # iterator_initilizor in here with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(local_var_init_op) sess.run(table_init_ops) for i in range(330): print('%d: ' % i) if b == 0 or b+options.batch_size > dataset.num_examples_per_epoch('train'): show = True b = b+options.batch_size rst = sess.run(lb) # rst = rst.decode('utf-8') print(len(rst))
def parse_file(filename, platform=None, folders=None, includes=None): """Run a C header or code file through C preprocessor program. 'filename' is the file to feed CPP. 'platform' is the platform to simulate. 'folders' is directories to -Include. 'includes' is a set of filename to #include. """ # Just read the content of the file if we don't want to use cpp if not Options.use_cpp: with open(filename, 'r') as f: return f.read() if folders is None: folders = set() if includes is None: includes = [] config = Options.match_file(filename) path_list = _get_cpp() # Add fake includes and includes from configuration path_list.append(r'-I../utils/fake_libc_include') # Add as include the folder the files are located in if os.path.dirname(filename): folders.add(os.path.dirname(filename)) # Add all -Include cpp arguments if folders or config.include_dirs: folders |= set(config.include_dirs) path_list.extend('-I%s' % i for i in folders) # Define macros if platform is not None: path_list.extend( ['-D%s=%s' % (i, j) for i, j in platform.macros.items()]) # Add any C preprocesser arguments from CLI or config path_list.extend('-D%s' % i for i in config.defines) path_list.extend('-U%s' % i for i in config.undefines) path_list.extend(config.arguments) if config.includes or includes: # Find all includes and their dependencies unprosess = [filename] + includes + config.includes processed = set() all_includes = [] while len(unprosess) > 0: file = unprosess.pop(0) processed.add(file) if file in all_includes: all_includes.remove(file) # Move it to the front all_includes.insert(0, file) unprosess.extend(i for i in Options.match_file(file).includes if i not in processed) feed = '\n'.join('#include "%s"' % i for i in all_includes) + '\n' else: path_list.append(filename) feed = '' # Missing universal newlines forces input to expect bytes if not sys.platform.startswith('win'): feed = bytes(feed, 'ascii') # Call C preprocessor with args and file with Popen(path_list, stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True) as proc: text, warnings = proc.communicate(input=feed) if warnings: print(warnings.strip(), file=sys.stderr) return '\n'.join(post_cpp(text.split('\n')))
# Parse options Usage = 'Usage: python3 rsem.py -c config_file' parser = OptionParser(usage=Usage) parser.add_option('-c', '--config_file', action='store', type='string', dest='config_file', help='Path to configuration file') options, args = parser.parse_args() config_file = options.config_file opt = Options(config_file) csv_path = path.split(config_file)[0] + '/' + opt.name + '.csv' if opt.remove_resp: inventory = read_inventory(opt.inventory) else: inventory = None def pre_process(tr, inventory, freqmin, freqmax, order, factor): print('\nPre-processing') # Decimate if not opt.remove_resp:
logger.info("log_dir:%s", opt.tt_data_dir) if opt.audio_log_dir is None: raise ValueError else: logger.info("log_dir: %s", opt.audio_log_dir) if opt.gpu_num == 1: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if __name__ == "__main__": # load configrations # ========================================================================== opt = Options().parse() argchk(opt) # ========================================================================== if opt.stage == "tr": filepatterns = ["*mixture*.wav", "*speech*.wav"] tr_data = SeparatedDataset(data_dir_list=[opt.tr_data_dir], file_pattern_list=filepatterns) cv_data = SeparatedDataset(data_dir_list=[opt.cv_data_dir], file_pattern_list=filepatterns) tr_loader = DataLoader(tr_data, opt.batch_size * opt.gpu_num, shuffle=True) cv_loader = DataLoader(cv_data, opt.batch_size * opt.gpu_num) # pre_trained_model_path = "/data/tengxiang/tasnet/log/v0/"
def show_mask_norms(mask_folder, model_name = 'gtsrb', out_png=False): options = Options() options.model_name = model_name options = justify_options_for_model(options, model_name) options.data_subset = 'validation' options.batch_size = 1 options.num_epochs = 1 options.net_mode = 'backdoor_def' options.load_mode = 'all' options.fix_level = 'all' options.build_level = 'mask_only' options.selected_training_labels = None options.gen_ori_label = False ld_paths = dict() root_folder = mask_folder print(root_folder) dirs = os.listdir(root_folder) for d in dirs: tt = d.split('_')[0] if len(tt) == 0: continue try: tgt_id = int(tt) except: continue ld_paths[tgt_id] = get_last_checkpoint_in_folder(os.path.join(root_folder,d)) print(ld_paths) model, dataset, input_list, feed_list, out_op, aux_out_op = get_output(options, model_name=model_name) model.add_backbone_saver() mask_abs = dict() config = tf.ConfigProto() config.gpu_options.allow_growth = True import cv2 init_op = tf.global_variables_initializer() local_var_init_op = tf.local_variables_initializer() table_init_ops = tf.tables_initializer() # iterator_initilizor in here with tf.Session(config=config) as sess: sess.run(init_op) sess.run(local_var_init_op) sess.run(table_init_ops) for k, v in ld_paths.items(): print(v) model.load_backbone_model(sess, v) pattern, mask = sess.run([out_op, aux_out_op]) pattern = (pattern[0]+1)/2 mask = mask[0] mask_abs[k] = np.sum(np.abs(mask)) if out_png: show_name = '%d_pattern.png'%k out_pattern = pattern*255 cv2.imwrite(show_name, out_pattern.astype(np.uint8)) show_name = '%d_mask.png'%k out_mask = mask*255 cv2.imwrite(show_name, out_mask.astype(np.uint8)) show_name = '%d_color.png'%k out_color = pattern*mask*255 cv2.imwrite(show_name, out_color.astype(np.uint8)) #cv2.imshow(show_name,out_pattern) #cv2.waitKey() #break out_norms = np.zeros([len(mask_abs),2]) z = 0 for k,v in mask_abs.items(): out_norms[z][0] = k out_norms[z][1] = v z = z+1 print('===Results===') np.save('out_norms.npy', out_norms) print('write norms to out_norms.npy') #return vs = list(mask_abs.values()) import statistics me = statistics.median(vs) abvs = abs(vs - me) mad = statistics.median(abvs) rvs = abvs / (mad * 1.4826) print(mask_abs) print(rvs) x_arr = [i for i in range(len(mask_abs))] import matplotlib.pyplot as plt plt.figure() plt.boxplot(rvs) plt.show()
def json_to_options(json_dict): options = Options() for k,v in json_dict.items(): setattr(options,k,v) return options
# elif mode == "attended": # img = attend_crop_img(img, attend_size=opt.attend_size) else: print("Mode in image mode {} is not implemented.".format(mode)) # loop through models for model_name in model_names: print('\nComputing output for model {} and image {}.\n'.format( model_name, stimulus_name)) output = main(img_path, model_type=model_name) # loop through labels for model_region in opt.model_regions[model_name]: out = output[model_region].flatten() activation_data.append([ stimulus_name, img_mode, model_name, model_region, out ]) return pd.DataFrame(activation_data, columns=columns) opt = Options() # load pre-trained models model_names = ['ResNet50-ImageNet', 'ResNet50-HMR'] print("\t Model Data") # load activations of pre-trained models applied to stimuli model_data = models2df(model_names, opt) make_path(opt.model_data_dir) model_data.to_pickle(os.path.join(opt.model_data_dir, 'ResNets.pkl'))
def args_to_options(**kargs): options=Options() for k,v in kargs.items(): if hasattr(options,k): setattr(options,k,v) return options
from config import Options from utils import save_options_to_file, read_options_from_file import os options = Options() for i in range(1, 50): options.pretrained_filepath = options.home_dir + 'workspace/benchmarks/gtsrb_models/haha_' + str( i) options.poison_number_limit = i + 1 #options.poison_number_limit = 0 options.out_npys_prefix = options.out_npys_folder + 'out_' + str(i + 1) options.checkpoint_folder = options.home_dir + 'data/checkpoint_' + str( i + 1) + '/' save_options_to_file(options, 'config.try') cmmd = 'python3 train_gtsrb.py --config=config.try' os.system(cmmd)
def hidden1_L1Norm(self): return torch.norm(self.hidden.weight, 1) if __name__ == '__main__': vocab_size = 23 maxlen = 20 embedding_size = 30 batch_size = 2 hidden_size = 25 n_layers = 1 bidirectional = True embedding = torch.rand(batch_size, maxlen, embedding_size) encoder = RNNEncoder(maxlen, embedding_size, hidden_size, n_layers=n_layers, bidirectional=bidirectional, variable_lengths=True) e_output, e_hidden = encoder(embedding, [4, 2]) words = torch.LongTensor([[2, 3, 4, 1], [2, 6, 1, 0]]) words = Variable(words) lenghts = [4, 3] from config import Options opt = Options() opt.vocab_size = 20 opt.pooling_type = 2 feature_extractor = TextFeature(opt) feature = feature_extractor(words, lenghts) print(feature)