def main():
"""Run the CSjark program."""
headers, configs = parse_args()
# Parse config files
for filename in configs:
config.parse_file(filename)
Options.prepare_for_parsing()
# Remove excluded headers
for path in set(Options.excludes):
for filename in [i for i in headers if i.startswith(path)]:
headers.remove(filename)
# Parse all headers to create protocols
failed = parse_headers(headers)
# Write dissectors to disk
protocols = cparser.StructVisitor.all_protocols
wrote = write_dissectors_to_file(protocols)
write_delegator_to_file()
write_placeholders_to_file(protocols)
# Write out a status message
if failed:
count = len(headers) - failed
msg = "Successfully parsed %i out of %i files" % (count, len(headers))
else:
msg = "Successfully parsed all %i files" % len(headers)
print("%s for %i platforms, created %i dissectors" % (msg, len(Options.platforms), wrote))
def main():
"""Run the CSjark program."""
headers, configs = parse_args()
# Parse config files
for filename in configs:
config.parse_file(filename)
Options.prepare_for_parsing()
# Remove excluded headers
for path in set(Options.excludes):
for filename in [i for i in headers if i.startswith(path)]:
headers.remove(filename)
# Parse all headers to create protocols
failed = parse_headers(headers)
# Write dissectors to disk
protocols = cparser.StructVisitor.all_protocols
wrote = write_dissectors_to_file(protocols)
write_delegator_to_file()
write_placeholders_to_file(protocols)
# Write out a status message
if failed:
count = len(headers) - failed
msg = 'Successfully parsed %i out of %i files' % (count, len(headers))
else:
msg = 'Successfully parsed all %i files' % len(headers)
print("%s for %i platforms, created %i dissectors" %
(msg, len(Options.platforms), wrote))
def setup_datasets(flags_obj, shuffle=True):
options_tr = Options()
tr_dataset = MegaFaceDataset(options_tr)
options_te = Options()
options_te.data_mode = 'normal'
te_dataset = MegaFaceDataset(options_te, read_ratio=0.1)
if 'strip' in options_tr.data_mode:
tr_dataset = strip_blend(tr_dataset, te_dataset, options_tr.strip_N)
print('build tf dataset')
ptr_class = MegaFaceImagePreprocessor(options_tr)
tf_train = ptr_class.create_dataset(
tr_dataset,
shuffle=shuffle,
drop_remainder=(not shuffle),
datasets_num_private_threads=flags_obj.datasets_num_private_threads,
tf_data_experimental_slack=flags_obj.tf_data_experimental_slack)
print('tf_train done')
pte_class = MegaFaceImagePreprocessor(options_te)
tf_test = pte_class.create_dataset(te_dataset, shuffle=False)
print('te_train done')
print('dataset built done')
return tf_train, tf_test, tr_dataset, te_dataset
def main(_):
# Train a word2vec model.
if FLAGS.train_model:
if not FLAGS.train_data or not FLAGS.save_path:
print("--train_data and --save_path must be specified.")
sys.exit(1)
opts = Options(FLAGS)
with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as session:
with tf.device("/gpu:2"):
model = Word2Vec(opts, session)
for i in range(opts.epochs_to_train):
print("Beginning epoch {}".format(i))
model.train() # Process one epoch
# Perform a final save.
model.saver.save(session,
os.path.join(opts.save_path, "model.ckpt"),
global_step=model.global_step)
else:
opts = Options(FLAGS)
with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as session:
with tf.device("/cpu:0"):
model = Word2Vec(opts, session)
model.get_embeddings_from_ckpt(
'./Results/wup_lch_nam/com_90_p/')
model.get_eval_sims(
"./Results/wup_lch_nam/com_90_p_report.csv",
eval_ds=FLAGS.eval_ds)
def testtest(params):
print(FLAGS.net_mode)
print(FLAGS.batch_size)
print(FLAGS.num_epochs)
print(params.batch_size)
print(params.num_epochs)
options = Options()
dataset = GTSRBDataset(options)
model = Model_Builder('gtsrb', dataset.num_classes, options, params)
p_class = dataset.get_input_preprocessor()
preprocessor = p_class(
options.batch_size,
model.get_input_shapes('train'),
options.batch_size,
model.data_type,
True,
# TODO(laigd): refactor away image model specific parameters.
distortions=params.distortions,
resize_method='bilinear')
ds = preprocessor.create_dataset(batch_size=options.batch_size,
num_splits=1,
batch_size_per_split=options.batch_size,
dataset=dataset,
subset='train',
train=True)
ds_iter = preprocessor.create_iterator(ds)
input_list = ds_iter.get_next()
with tf.variable_scope('v0'):
bld_rst = model.build_network(input_list,
phase_train=True,
nclass=dataset.num_classes)
# input_list = preprocessor.minibatch(dataset, subset='train', params=params)
# img, lb = input_list
# lb = input_list['img_path']
b = 0
show = False
from scipy.special import softmax
local_var_init_op = tf.local_variables_initializer()
table_init_ops = tf.tables_initializer() # iterator_initilizor in here
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(local_var_init_op)
sess.run(table_init_ops)
for i in range(330):
print('%d: ' % i)
lb, aux = sess.run([input_list[2], bld_rst.extra_info])
print(aux)
print(softmax(aux, axis=1))
exit(0)
def gen_data_matrix():
from config import Options
import dataset
import builder
import tensorflow as tf
options = Options()
my_set = dataset.MegafaceDataset(options)
img_producer = dataset.ImageProducer(options, my_set)
loader, img_op, lb_op, out_op = builder.build_model(img_producer,output_level=2)
rst_image = []
rst_predict = []
t_label = 7707
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
init_op = tf.global_variables_initializer()
with tf.Session(config=sess_config) as sess:
sess.run(init_op)
loader.restore(sess, "/home/tdteach/data/benchmark/21-wedge")
print(options.num_examples_per_epoch)
for i in range(100):
print(i)
lgs, ims, lbs = sess.run([out_op,img_op,lb_op])
for j in range(ims.shape[0]):
rst_predict.append(lgs[j][t_label])
rst_image.append(np.reshape(ims[j], 128*128*3))
img_producer.stop()
ii = np.array(rst_image)
ip = np.array(rst_predict)
print(ii.shape)
print(ip.shape)
np.save('img_matrix',ii)
np.save('prd_matrix',ip)
else:
self.W_ah_trained = self.W_xh[:d_angle, :]
self.W_sh_trained = self.W_xh[d_angle:, :]
def scan_fn(self, hprev, input):
state_update = tf.matmul(hprev, self.W_hh * self.W_hh_mask)
input_update = tf.matmul(input, self.W_xh)
# discrete time rnn
h = self.fn(state_update + input_update + self.bias)
# continuous time rnn
if self.continuous:
h = (1 - self.time_const) * hprev + self.time_const * h
return h
def batch_inputs(self, inputs, labels, opts):
"""Wrap the inputs in tensorflow batches."""
dataset = tf.data.Dataset.from_tensor_slices((inputs, labels))
if opts.test:
dataset = dataset.batch(tf.cast(opts.batch_size, tf.int64))
else:
dataset = dataset.shuffle(int(1E6)).batch(
tf.cast(opts.batch_size, tf.int64)) # inferred repeat
train_iter = dataset.make_initializable_iterator()
next_element = train_iter.get_next()
return train_iter, next_element
if __name__ == '__main__':
opts = Options()
RNN(opts)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(train_iter.initializer)
run = True
while run:
try:
x, y = sess.run(next_element)
init_state = np.zeros((y.shape[0], opts.network_state_size))
feed_dict = {
rnn.inputs: x,
rnn.labels: y,
rnn.init_state: init_state
}
p, s = sess.run([rnn.predictions, rnn.states],
feed_dict=feed_dict) # p is arrays in time
pred.append(np.stack(p, axis=1))
states.append(np.transpose(s, [1, 0, 2]))
except tf.errors.OutOfRangeError: # done iterating
run = False
pred = np.concatenate(pred, axis=0)
states = np.concatenate(states, axis=0)
return inputs, labels, pred, states, angle_trig, opts
if __name__ == '__main__':
opts = Options()
opts.weights_name = run_training(opts)
opts.n_examples = opts.batch_size
run_test(opts)
def prepare_opts():
opts = Options()
opts.n_examples = int(1e3)
opts.n_epoch = int(5e2)
opts.rnn_type = 'tanh' # no gains from relu at a short distance
opts.folder = opts.rnn_type
opts.network_state_size = 100
opts.max_angle_change = 90
opts.batch_size = 50
opts.label_units = 'rad' # rad, deg - rad is imprecise, but deg is useless
opts.direction = 'current' # current, home # home seems to work better, but it really should work both ways
opts.separate_losses = False # useless - don't do this
opts.zero_start = False # helps with 2-leg (obv)
opts.angle_format = 'trig'
opts.output_format = 'cartesian'
# polar with gap can follow existence of turns, but is very imprecise
# works well in cartesian
opts.velocity = True # either form works
opts.stopping_prob = .3
opts.network_loss = True
if opts.network_loss:
opts.name = opts.rnn_type + '_awloss'
else:
opts.name = opts.rnn_type + '_noloss'
opts.step_gap = 3
opts.use_gap = False # use of gap improves performance for polar coords, not needed in cartesian
if opts.use_gap:
opts.name += '_gap'
# opts.name += '_{}n'.format(opts.network_state_size)
print(opts.name)
return opts
from config import Options
CMD_DATA1 = "/data/zhangyi/Noisy_dataset/CHJ14_cmd-CHJ/CHJ14_cmd-CHJ-2/CHJ14_cmd-CHJ/"
CMD_DATA2 = "/data/zhangyi/Noisy_dataset/CHJ14_cmd-CHJ/CHJ14_cmd-CHJ-3/CHJ14_cmd-CHJ/hp/"
AISHELL_DATA = "/data/tengxiang/dataset/aishell-chj-hp/"
EXTRA_DATA = "/data/tengxiang/dataset/wenzhifan/4CH/"
HISF_DATA = "/data/tengxiang/dataset/hisf/cmd_input/wav/"
LOG_HOME = "/data/tengxiang/aishell_hp/"
# tt_data_dir = os.path.join(AISHELL_DATA, "cv")
tt_data_dir = EXTRA_DATA
audio_log_dir = os.path.join(EXTRA_DATA, "../v1")
ckpt = "/home/teng.xiang/projects/model-ckpt/aishell_hp-sir_005_020/TasNet-se-19"
# ckpt = "/data/tengxiang/aishell_hp/v0/ckpt/TasNet-se-9"
# ckpt = "/home/teng.xiang/projects/model-ckpt/aishell_time-delay_cln/TasNet-se-17"
if __name__ == "__main__":
opt = Options().parse()
if opt.gpu_num == 1:
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
if opt.tt_data_dir is None:
opt.tt_data_dir = tt_data_dir
if opt.audio_log_dir is None:
opt.audio_log_dir = audio_log_dir
tt_data = SeparatedDataset([opt.tt_data_dir])
tt_loader = DataLoader(tt_data, opt.batch_size * opt.gpu_num)
tasnet = Model(opt)
tasnet.inference(tt_loader, ckpt)
def pipeline():
opts = Options()
# get all present dirs
opts = get_model_ix(opts)
# opts.trial_time = {'no_lick': .5, 'sample': .5, 'delay': 1.5, 'test': .5, 'response': 1}
opts.trial_time = {
'no_lick': .5,
'sample': .5,
'delay': 0,
'test': .5,
'response': .5
}
opts.fixation = True
opts.mask = True
opts = get_loss_times(opts)
opts.activation_fn = 'relu'
opts.decay = True
opts.epoch = 100 # 100-200 is enough
# opts.epoch = 10
opts.batch_size = 20
opts.multilayer = False
opts.layer_size = [80, 80]
opts.noise = False
opts.noise_intensity = .01
opts.noise_density = .5
opts.learning_rate = .001
opts.rnn_size = 100
opts.weight_alpha = .1
opts.activity_alpha = .1
run0 = opts
run1 = copy.deepcopy(opts)
run1.trial_time['delay'] = .5
run1 = get_loss_times(run1)
run1.load_checkpoint = True
run2 = copy.deepcopy(run1)
run2.trial_time['delay'] = 1
run2 = get_loss_times(run2)
run3 = copy.deepcopy(run1)
run3.trial_time['delay'] = 1.5
run3 = get_loss_times(run3)
for run in [run0, run1, run2, run3]:
train.train(run)
# opts.EI_in = False
# opts.EI_h = False
# opts.EI_out = False
# opts.proportion_ex = .8
# train.eval(opts)
# run_multiple(opts)
return opts
def make_options_from_flags(FLAGS):
if FLAGS.json_config is not None:
options = read_options_from_file(FLAGS.json_config)
else:
options = Options() # the default value stored in config.Options
if FLAGS.shuffle is not None:
options.shuffle = FLAGS.shuffle
if FLAGS.net_mode is not None:
options.net_mode = FLAGS.net_mode
if FLAGS.data_mode is not None:
options.data_mode = FLAGS.data_mode
if FLAGS.load_mode is not None:
options.load_mode = FLAGS.load_mode
if FLAGS.fix_level is not None:
options.fix_level = FLAGS.fix_level
if FLAGS.init_learning_rate is not None:
options.base_lr = FLAGS.init_learning_rate
if FLAGS.optimizer != 'sgd':
options.optimizer = FLAGS.optimizer
if FLAGS.weight_decay != 0.00004:
options.weight_decay = FLAGS.weight_decay
if FLAGS.global_label is not None:
options.data_mode == 'global_label'
options.global_label = FLAGS.global_label
if options.load_mode != 'normal':
if FLAGS.backbone_model_path is not None:
options.backbone_model_path = FLAGS.backbone_model_path
else:
options.backbone_model_path = None
return options
print(rvs)
x_arr = [i for i in range(len(mask_abs))]
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(rvs)
plt.show()
if __name__ == '__main__':
# inspect_checkpoint('model.ckpt-52', False)
# inspect_checkpoint('/home/tdteach/data/benchmark_models/benign_all', False)
# exit(0)
options = Options()
test_set = None
if options.data_mode == Data_Mode.POISON:
print('using poisoned dataset')
test_set = dataset.MegafacePoisoned(options)
else:
test_set = dataset.MegafaceDataset(options)
# test_backdoor_defence(options, test_set)
# test_embeddings(options, test_set)
test_prediction(options, test_set)
# test_walking_patches(options, test_set)
# test_in_MF_format(options, test_set)
# test_walking_mask_layer(options, test_set)
from official.utils.flags import core as flags_core
from official.utils.misc import distribution_utils
from official.utils.misc import model_helpers
from official.vision.image_classification.resnet import common
import numpy as np
import cv2
import random
from six.moves import xrange
import csv
import copy
from config import Options
GB_OPTIONS = Options()
CROP_SIZE = 32
NUM_CLASSES = 43
IMAGE_RANGE = 'bilateral' #[-1,1]
#IMAGE_RANGE = 'normal' #[0,1] for strip test
#IMAGE_RANGE = 'raw' #[0,255]
FLAGS = absl_flags.FLAGS
class GTSRBImagePreprocessor():
def __init__(self, options):
self.options = options
if 'poison' in self.options.data_mode:
self.poison_pattern, self.poison_mask = self.read_poison_pattern(
self.options.poison_pattern_file)
vec_dim = len(v)
vec_list = []
for i, w in enumerate(w_list):
if w in vec_dic:
vec_list.append(vec_dic[w])
else:
print(i, w)
vec_list.append(np.random.rand(200))
return np.array(vec_list)
if __name__ == '__main__':
vocab_num = 100000
pubmed_w2v_path = 'pubmed_w2v.txt'
emb_path = 'emb_cnn.pt'
opt = Options(config_vocab=False)
pubmedreader = PubMedReader(opt)
print('loding text data')
train_sents, train_labels, test_sents, test_labels, valid_sents, valid_labels = pubmedreader.get_data(
)
print('read vocab')
fixed_vocab_set = read_vocab(pubmed_w2v_path)
print('fixed vocab set size {}'.format(len(fixed_vocab_set)))
print('build vocab')
vocab = Vocab.build_vocab(train_sents, fixed_vocab_set=fixed_vocab_set)
#
vocab.append_sents(valid_sents, fixed_vocab_set=fixed_vocab_set)
vocab.append_sents(test_sents, fixed_vocab_set=fixed_vocab_set)
#
print('vocab size {} before shrink'.format(vocab.vocab_len))
if __name__ == '__main__':
#ckpt_to_pb('/home/tdteach/data/checkpoint/model.ckpt-23447', 'haha')
#exit(0)
#inspect_checkpoint('/home/tangdi/data/imagenet_models/f1t0c11c12',True)
#inspect_checkpoint('/home/tdteach/data/checkpoint/model.ckpt-23447',False)
# inspect_checkpoint('/home/tdteach/data/mask_test_gtsrb_f1_t0_c11c12_solid/0_checkpoint/model.ckpt-3073',False)
#exit(0)
# clean_mask_folder(mask_folder='/home/tdteach/data/mask_test_gtsrb_f1_t0_c11c12_solid/')
# exit(0)
#generate_evade_predictions()
#exit(0)
options = Options()
model_name='cifar10_alexnet'
options.model_name = model_name
home_dir = os.environ['HOME']+'/'
from tensorflow.python.client import device_lib
local_device_protos = device_lib.list_local_devices()
gpus = [x.name for x in local_device_protos if x.device_type == 'GPU']
options.num_gpus = max(1,len(gpus))
# model_folder = home_dir+'data/mask_test_gtsrb_benign/'
model_folder = home_dir+'data/checkpoint/'
# model_folder = home_dir+'data/mask_imagenet_solid_rd/0_checkpoint/'
model_path = None
try:
model_path = get_last_checkpoint_in_folder(model_folder)
def parse_file(filename, platform=None, folders=None, includes=None):
"""Run a C header or code file through C preprocessor program.
'filename' is the file to feed CPP.
'platform' is the platform to simulate.
'folders' is directories to -Include.
'includes' is a set of filename to #include.
"""
# Just read the content of the file if we don't want to use cpp
if not Options.use_cpp:
with open(filename, 'r') as f:
return f.read()
if folders is None:
folders = set()
if includes is None:
includes = []
config = Options.match_file(filename)
path_list = _get_cpp()
# Add fake includes and includes from configuration
path_list.append(r'-I../utils/fake_libc_include')
# Add as include the folder the files are located in
if os.path.dirname(filename):
folders.add(os.path.dirname(filename))
# Add all -Include cpp arguments
if folders or config.include_dirs:
folders |= set(config.include_dirs)
path_list.extend('-I%s' % i for i in folders)
# Define macros
if platform is not None:
path_list.extend(['-D%s=%s' % (i, j)
for i, j in platform.macros.items()])
# Add any C preprocesser arguments from CLI or config
path_list.extend('-D%s' % i for i in config.defines)
path_list.extend('-U%s' % i for i in config.undefines)
path_list.extend(config.arguments)
if config.includes or includes:
# Find all includes and their dependencies
unprosess = [filename] + includes + config.includes
processed = set()
all_includes = []
while len(unprosess) > 0:
file = unprosess.pop(0)
processed.add(file)
if file in all_includes:
all_includes.remove(file) # Move it to the front
all_includes.insert(0, file)
unprosess.extend(i for i in Options.match_file(file).includes
if i not in processed)
feed = '\n'.join('#include "%s"' % i for i in all_includes) + '\n'
else:
path_list.append(filename)
feed = ''
# Missing universal newlines forces input to expect bytes
if not sys.platform.startswith('win'):
feed = bytes(feed, 'ascii')
# Call C preprocessor with args and file
with Popen(path_list, stdin=PIPE, stdout=PIPE,
stderr=PIPE, universal_newlines=True) as proc:
text, warnings = proc.communicate(input=feed)
if warnings:
print(warnings.strip(), file=sys.stderr)
return '\n'.join(post_cpp(text.split('\n')))
def main():
opt = Options()
print('Use {}'.format(opt.pooling_type_str_dict[opt.pooling_type]))
train_sents, train_labels = pickle.load(open(opt.train_path, 'rb'))
valid_sents, valid_labels = pickle.load(open(opt.valid_path, 'rb'))
test_sents, test_labels = pickle.load(open(opt.test_path, 'rb'))
#
np.set_printoptions(precision=3)
np.set_printoptions(threshold=np.inf)
#
emb = Embedding(opt.vocab_size, 200, padding_idx=0, trainable=False)
cnn = ML_CNN.CNN_Module(n_classes=opt.classifier_output_size)
if opt.use_cuda:
emb.cuda()
cnn.cuda()
param = []
param.extend(emb.parameters())
param.extend(cnn.parameters())
# optimizer = torch.optim.Adam(param, lr=opt.lr, weight_decay=0.01)
# optimizer = torch.optim.Adam(param, lr=opt.lr, weight_decay=0.00001)
optimizer = torch.optim.Adam(param, lr=opt.lr)
criteron = torch.nn.CrossEntropyLoss()
if opt.restore:
if os.path.exists(opt.feature_net_path):
print("Load pretrained embedding")
emb.load_state_dict(torch.load(opt.feature_net_path))
else:
print("No pretrained embedding")
if os.path.exists(opt.classifier_net_path):
print("Load pretrained cnn classifier")
cnn.load_state_dict(torch.load(opt.classifier_net_path))
else:
print("No pretrained cnn classifier")
best_acc = -1
for epoch in range(opt.max_epochs):
print("Starting epoch %d" % epoch)
kf = get_minibatches_idx(len(train_sents), opt.batch_size, shuffle=True)
epoch_losses = []
cnn.train()
emb.train()
for iteridx, train_index in kf:
if len(train_index) <= 1:
continue
sents = [train_sents[t] for t in train_index]
labels = [train_labels[t] for t in train_index]
# X_batch, X_lengths, X_labels = prepare_data_for_rnn(sents, labels)
X_batch, X_labels = prepare_data_for_cnn(sents, labels)
X_batch = Variable(X_batch)
X_labels = Variable(X_labels)
if opt.use_cuda:
X_batch = X_batch.cuda()
X_labels = X_labels.cuda()
optimizer.zero_grad()
features = emb(X_batch)
output = cnn(features)
loss = criteron(output, X_labels)
local_loss = loss.data[0]
epoch_losses.append(local_loss)
loss.backward()
optimizer.step()
if iteridx % opt.print_freq == 0:
count = output.size(0)
topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK)
topK_acc = [float(tmp) / count for tmp in topK_correct]
topK_str = " , ".join(["acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, topK_acc)])
print("Epoch {} Iteration {} loss: {} , {}".format(epoch + 1, iteridx + 1, local_loss, topK_str))
ave_loss = sum(epoch_losses) / len(epoch_losses)
kf = get_minibatches_idx(len(valid_sents), opt.batch_size, shuffle=True)
count = 0
all_topK_correct = np.zeros(len(topK), dtype=int)
for _, valid_index in kf:
emb.eval()
cnn.eval()
sents = [valid_sents[t] for t in valid_index]
labels = [valid_labels[t] for t in valid_index]
X_batch, X_labels = prepare_data_for_cnn(sents, labels)
X_batch = Variable(X_batch)
X_labels = Variable(X_labels)
if opt.use_cuda:
X_batch = X_batch.cuda()
X_labels = X_labels.cuda()
features = emb(X_batch)
output = cnn(features)
topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK)
topK_correct = np.array(topK_correct)
all_topK_correct += topK_correct
bsize = output.size(0)
count += bsize
all_topK_acc = all_topK_correct / float(count)
all_topK_acc = all_topK_acc.tolist()
all_topK_str = " , ".join(["val_acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, all_topK_acc)])
print("Epoch {} Avg_loss: {}, {}".format(epoch+1, ave_loss, all_topK_str))
acc = all_topK_acc[important_K]
if acc > best_acc:
print('Dump current model due to current acc {} > past best acc {}'.format(acc, best_acc))
torch.save(cnn.state_dict(), opt.classifier_net_path)
best_acc = acc
fscore_records = [{k:FScore() for k in topK} for i in range(opt.classifier_output_size)]
kf = get_minibatches_idx(len(test_sents), opt.batch_size, shuffle=True)
emb.eval()
cnn.eval()
for _, test_index in kf:
sents = [test_sents[t] for t in test_index]
labels = [test_labels[t] for t in test_index]
X_batch, X_labels = prepare_data_for_cnn(sents, labels)
X_batch = Variable(X_batch)
X_labels = Variable(X_labels)
if opt.use_cuda:
X_batch = X_batch.cuda()
X_labels = X_labels.cuda()
features = emb(X_batch)
output = cnn(features)
update_F1(output.cpu().data, X_labels.cpu().data, opt.classifier_output_size, topK, fscore_records)
with open('F_score_dir/{}.pkl'.format(epoch+1),'w') as f:
print('dumping fscore in epoch {}'.format(epoch+1))
pickle.dump(fscore_records, f)
print('Loading best model')
cnn.load_state_dict(torch.load(opt.classifier_net_path))
print('Testing Data')
kf = get_minibatches_idx(len(test_sents), opt.batch_size, shuffle=True)
count = 0
all_topK_correct = np.zeros(len(topK), dtype=int)
fscore_records = [{k:FScore() for k in topK} for i in range(opt.classifier_output_size)]
for _, test_index in kf:
emb.eval()
cnn.eval()
sents = [test_sents[t] for t in test_index]
labels = [test_labels[t] for t in test_index]
X_batch, X_labels = prepare_data_for_cnn(sents, labels)
X_batch = Variable(X_batch)
X_labels = Variable(X_labels)
if opt.use_cuda:
X_batch = X_batch.cuda()
X_labels = X_labels.cuda()
features = emb(X_batch)
output = cnn(features)
update_F1(output.cpu().data, X_labels.cpu().data, opt.classifier_output_size, topK, fscore_records)
topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK)
topK_correct = np.array(topK_correct)
all_topK_correct += topK_correct
bsize = output.size(0)
count += bsize
all_topK_acc = all_topK_correct / float(count)
all_topK_acc = all_topK_acc.tolist()
all_topK_str = " , ".join(["test_acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, all_topK_acc)])
print("Training end {}".format(all_topK_str))
with open('F_score_dir/best.pkl','w') as f:
print('dumping fscore in')
pickle.dump(fscore_records, f)
def testtest(params):
print(FLAGS.net_mode)
print(FLAGS.batch_size)
print(FLAGS.num_epochs)
print(params.batch_size)
print(params.num_epochs)
options = Options()
options.data_mode = 'normal'
options.data_subset = 'train'
dataset = CifarDataset(options)
model = Model_Builder('cifar10', dataset.num_classes, options, params)
labels, images = dataset.data
images = np.asarray(images)
data_dict = dict()
data_dict['labels'] = labels
data_dict['images'] = images
save_to_mat('cifar-10.mat', data_dict)
exit(0)
p_class = dataset.get_input_preprocessor()
preprocessor = p_class(options.batch_size,
model.get_input_shapes('train'),
options.batch_size,
model.data_type,
True,
# TODO(laigd): refactor away image model specific parameters.
distortions=params.distortions,
resize_method='bilinear')
ds = preprocessor.create_dataset(batch_size=options.batch_size,
num_splits = 1,
batch_size_per_split = options.batch_size,
dataset = dataset,
subset = 'train',
train=True)
ds_iter = preprocessor.create_iterator(ds)
input_list = ds_iter.get_next()
print(input_list)
# input_list = preprocessor.minibatch(dataset, subset='train', params=params)
# img, lb = input_list
# lb = input_list['img_path']
lb = input_list
print(lb)
b = 0
show = False
local_var_init_op = tf.local_variables_initializer()
table_init_ops = tf.tables_initializer() # iterator_initilizor in here
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(local_var_init_op)
sess.run(table_init_ops)
for i in range(330):
print('%d: ' % i)
if b == 0 or b+options.batch_size > dataset.num_examples_per_epoch('train'):
show = True
b = b+options.batch_size
rst = sess.run(lb)
# rst = rst.decode('utf-8')
print(len(rst))
def parse_file(filename, platform=None, folders=None, includes=None):
"""Run a C header or code file through C preprocessor program.
'filename' is the file to feed CPP.
'platform' is the platform to simulate.
'folders' is directories to -Include.
'includes' is a set of filename to #include.
"""
# Just read the content of the file if we don't want to use cpp
if not Options.use_cpp:
with open(filename, 'r') as f:
return f.read()
if folders is None:
folders = set()
if includes is None:
includes = []
config = Options.match_file(filename)
path_list = _get_cpp()
# Add fake includes and includes from configuration
path_list.append(r'-I../utils/fake_libc_include')
# Add as include the folder the files are located in
if os.path.dirname(filename):
folders.add(os.path.dirname(filename))
# Add all -Include cpp arguments
if folders or config.include_dirs:
folders |= set(config.include_dirs)
path_list.extend('-I%s' % i for i in folders)
# Define macros
if platform is not None:
path_list.extend(
['-D%s=%s' % (i, j) for i, j in platform.macros.items()])
# Add any C preprocesser arguments from CLI or config
path_list.extend('-D%s' % i for i in config.defines)
path_list.extend('-U%s' % i for i in config.undefines)
path_list.extend(config.arguments)
if config.includes or includes:
# Find all includes and their dependencies
unprosess = [filename] + includes + config.includes
processed = set()
all_includes = []
while len(unprosess) > 0:
file = unprosess.pop(0)
processed.add(file)
if file in all_includes:
all_includes.remove(file) # Move it to the front
all_includes.insert(0, file)
unprosess.extend(i for i in Options.match_file(file).includes
if i not in processed)
feed = '\n'.join('#include "%s"' % i for i in all_includes) + '\n'
else:
path_list.append(filename)
feed = ''
# Missing universal newlines forces input to expect bytes
if not sys.platform.startswith('win'):
feed = bytes(feed, 'ascii')
# Call C preprocessor with args and file
with Popen(path_list,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True) as proc:
text, warnings = proc.communicate(input=feed)
if warnings:
print(warnings.strip(), file=sys.stderr)
return '\n'.join(post_cpp(text.split('\n')))
# Parse options
Usage = 'Usage: python3 rsem.py -c config_file'
parser = OptionParser(usage=Usage)
parser.add_option('-c',
'--config_file',
action='store',
type='string',
dest='config_file',
help='Path to configuration file')
options, args = parser.parse_args()
config_file = options.config_file
opt = Options(config_file)
csv_path = path.split(config_file)[0] + '/' + opt.name + '.csv'
if opt.remove_resp:
inventory = read_inventory(opt.inventory)
else:
inventory = None
def pre_process(tr, inventory, freqmin, freqmax, order, factor):
print('\nPre-processing')
# Decimate
if not opt.remove_resp:
logger.info("log_dir:%s", opt.tt_data_dir)
if opt.audio_log_dir is None:
raise ValueError
else:
logger.info("log_dir: %s", opt.audio_log_dir)
if opt.gpu_num == 1:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if __name__ == "__main__":
# load configrations
# ==========================================================================
opt = Options().parse()
argchk(opt)
# ==========================================================================
if opt.stage == "tr":
filepatterns = ["*mixture*.wav", "*speech*.wav"]
tr_data = SeparatedDataset(data_dir_list=[opt.tr_data_dir],
file_pattern_list=filepatterns)
cv_data = SeparatedDataset(data_dir_list=[opt.cv_data_dir],
file_pattern_list=filepatterns)
tr_loader = DataLoader(tr_data,
opt.batch_size * opt.gpu_num,
shuffle=True)
cv_loader = DataLoader(cv_data, opt.batch_size * opt.gpu_num)
# pre_trained_model_path = "/data/tengxiang/tasnet/log/v0/"
def show_mask_norms(mask_folder, model_name = 'gtsrb', out_png=False):
options = Options()
options.model_name = model_name
options = justify_options_for_model(options, model_name)
options.data_subset = 'validation'
options.batch_size = 1
options.num_epochs = 1
options.net_mode = 'backdoor_def'
options.load_mode = 'all'
options.fix_level = 'all'
options.build_level = 'mask_only'
options.selected_training_labels = None
options.gen_ori_label = False
ld_paths = dict()
root_folder = mask_folder
print(root_folder)
dirs = os.listdir(root_folder)
for d in dirs:
tt = d.split('_')[0]
if len(tt) == 0:
continue
try:
tgt_id = int(tt)
except:
continue
ld_paths[tgt_id] = get_last_checkpoint_in_folder(os.path.join(root_folder,d))
print(ld_paths)
model, dataset, input_list, feed_list, out_op, aux_out_op = get_output(options, model_name=model_name)
model.add_backbone_saver()
mask_abs = dict()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
import cv2
init_op = tf.global_variables_initializer()
local_var_init_op = tf.local_variables_initializer()
table_init_ops = tf.tables_initializer() # iterator_initilizor in here
with tf.Session(config=config) as sess:
sess.run(init_op)
sess.run(local_var_init_op)
sess.run(table_init_ops)
for k, v in ld_paths.items():
print(v)
model.load_backbone_model(sess, v)
pattern, mask = sess.run([out_op, aux_out_op])
pattern = (pattern[0]+1)/2
mask = mask[0]
mask_abs[k] = np.sum(np.abs(mask))
if out_png:
show_name = '%d_pattern.png'%k
out_pattern = pattern*255
cv2.imwrite(show_name, out_pattern.astype(np.uint8))
show_name = '%d_mask.png'%k
out_mask = mask*255
cv2.imwrite(show_name, out_mask.astype(np.uint8))
show_name = '%d_color.png'%k
out_color = pattern*mask*255
cv2.imwrite(show_name, out_color.astype(np.uint8))
#cv2.imshow(show_name,out_pattern)
#cv2.waitKey()
#break
out_norms = np.zeros([len(mask_abs),2])
z = 0
for k,v in mask_abs.items():
out_norms[z][0] = k
out_norms[z][1] = v
z = z+1
print('===Results===')
np.save('out_norms.npy', out_norms)
print('write norms to out_norms.npy')
#return
vs = list(mask_abs.values())
import statistics
me = statistics.median(vs)
abvs = abs(vs - me)
mad = statistics.median(abvs)
rvs = abvs / (mad * 1.4826)
print(mask_abs)
print(rvs)
x_arr = [i for i in range(len(mask_abs))]
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(rvs)
plt.show()
def json_to_options(json_dict):
options = Options()
for k,v in json_dict.items():
setattr(options,k,v)
return options
# elif mode == "attended":
# img = attend_crop_img(img, attend_size=opt.attend_size)
else:
print("Mode in image mode {} is not implemented.".format(mode))
# loop through models
for model_name in model_names:
print('\nComputing output for model {} and image {}.\n'.format(
model_name, stimulus_name))
output = main(img_path, model_type=model_name)
# loop through labels
for model_region in opt.model_regions[model_name]:
out = output[model_region].flatten()
activation_data.append([
stimulus_name, img_mode, model_name, model_region, out
])
return pd.DataFrame(activation_data, columns=columns)
opt = Options()
# load pre-trained models
model_names = ['ResNet50-ImageNet', 'ResNet50-HMR']
print("\t Model Data")
# load activations of pre-trained models applied to stimuli
model_data = models2df(model_names, opt)
make_path(opt.model_data_dir)
model_data.to_pickle(os.path.join(opt.model_data_dir, 'ResNets.pkl'))
def args_to_options(**kargs):
options=Options()
for k,v in kargs.items():
if hasattr(options,k):
setattr(options,k,v)
return options
from config import Options
from utils import save_options_to_file, read_options_from_file
import os
options = Options()
for i in range(1, 50):
options.pretrained_filepath = options.home_dir + 'workspace/benchmarks/gtsrb_models/haha_' + str(
i)
options.poison_number_limit = i + 1
#options.poison_number_limit = 0
options.out_npys_prefix = options.out_npys_folder + 'out_' + str(i + 1)
options.checkpoint_folder = options.home_dir + 'data/checkpoint_' + str(
i + 1) + '/'
save_options_to_file(options, 'config.try')
cmmd = 'python3 train_gtsrb.py --config=config.try'
os.system(cmmd)
def hidden1_L1Norm(self):
return torch.norm(self.hidden.weight, 1)
if __name__ == '__main__':
vocab_size = 23
maxlen = 20
embedding_size = 30
batch_size = 2
hidden_size = 25
n_layers = 1
bidirectional = True
embedding = torch.rand(batch_size, maxlen, embedding_size)
encoder = RNNEncoder(maxlen,
embedding_size,
hidden_size,
n_layers=n_layers,
bidirectional=bidirectional,
variable_lengths=True)
e_output, e_hidden = encoder(embedding, [4, 2])
words = torch.LongTensor([[2, 3, 4, 1], [2, 6, 1, 0]])
words = Variable(words)
lenghts = [4, 3]
from config import Options
opt = Options()
opt.vocab_size = 20
opt.pooling_type = 2
feature_extractor = TextFeature(opt)
feature = feature_extractor(words, lenghts)
print(feature)
