def AddDomainAdversarialClassifier(mapped_input, use_pooling=False):
"""Takes a Keras object containing embeddings that have been mapped
to a common representation, and returns a binary domain classifier
with gradient reversal.
"""
if use_pooling:
domain_pooled_input = GlobalAveragePooling1D()(mapped_input)
grad_rev = GradientReversal(1)(domain_pooled_input)
else:
grad_rev = GradientReversal(1)(mapped_input)
domain_transformer = Dense(1)(grad_rev)
domain_classifier = Activation('sigmoid')(domain_transformer)
return domain_classifier
def _build_models(self, hp_lambda=1., lr=0.001):
# Input images from both domains
img = Input(shape=self.img_shape)
self.e = self._build_extracter()
self.c = self._build_classifier()
self.d = self._build_discriminator()
f = self.e(img)
gradInv = self.gradInv = GradientReversal(hp_lambda=hp_lambda)
K.set_value(gradInv.hp_lambda, hp_lambda)
fInv = gradInv(f)
cls = self.c(f)
dom = self.d(fInv)
self.model = Model(inputs=img, outputs=cls, name="model")
self.compile(self.model, lr, name='classifier')
self.classifier = Model(inputs=img, outputs=cls, name="classifier")
self.compile(self.classifier, lr, name='classifier')
self.discriminator = Model(inputs=img, outputs=dom, name="discriminator")
self.compile(self.discriminator, lr * 0.1, name='discrimimator')
def build_adv_style_classifier(self):
fx, l2_val = 'relu', self.l2_val
hp_lambda = self.hp_lambda
Flip = GradientReversal(hp_lambda)
encoded = Input(shape=(self.enc_shape))
layer = Flip(encoded)
h = Flatten(input_shape=self.enc_shape)(layer)
h1 = Dense(256,
kernel_regularizer=regularizers.l2(l2_val),
activation=fx)(h)
h2 = Dense(256,
kernel_regularizer=regularizers.l2(l2_val),
activation=fx)(h1)
h3 = Dense(256,
kernel_regularizer=regularizers.l2(l2_val),
activation=fx)(h2)
label_style = Dense(self.num_styles, activation='softmax')(h3)
# reverse gradients during back-prop
return Model(encoded, label_style, name="ADV")
def writers_adversarial_network(class_num, writers_num):
"""
Our WSAN
:param class_num: The total number of chinese character
:param writers_num:
:return:
"""
# init softmax
random_normal = RandomNormal(stddev=0.001, seed=1995)
reg = 1e-9
top5_acc = functools.partial(top_k_categorical_accuracy, k=5)
top5_acc.__name__ = 'top5_acc'
inputs = Input(shape=(64, 64, 1))
x = Conv2D(80, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(80, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.1)(x)
x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(76, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.1)(x)
x = Conv2D(152, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(152, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
gap1 = GlobalAveragePooling2D()(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.2)(x)
x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(192, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
# x = Dropout(0.2)(x)
x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False,
kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
gap2 = GlobalAveragePooling2D()(x)
comb_gap = layers.Concatenate(axis=-1)([gap1, gap2])
source_classifier = Dropout(0.5)(comb_gap)
source_classifier = Dense(class_num, activation='softmax', name='CR',
kernel_initializer=random_normal, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(
source_classifier)
grl = GradientReversal(hp_lambda=0.01)(comb_gap) # grl
domain_classifier = Dropout(0.5)(grl)
domain_classifier = Dense(writers_num, activation='softmax', name='SR',
kernel_initializer=random_normal, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(
domain_classifier)
opt = optimizers.Adadelta(lr=0.3, rho=0.95, epsilon=None, decay=1e-5)
comb_model = Model(inputs=inputs, outputs=[source_classifier, domain_classifier])
# comb_model.summary()
# 设置多GPU
comb_model = multi_gpu_model(comb_model, gpus=2)
comb_model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy', top5_acc])
source_model = Model(inputs=inputs, outputs=[source_classifier])
source_model.compile(optimizer=opt, loss='categorical_crossentropy',
metrics=['accuracy', top5_acc])
domain_model = Model(inputs=inputs, outputs=[domain_classifier])
domain_model.compile(optimizer=opt, loss='categorical_crossentropy',
metrics=['accuracy'])
return comb_model, source_model, domain_model
def train(config = {"minNJetBin": 7, "maxNJetBin": 11, "gr_lambda": 0, "nNodes":70, "nNodesD":10,
"nHLayers":1, "nHLayersD":1, "drop_out":0.7, "batch_size":2048, "epochs":100,
"lr":0.001, "verbose":1, "Mask":False, "Mask_nJet":7}):
# Define ouputDir based on input config
outputDir = "Output_1/"
for key in sorted(config.keys()):
outputDir += key+"_"+str(config[key])+"_"
config["outputDir"] = outputDir
# Define vars for training
jVec = ["Jet_pt_", "Jet_eta_", "Jet_phi_", "Jet_m_"]
lepton = ["GoodLeptons_pt_1", "GoodLeptons_eta_1", "GoodLeptons_phi_1", "GoodLeptons_m_1"]
MET = ["lvMET_cm_pt", "lvMET_cm_eta", "lvMET_cm_phi", "lvMET_cm_m"]
eventShapeVars = ["fwm2_top6", "fwm3_top6", "fwm4_top6", "fwm5_top6", "jmt_ev0_top6", "jmt_ev1_top6", "jmt_ev2_top6"]
numJets = ["NGoodJets_double"]
nJets = 7
if config["Mask"]: nJets = config["Mask_nJet"]
jVecs = list(y+str(x+1) for y in jVec for x in range(nJets))
config["allVars"] = jVecs + lepton
# Import data
print("----------------Preparing data------------------")
#config["dataSet"] = "EventShapeTrainingData_V3/"
#config["dataSet"] = "BackGroundMVA_V4_CM_GoodJets/"
#config["dataSet"] = "BackGroundMVA_V5_CM_Jets/"
#config["dataSet"] = "BackGroundMVA_V6_noCM_GoodJets/"
#config["dataSet"] = "BackGroundMVA_V8_All_GoodJets/"
config["dataSet"] = "BackGroundMVA_V9_CM_All_GoodJets_Inclusive/"
config["massModels"] = ["350","450","550","650","750","850"]
#ttMClist = ["TTJets*", "TT"]
ttMClist = ["T*", "TT"]
config["ttbarMC"] = ttMClist[0]
config["otherttbarMC"] = ttMClist[1]
print "Using "+config["dataSet"]+" data set"
print "Training variables:"
print config["allVars"]
print "Training on mass models: ", config["massModels"]
print "Training on ttbarMC: ", config["ttbarMC"]
if os.path.exists(config["outputDir"]):
print "Removing old training files: ", config["outputDir"]
shutil.rmtree(config["outputDir"])
os.makedirs(config["outputDir"]+"/log_graph")
sgTrainSet = sum( (glob(config["dataSet"]+"trainingTuple_*_division_0_*_"+mass+"*_training_0.h5") for mass in config["massModels"]) , [])
bgTrainSet = glob(config["dataSet"]+"trainingTuple_*_division_0_"+config["ttbarMC"]+"_training_0.h5")
sgTestSet = sum( (glob(config["dataSet"]+"trainingTuple_*_division_2_*_"+mass+"*_test_0.h5") for mass in config["massModels"]) , [])
bgTestSet = glob(config["dataSet"]+"trainingTuple_*_division_2_"+config["ttbarMC"]+"_test_0.h5")
trainData, trainSg, trainBg = get_data(sgTrainSet, bgTrainSet, config)
testData, testSg, testBg = get_data(sgTestSet, bgTestSet, config)
bgTrainTT = glob(config["dataSet"]+"trainingTuple_*_division_0_TT_training_0.h5")
trainDataTT, trainSgTT, trainBgTT = get_data(sgTrainSet, bgTrainTT, config)
# Make and train model
print("----------------Preparing training model------------------")
gr_lambda = 4
nNodes = 70
nNodesD = 10
nHLayers = 1
nHLayersD = 1
drop_out = 0.7
batch_size = 2048
epochs = 100
lr = 0.001
class_weight = {0: {0: 1.0, 1: 1.0}, 1: {0: 1.0, 1: 5.0, 2: 25.0, 3: 125.0, 4: 625.0}}
sample_weight = None#{0: trainData["Weight"][:,0].tolist(), 1: trainData["Weight"][:,0].tolist()}
#optimizer = keras.optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0)
optimizer = keras.optimizers.Adam(lr=config["lr"], beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
n_hidden_layers = list(config["nNodes"] for x in range(config["nHLayers"]))
n_hidden_layers_D = list(config["nNodesD"] for x in range(config["nHLayersD"]))
Flip = GradientReversal(config["gr_lambda"])
main_input = keras.layers.Input(shape=(trainData["data"].shape[1],), name='main_input')
# Set the rescale inputs to have unit variance centered at 0 between -1 and 1
layer = keras.layers.Lambda(lambda x: (x - K.constant(trainDataTT["mean"])) * K.constant(trainDataTT["scale"]), name='normalizeData')(main_input)
layer = keras.layers.Dense(config["nNodes"], activation='relu')(layer)
for n in n_hidden_layers:
layer = keras.layers.BatchNormalization()(layer)
layer = keras.layers.Dense(n, activation='relu')(layer)
layer = keras.layers.Dropout(config["drop_out"])(layer)
first_output = keras.layers.Dense(trainData["labels"].shape[1], activation='softmax', name='first_output')(layer)
layer = Flip(first_output)
#layer = keras.layers.Dense(nNodesD, activation='relu')(first_output)
for n in n_hidden_layers_D:
layer = keras.layers.BatchNormalization()(layer)
layer = keras.layers.Dense(n, activation='relu')(layer)
layer = keras.layers.Dropout(config["drop_out"])(layer)
second_output = keras.layers.Dense(trainData["domain"].shape[1], activation='softmax', name='second_output')(layer)
model = keras.models.Model(inputs=main_input, outputs=[first_output, second_output], name='model')
model.compile(loss=[make_loss_model(c=1.0) , make_loss_adversary(c=1.0)], optimizer=optimizer, metrics=['accuracy'])
tbCallBack = keras.callbacks.TensorBoard(log_dir="./"+outputDir+"/log_graph", histogram_freq=0, write_graph=True, write_images=True)
log_model = keras.callbacks.ModelCheckpoint(outputDir+"/BestNN.hdf5", monitor='val_loss', verbose=config["verbose"], save_best_only=True)
callbacks = []
if config["verbose"] == 1:
callbacks = [log_model, tbCallBack]
result_log = model.fit(trainData["data"], [trainData["labels"], trainData["domain"]], batch_size=config["batch_size"], epochs=config["epochs"], class_weight=class_weight,
validation_data=(testData["data"], [testData["labels"], testData["domain"]]), callbacks=callbacks, sample_weight=sample_weight)
# Model Visualization
keras.utils.plot_model(model, to_file=outputDir+"/model.png", show_shapes=True)
# Save trainig model as a protocol buffers file
inputName = model.input.op.name.split(':')[0]
outputName = model.output[0].op.name.split(':')[0]
print "Input name:", inputName
print "Output name:", outputName
config["input_output"] = [inputName, outputName]
saver = tf.train.Saver()
saver.save(keras.backend.get_session(), outputDir+"/keras_model.ckpt")
export_path="./"+outputDir+"/"
freeze_graph_binary = "python freeze_graph.py"
graph_file=export_path+"keras_model.ckpt.meta"
ckpt_file=export_path+"keras_model.ckpt"
output_file=export_path+"keras_frozen.pb"
command = freeze_graph_binary+" --input_meta_graph="+graph_file+" --input_checkpoint="+ckpt_file+" --output_graph="+output_file+" --output_node_names="+outputName+" --input_binary=true"
os.system(command)
#Plot results
print("----------------Validation of training------------------")
val = Validation(model, config, sgTrainSet, trainData, trainSg, trainBg, result_log)
config, metric = val.plot()
#Clean up training
K.clear_session()
tf.reset_default_graph()
return config, metric
bias = False
eps = 1.1e-5
#### adverse-surreal
K.clear_session()
top_model = eegnet(**params)
x = Flatten()(top_model.output)
clf = Dense(params['num_classes'],
activation='softmax',
kernel_initializer=initializers.he_normal(seed=random_seed),
name='clf',
use_bias=True)(x)
dann_in = GradientReversal(hp_lambda=params['hp_lambda'])(
x) ## hp_lambda controls the effect of inverse gradient
dsc = Dense(1,
activation='sigmoid',
kernel_initializer=initializers.he_normal(seed=random_seed),
name='dsc',
use_bias=True)(dann_in)
model = Model(top_model.input, [clf, dsc])
# model.summary()
if load_path:
model.load_weights(filepath=load_path, by_name=False)
model_json = model.to_json()
with open(
os.path.join(model_dir, log_name, 'model.json').replace('\\', '/'),
"w") as json_file:
json_file.write(model_json)
def build(self):
def slice_reshape(x):
# print 'self.i', self.i, self.config['text1_maxlen']
x1 = K.tf.slice(x, [0, self.i, 0],
[-1, 1, self.config['text1_maxlen']])
x2 = K.tf.reshape(tensor=x1,
shape=(-1, self.config['text1_maxlen']))
return x2
def concate(x):
return K.tf.concat([xx for xx in x], axis=3)
def stack(x):
return K.tf.stack([xx for xx in x], axis=1)
query = Input(name='query',
shape=(
self.config['text1_max_utt_num'],
self.config['text1_maxlen'],
)) # get the data by name
# show_layer_info('Input query', query)
doc = Input(name='doc', shape=(self.config['text2_maxlen'], ))
# show_layer_info('Input doc', doc)
embedding = Embedding(self.config['vocab_size'],
self.config['embed_size'],
weights=[self.config['embed']],
trainable=self.embed_trainable)
d_embed = embedding(doc)
# show_layer_info('Doc Embedding', d_embed)
accum_stack = []
q_embeds = []
q_bigru_reps = []
d_bigru_reps = []
for i in range(self.config['text1_max_utt_num']):
self.i = i
query_cur_utt = Lambda(slice_reshape)(query)
# show_layer_info('query_cur_utt', query_cur_utt)
q_embed = embedding(query_cur_utt)
q_embeds.append(q_embed)
# show_layer_info('Query Embedding', q_embed)
q_rep = Bidirectional(
GRU(self.config['hidden_size'],
return_sequences=True,
dropout=self.config['dropout_rate']))(q_embed)
q_bigru_reps.append(q_rep)
# show_layer_info('Bidirectional-GRU', q_rep)
d_rep = Bidirectional(
GRU(self.config['hidden_size'],
return_sequences=True,
dropout=self.config['dropout_rate']))(d_embed)
d_bigru_reps.append(d_rep)
# show_layer_info('Bidirectional-GRU', d_rep)
cross1 = Match(match_type='dot')(
[q_embed, d_embed]
) # dot product of embeddings (Can try other interaction functions here such as cosine, ind, bi-linear, etc.)
cross2 = Match(match_type='dot')(
[q_rep, d_rep]) # dot product of GRU output representations
# show_layer_info('Match-dot1', cross1)
# show_layer_info('Match-dot2', cross2)
cross = Lambda(concate)([cross1, cross2])
z = Reshape(
(self.config['text1_maxlen'], self.config['text2_maxlen'],
2))(cross) # batch_size * t1_len * t2_len * 2 channels
# show_layer_info('Reshape', z)
for j in range(self.config['num_conv2d_layers']):
z = Conv2D(filters=self.config['2d_kernel_counts'][j],
kernel_size=self.config['2d_kernel_sizes'][j],
padding='valid',
activation='relu')(z)
# show_layer_info('Conv2D', z)
z = MaxPooling2D(
pool_size=(self.config['2d_mpool_sizes'][j][0],
self.config['2d_mpool_sizes'][j][1]))(z)
# show_layer_info('MaxPooling2D', z)
z = Flatten()(z)
# show_layer_info('Flatten-z', z)
z = Dense(50, activation="tanh")(
z) # MLP 50 is the setting in Wu et al in ACL'17
# show_layer_info('Dense-z', z)
accum_stack.append(z)
accum_stack = Lambda(stack)(
accum_stack) # batch_size * max_turn_num * 50
# show_layer_info('accum_stack', accum_stack)
# GRU for Matching Accumulation
accum_stack_gru_hidden = Bidirectional(
GRU(self.config['hidden_size'],
return_sequences=True,
dropout=self.config['dropout_rate']))(accum_stack)
# show_layer_info('accum_stack_gru_hidden', accum_stack_gru_hidden)
accum_stack_gru_hidden_flat = Reshape((-1, ))(accum_stack_gru_hidden)
# show_layer_info('accum_stack_gru_hidden_flat', accum_stack_gru_hidden_flat)
accum_stack_gru_hidden_flat_drop = Dropout(
rate=self.config['dropout_rate'])(accum_stack_gru_hidden_flat)
# show_layer_info('Dropout', accum_stack_gru_hidden_flat_drop)
#DMN-DAL or DMN-MTL
if (self.config["domain_training_type"] == "DMN-ADL"):
flip_grad = True
else:
flip_grad = False
GRL = GradientReversal(hp_lambda=self.l, really_flip=flip_grad)
word_embed_rep = Flatten()(concatenate(q_embeds + [d_embed]))
word_bigru_rep = Flatten()(concatenate(q_bigru_reps + d_bigru_reps))
q_d_rep = concatenate([word_embed_rep, word_bigru_rep])
match_representations = accum_stack_gru_hidden_flat_drop
if 'input_to_domain_clf' in self.config and self.config[
'input_to_domain_clf'] == 'query_doc':
in_domain_clf = GRL(q_d_rep)
else:
in_domain_clf = GRL(match_representations)
# show_layer_info('in_domain_clf', in_domain_clf)
number_of_domains = 2
if 'number_of_categories' in self.config:
number_of_domains = self.config['number_of_categories']
elif 'train_clf_with_ood' in self.config and self.config[
'train_clf_with_ood']:
number_of_domains = 3
out_domain = Dense(number_of_domains,
activation='softmax')(in_domain_clf)
# show_layer_info('out_domain', out_domain)
model_clf = Model(inputs=[query, doc], outputs=out_domain)
# print(model_clf.summary())
# MLP
if self.config['target_mode'] == 'classification':
out_ = Dense(
2, activation='softmax')(accum_stack_gru_hidden_flat_drop)
elif self.config['target_mode'] in ['regression', 'ranking']:
out_ = Dense(1)(accum_stack_gru_hidden_flat_drop)
# show_layer_info('Dense', out_)
#model = Model(inputs=[query, doc, dpool_index], outputs=out_)
model = Model(inputs=[query, doc], outputs=out_)
# print(model.summary())
return model, model_clf, self.l