def run(self):
"""Run the ETL pipeline."""
spark = self.initiate_spark()
# connection, cursor = self.connect_to_redshift()
# make_tables(connection, cursor, self.cfg.get('sql'))
transformer(spark, self.cfg.get('spark'), local_write=True)
def __init__(self, items, mode=const.EXPLORE): self.mode = mode self.logservice = logservice() self.db = memoryDB.memoryDB() self.transformer = transformer.transformer(self.db) self.sizeOfTarget = 20 self.trainer.setItems([(o.id, self.transformer.transform(o, 20)) for o in items])
def upload_file():
downloadLink = {}
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
for file in request.files.getlist('file'):
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
metaData = transformer.transformer(file.filename)
if localTest == 1:
downloadLink[filename] = (metaData,
url_for('uploaded_file', filename=metaData[0]))
else:
# 注：此处'/eBird_to_birdreportcn'是为了适配服务器端
downloadLink[filename] = (metaData,
'/eBird_to_birdreportcn'+url_for('uploaded_file', filename=metaData[0]))
return render_template('eBird to birdreportcn.html', **{"downloadLink": downloadLink})
def main():
log.basicConfig(format = '[ %(levelname)s ] %(message)s',
level = log.INFO, stream = sys.stdout)
args = build_argparser().parse_args()
try:
model_wrapper = openvino_io_model_wrapper()
data_transformer = transformer()
io = io_adapter.get_io_adapter(args, model_wrapper, data_transformer)
iecore = utils.create_ie_core(args.extension, args.cldnn_config, args.device,
args.nthreads, None, args.dump, 'sync', log)
net = utils.create_network(iecore, args.model_xml, args.model_bin, log)
input_shapes = utils.get_input_shape(model_wrapper, net)
for layer in input_shapes:
log.info('Shape for input layer {0}: {1}'.format(layer, input_shapes[layer]))
utils.reshape_input(net, args.batch_size)
log.info('Prepare input data')
io.prepare_input(net, args.input)
log.info('Create executable network')
exec_net = utils.load_network(iecore, net, args.device, args.priority, 1)
log.info('Starting inference ({} iterations) on {}'.
format(args.number_iter, args.device))
result, time = infer_sync(exec_net, args.number_iter, io.get_slice_input)
average_time, latency, fps = process_result(time, args.batch_size, args.mininfer)
if not args.raw_output:
io.process_output(result, log)
result_output(average_time, fps, latency, log)
else:
raw_result_output(average_time, fps, latency)
del net
del exec_net
del iecore
except Exception as ex:
print('ERROR! : {0}'.format(str(ex)))
sys.exit(1)
def finalParameters(n=10):
with tf.Session() as sess:
transformed = sess.run(transformer(inp_arr[-1][0:n], rot_arr[-1][0:n]))
for i in range(n):
plt.imshow(inp_arr[-1][i, :, :, 0])
plt.figure()
plt.imshow(transformed[i, :, :, 0])
plt.show()
def plottransformed(arr, param):
side = int(np.sqrt(np.size(arr)) + 0.5)
plt.imshow(np.reshape(arr, (side, side)))
plt.figure()
toplot = transformer(np.reshape(arr, (1, side, side, 1)), param)
with tf.Session() as sess:
plt.imshow(np.reshape(sess.run(toplot), (side, side)))
plt.show()
def dataUpdate(base_path, model_type, predict_label_year, upload_cms):
# Scrape data from different sources and get latest year
predict_feature_year = str(int(predict_label_year[:4]) - 1)
train_label_years = [str(i+1) + '-' + str(i+3)
for i in range(2010, int(predict_feature_year)-3, 1)]
latest_label_year = hospitalizationScraper(model_type, train_label_years, base_path)
predict_feature_year_lst = [str(i) for i in range(int(latest_label_year[:4]), int(predict_label_year[:4]))] # Yilun
train_feature_years = [str(i) for i in range(2010, int(predict_feature_year)-3, 1)]
latest_ACS = acsScraper(train_feature_years, predict_feature_year_lst, base_path)
latest_AQI = aqiScraper(train_feature_years, predict_feature_year_lst, base_path)
if upload_cms is None:
latest_CMS = cmsScraper(train_feature_years, predict_feature_year, base_path)
else:
latest_CMS = int(predict_feature_year)
# Compare latest_year with predict_feature_year
latest_year = min(latest_ACS, latest_AQI, latest_CMS)
if latest_year < int(predict_feature_year):
latest_year_message = 'Unable to fetch the latest year!\n'
if latest_ACS < int(predict_feature_year):
latest_year_message += 'ACS data is only available till {}!\n'.format(latest_ACS)
if latest_AQI < int(predict_feature_year):
latest_year_message += 'AQI data is only available till {}!\n'.format(latest_AQI)
if (latest_CMS < int(predict_feature_year)) & (upload_cms is None):
latest_year_message += 'CMS data is only available till {}!\n'.format(latest_CMS)
else:
# Data preprocessing for each data sources
hospitalizationPreprocessor(train_label_years, base_path)
acsPreprocessor(base_path)
aqiPreprocessor(train_feature_years, predict_feature_year_lst, base_path)
if upload_cms is None:
cmsPreprocessor(base_path)
# Merge data sources
combiner(train_feature_years, predict_feature_year_lst, base_path)
# Impute missing values and transform columns
imputer(train_feature_years, predict_feature_year, base_path)
transformer(train_feature_years, predict_feature_year, base_path)
latest_year_message = 'Successfully update data to the latest year!'
return latest_year_message, train_feature_years, predict_feature_year
def main():
_t = transformer.transformer()
img = create_test_img()
ares_img = resample_analytical(img)
nres_img = resample_numerical(img, _t)
atxm_img = transform_analytical(img)
ntxm_img = transform_numerical(img, _t)
print "voxel resize correct: " + str(np.allclose(nres_img[1], nvox))
visualize_results(img, ares_img, nres_img[0], atxm_img, ntxm_img)
def upload():
def allowed_file(filename):
return ('.' in filename) and (filename.rsplit('.', 1)[1]
in app.config['ALLOWED_EXTENSIONS'])
if request.method == 'POST':
file = request.files['file']
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
if not os.path.exists(app.config['UPLOAD_FOLDER']):
os.makedirs(app.config['UPLOAD_FOLDER'])
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
transformer(filepath=app.config['UPLOAD_FOLDER'],
filename=filename)
return redirect(url_for('uploads', filename=filename))
return render_template('index.html', title='upload')
def initialize_variables(CONS, phi, level):
"""Resample target transform and initial velocity
initialize other objects for scale level"""
# container to hold all variables
VARS = {}
# smooth and downsample the images
aaSmoother = smoother.smoother(1 + level, 0, 1, 2, CONS['spacing'],
CONS['grid'], CONS['dtype'])
fix_smooth = np.copy(aaSmoother.smooth(CONS['fixed']))
mov_smooth = np.copy(aaSmoother.smooth(CONS['moving']))
VARS['fixed'] = zoom(fix_smooth, 1. / 2**level, mode='wrap')
VARS['moving'] = zoom(mov_smooth, 1. / 2**level, mode='wrap')
# initialize a few odds and ends
shape = VARS['fixed'].shape
VARS['spacing'] = CONS['spacing'] * 2**level
VARS['warped_transform'] = np.empty(shape + (len(shape), ),
dtype=CONS['dtype'])
# initialize or resample the deformation
if phi is None:
VARS['phi'] = np.zeros(shape + (len(shape), ), dtype=CONS['dtype'])
else:
zoom_factor = np.array(shape) / np.array(phi.shape[:-1])
phi_ = [
zoom(phi[..., i], zoom_factor, mode='nearest') for i in range(3)
]
VARS['phi'] = np.ascontiguousarray(np.moveaxis(np.array(phi_), 0, -1))
# initialize the transformer
VARS['transformer'] = transformer.transformer(shape, VARS['spacing'],
CONS['dtype'])
VARS['transformer'].set_initial_transform(CONS['initial_transform'])
# initialize the smoothers
VARS['field_smoother'] = smoother.smoother(
CONS['field_abcd'][0] * 2**level, *CONS['field_abcd'][1:],
VARS['spacing'], shape, CONS['dtype'])
VARS['grad_smoother'] = smoother.smoother(CONS['grad_abcd'][0] * 2**level,
*CONS['grad_abcd'][1:],
VARS['spacing'], shape,
CONS['dtype'])
# initialize the matcher
VARS['matcher'] = matcher.matcher(VARS['fixed'], VARS['moving'],
CONS['lcc_radius'])
return VARS
def play():
moves = []
# Get rotation factor from first move
display.draw_grid(init_board)
move = input.get_move(players[0]) - 1
trans = transformer.transformer(move)
move = trans.transform(move)
current_board = update_board(init_board, players[0], move)
moves.append(move)
# Get rotation / reflection from second move
move = engine.get_move(current_board)
trans.set_reflection(move)
move = trans.rotate(move) if trans.first_move == 4 else trans.reflect(move)
current_board = update_board(current_board, players[1], move)
moves.append(move)
# Continue
turn = 0
player = 0
winner = None
while turn < 7 and winner == None:
if player == 0:
display.draw_grid(trans.undo_transformation(current_board))
move = trans.transform(input.get_move(players[player]) - 1)
else:
move = engine.get_move(current_board)
current_board = update_board(current_board, players[player], move)
moves.append(move)
winner = engine.check_winner(current_board)
turn += 1
player = (player + 1) % 2
display.draw_grid(trans.undo_transformation(current_board))
if winner != None:
engine.update_weights(moves, (player + 1) % 2)
print '\n{0} wins!'.format(winner)
def main():
#load data
df = pd.read_csv(args.data_path)
print 'Raw data fields', df.columns
print 'Nb os samples', len(df)
print 'Missing values?', df.isnull().values.any()
print df.head()
features = ['type', 'amount', 'oldbalanceOrg', 'newbalanceOrig']
X_train_val, X_test, y_train_val, y_test = build_dataset(df, features)
print 'sample X_train:', X_train_val[0]
print 'sample X_test', X_test[0]
tf = transformer(categorical=True)
clf = RandomForestClassifier(n_estimators=50,
class_weight='balanced',
verbose=1)
p = Pipeline([('trans', tf), ('clf', clf)])
p.fit(X_train_val, y_train_val)
filename_p = 'model_pipeline.sav'
filename_Xtest = '../../explainers/data/paysim_data/test_data/X_test.npy'
filename_ytest = '../../explainers/data/paysim_data/test_data/y_test.npy'
joblib.dump(p, filename_p)
np.save(filename_Xtest, X_test)
np.save(filename_ytest, y_test)
p_loaded = joblib.load('model_pipeline.sav')
preds_test = p_loaded.predict(X_test)
proba_test = p_loaded.predict_proba(X_test)
preds_test.sum() / float(len(preds_test))
calculate_print_scores(y_test, preds_test, proba_test)
print clf.feature_importances_
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
#Hyperparameters
image_size = 64
batch_size = 128
num_z = 100 # length of the noise vector input to the generator i.e the size of the latent vector
num_channels = 3 # num of channels in the training image
num_gf = 64 # size of feature maps in generator
num_df = 64 # size of feature maps in discriminator
lr = 0.0002 # Learning rate for optimisers
beta1 = 0.5 # Beta1 hyperparameter for Adam optimizers
num_epochs = 5
#Image Transformations
transform = transformer(image_size)
#Loading the train data
trainloader = get_traindata(transform,batch_size)
# Plot some training images
real_batch = next(iter(trainloader))
plt.figure(figsize=(8,8))
plt.axis("off")
plt.title("Training Images")
plt.imshow(np.transpose(vutils.make_grid(real_batch[0].to(device)[:64], padding=2, normalize=True).cpu(),(1,2,0)))
# This function takes an initialized model as input and reinitializes all
# convolutional, convolutional-transpose, and batch normalization layers.
# all model weights are randomly initialized from a Normal
# distribution with mean=0, stdev=0.02
def model_fn(self, features, mode):
# todo can estimators handle dropout for us or do we need to do it on our own?
hparams = self.hparams(mode)
tf.logging.log(tf.logging.INFO, "Running in {} mode.".format(mode))
with tf.variable_scope("LISA", reuse=tf.AUTO_REUSE):
# features = tf.Print(features, [features, tf.shape(features)], 'input features')
batch_shape = tf.shape(features)
batch_size = batch_shape[0]
batch_seq_len = batch_shape[1]
layer_config = self.model_config['layers']
sa_hidden_size = layer_config['head_dim'] * layer_config[
'num_heads']
feats = {
f: features[:, :, idx]
for f, idx in self.feature_idx_map.items()
}
# todo this assumes that word_type is always passed in
words = feats['word_type']
# print("debug <input features>:", features)
# for masking out padding tokens
tokens_to_keep = tf.where(tf.equal(words, constants.PAD_VALUE),
tf.zeros([batch_size, batch_seq_len]),
tf.ones([batch_size, batch_seq_len]))
# Extract named features from monolithic "features" input
feats = {
f: tf.multiply(tf.cast(tokens_to_keep, tf.int32), v)
for f, v in feats.items()
}
# feats = {f: tf.Print(feats[f], [feats[f]]) for f in feats.keys()}
# print("<debug features>: ",feats)
# print("debug <model_config>:", self.model_config)
# Extract named labels from monolithic "features" input, and mask them
# todo fix masking -- is it even necessary?
labels = {}
for l, idx in self.label_idx_map.items():
# print("debug <label_idx_map idx>: ", idx)
these_labels = features[:, :, idx[0]:idx[0] + 1] if idx[
1] != -1 else features[:, :, idx[0]:]
these_labels_masked = tf.multiply(
these_labels,
tf.cast(tf.expand_dims(tokens_to_keep, -1), tf.int32))
# check if we need to mask another dimension
# these_labels_masked_print = tf.Print(these_labels_masked, [tf.shape(these_labels_masked), these_labels_masked, idx],
# 'thses labels masked')
if idx[1] == -1:
last_dim = tf.shape(these_labels)[2]
this_mask = tf.where(
tf.equal(these_labels_masked, constants.PAD_VALUE),
tf.zeros([batch_size, batch_seq_len, last_dim],
dtype=tf.int32),
tf.ones([batch_size, batch_seq_len, last_dim],
dtype=tf.int32))
these_labels_masked = tf.multiply(these_labels_masked,
this_mask)
else:
these_labels_masked = tf.squeeze(
these_labels_masked,
-1,
name='these_labels_masked_squeezing')
# these_labels_masked = tf.Print(these_labels_masked, [tf.shape(these_labels_masked), these_labels_masked], 'thses labels masked after squeezed')
labels[l] = these_labels_masked
# labels = [tf.Print(l, [l]) for l in labels]
# labels = [tf.Print("label_l", l)for l in labels]
# load transition parameters
transition_stats = util.load_transition_params(
self.task_config, self.vocab, hparams.train_with_crf)
# Create embeddings tables, loading pre-trained if specified
embeddings = {}
for embedding_name, embedding_map in self.model_config[
'embeddings'].items():
embedding_dim = embedding_map['embedding_dim']
if 'pretrained_embeddings' in embedding_map:
input_pretrained_embeddings = embedding_map[
'pretrained_embeddings']
include_oov = True
embedding_table = self.get_embedding_table(
embedding_name,
embedding_dim,
include_oov,
pretrained_fname=input_pretrained_embeddings,
cwr_ood=hparams.cwr_ood)
else:
num_embeddings = self.vocab.vocab_names_sizes[
embedding_name]
embedding_table = self.get_embedding_table(
embedding_name,
embedding_dim,
include_oov,
num_embeddings=num_embeddings)
# embedding_table = tf.Print(embedding_table, [tf.shape(embedding_table), embedding_table])
embeddings[embedding_name] = embedding_table
tf.logging.log(tf.logging.INFO,
"Created embeddings for '%s'." % embedding_name)
tf.logging.log(tf.logging.INFO, embeddings[embedding_name])
inputs_list = []
gp_embs = []
with tf.device("CPU:0"):
if hparams.cwr != "None" or hparams.glove_300d:
if hparams.cwr != "None":
cached_cwr_embeddings = tf.get_variable(
"cwr_embedding",
shape=self.cwr_embedding.shape,
trainable=False)
def init_fn(scaffold, sess):
if hparams.cwr != "None":
sess.run(
cached_cwr_embeddings.initializer, {
cached_cwr_embeddings.initial_value:
self.cwr_embedding
})
scaffold = tf.train.Scaffold(init_fn=init_fn)
for input_name, input_transformation_name in self.model_config[
'inputs'].items():
# print("debug <actual inputs>:", input_name, input_transformation_name)
input_values = feats[input_name]
# input_values = tf.Print(input_values, ["input value under {}".format(input_name), input_values, tf.shape(input_values)])
if input_transformation_name == "cached_embeddings":
# if hparams.cwr_ood:
# ROOT_emb = tf.get_variable("root_emb", shape=[1, 3072], initializer=tf.random_normal_initializer(), trainable=False)
# cached_cwr_embeddings_oov = tf.concat([cached_cwr_embeddings, ROOT_emb], axis=0)
# input_embedding_lookup = tf.nn.embedding_lookup(cached_cwr_embeddings_oov, input_values)
# else:
input_embedding_lookup = tf.nn.embedding_lookup(
cached_cwr_embeddings, input_values)
with tf.variable_scope("cwr_assembly"):
num_layers = 3 #input_embedding_lookup.get_shape()[2]
weight = tf.get_variable("cwr_weight",
shape=[num_layers])
scale = tf.get_variable("cwr_scale", shape=[])
input_embedding_lookup = scale * tf.math.reduce_sum(
tf.split(input_embedding_lookup,
axis=-1,
num_or_size_splits=num_layers) *
tf.reshape(tf.nn.softmax(weight),
shape=[num_layers, 1, 1, 1]),
axis=0)
elif input_transformation_name == "bert_embeddings":
input_embedding_lookup = tf.nn.embedding_lookup(
cached_cwr_embeddings, input_values)
elif input_transformation_name == "embeddings":
print("embeddings", input_name, embeddings[input_name])
input_embedding_lookup = tf.nn.embedding_lookup(
embeddings[input_name], input_values)
elif input_transformation_name == "predicate":
print("embeddings", input_name, embeddings[input_name])
input_embedding_lookup = tf.nn.embedding_lookup(
embeddings[input_name], input_values)
gp_embs.append(input_embedding_lookup)
continue
else:
print("unknown input transformation {}".format(
input_transformation_name))
raise NotImplementedError
# input_embedding_lookup = tf.Print(input_embedding_lookup, ["input embedding under {}".format(input_name), input_embedding_lookup])
inputs_list.append(input_embedding_lookup)
tf.logging.log(tf.logging.INFO,
"Added %s to inputs list." % input_name)
# TODO a mere workaround with one element concat
current_input = tf.concat(inputs_list, axis=-1)
## <guard: condition to enter sentence features>
## suppose the dim is of (B, S, H)
if hparams.input_project_layer_norm:
current_input = tf.contrib.layers.layer_norm(current_input)
sentence_feature = tf.reduce_sum(
current_input * tf.expand_dims(tokens_to_keep, -1), axis=1)
sentence_feature /= tf.expand_dims(
tf.reduce_sum(tokens_to_keep, axis=1),
-1) #To get the mean of all embeddings
feats['sentence_feature'] = sentence_feature
## <guard: condition to enter sentence features>
current_input = tf.nn.dropout(current_input, hparams.input_dropout)
if len(gp_embs) > 0:
current_input = tf.concat([current_input, gp_embs[0]], axis=-1)
with tf.variable_scope('project_input'):
current_input = nn_utils.MLP(current_input,
sa_hidden_size,
n_splits=1)
# current_input = tf.Print(current_input, [tf.shape(current_input)], "input shape")
predictions = {}
eval_metric_ops = {}
export_outputs = {}
loss = tf.constant(0.)
items_to_log = {}
num_layers = max(self.task_config.keys()) + 1
tf.logging.log(
tf.logging.INFO,
"Creating transformer model with %d layers" % num_layers)
with tf.variable_scope('transformer'):
current_input = transformer.add_timing_signal_1d(current_input)
for i in range(num_layers):
# print("debug: <constructing {}-th layer>".format(i))
with tf.variable_scope('layer%d' % i):
special_attn = [
[], []
] #first bracket is for hard injection attns, the sencond is for discounting attns
special_values = []
if i in self.attention_config:
this_layer_attn_config = self.attention_config[i]
# print("debug: <layer_{} config>: ".format(i), this_layer_attn_config)
print(
"debug <attention configuration>@{}: ".format(
i), this_layer_attn_config)
for attn_fn_item in this_layer_attn_config.keys():
for attn_fn, attn_fn_map in this_layer_attn_config[
attn_fn_item].items():
# print("debug <attn_fn, attn_fn_map>: ", attn_fn, ' ', attn_fn_map)
if 'length' in attn_fn_map.keys(
) or hparams.use_hparams_headcounts:
hc = hparams.__dict__['{}_headcount'.format(
attn_fn
)] if hparams.use_hparams_headcounts else attn_fn_map[
'length']
tf.logging.log(
tf.logging.INFO,
"{} is using {} attention mode with {} heads"
.format(
attn_fn_item, hparams.__dict__[
'{}_injection'.format(
attn_fn)], hc))
for idx in range(
hc
): # To make sure that the three special attentions are different
with tf.variable_scope(
'{}_{}'.format(
attn_fn_item, idx)):
attention_fn_params = attention_fns.get_params(
mode, attn_fn_map,
predictions, feats, labels,
hparams, self.model_config,
tokens_to_keep)
this_special_attn, special_attn_weight = attention_fns.dispatch(
attn_fn_map['name'])(
**attention_fn_params)
# todo patches everywhere!
# this_special_attn = tf.Print(this_special_attn, [this_special_attn])
if special_attn_weight is not None and hparams.output_attention_weight:
for i in range(
special_attn_weight.
get_shape()[0]):
items_to_log[
"{}_{}_weight_{}".
format(
attn_fn, idx, i
)] = special_attn_weight[
i]
if hparams.__dict__[
'{}_injection'.format(
attn_fn
)] == 'injection':
special_attn[0].append(
this_special_attn)
elif hparams.__dict__[
'{}_injection'.format(
attn_fn
)] == 'discounting':
special_attn[1].append(
this_special_attn)
else:
tf.logging.log(
tf.logging.ERROR,
"The spcified injection method {} has not been implemented"
.format(attn_fn_map[
'injection_method']))
raise NotImplementedError
# print(special_attn)
else:
with tf.variable_scope(
'{}'.format(attn_fn)):
attention_fn_params = attention_fns.get_params(
mode, attn_fn_map, predictions,
feats, labels, hparams,
self.model_config,
tokens_to_keep)
this_special_attn, _ = attention_fns.dispatch(
attn_fn_map['name'])(
**attention_fn_params)
if hparams.__dict__[
'{}_injection'.format(
attn_fn)] == 'injection':
special_attn[0].append(
this_special_attn)
elif hparams.__dict__[
'{}_injection'.format(
attn_fn)] == 'discounting':
special_attn[1].append(
this_special_attn)
else:
tf.logging.log(
tf.logging.ERROR,
"The spcified injection method {} has not been implemented"
.format(attn_fn_map[
'injection_method']))
raise NotImplementedError
# print("debug <layer_{} special attention>: ".format(i), special_attn )
if 'value_fns' in this_layer_attn_config:
tf.logging.log(
tf.logging.ERROR,
"special value section has been dropped temporarily"
)
raise NotImplementedError
for value_fn, value_fn_map in this_layer_attn_config[
'value_fns'].items():
value_fn_params = value_fns.get_params(
mode, value_fn_map, predictions, feats,
labels, embeddings)
this_special_values = value_fns.dispatch(
value_fn_map['name'])(
**value_fn_params)
special_values.append(this_special_values)
# print("debug <layer_{} special values>: ".format(i), special_values)
if hparams.attn_debug:
print(special_attn)
# special_attn[1][1] = tf.Print(special_attn[1][1], [special_attn[1][0], special_attn[1][1]], "debug_check equal attn")
# assert_op = tf.assert_none_equal(special_attn[1][0], special_attn[1][1])
# tf.logging.log(tf.logging.INFO, "attention behavior is identical")
current_input = transformer.transformer(
current_input,
tokens_to_keep,
layer_config['head_dim'],
layer_config['num_heads'],
hparams.attn_dropout,
hparams.ff_dropout,
hparams.prepost_dropout,
layer_config['ff_hidden_size'],
special_attn,
special_values,
special_attention_mode=hparams.
special_attention_mode)
# current_input = tf.Print(current_input, [tf.shape(current_input)], "LISA input after transformer")
if i in self.task_config:
# if normalization is done in layer_preprocess, then it should also be done
# on the output, since the output can grow very large, being the sum of
# a whole stack of unnormalized layer outputs.
current_input = nn_utils.layer_norm(current_input)
# todo test a list of tasks for each layer
for task, task_map in self.task_config[i].items():
# print("debug <task>: ", task)
# print("debug <task map>:" , task_map)
task_labels = labels[task]
# task_labels = tf.Print(task_labels, [task_labels] , 'task_label'.format(task))
task_vocab_size = self.vocab.vocab_names_sizes[
task] if task in self.vocab.vocab_names_sizes else -1
# Set up CRF / Viterbi transition params if specified
with tf.variable_scope(
"crf"
): # to share parameters, change scope here
# transition_stats_file = task_map['transition_stats'] if 'transition_stats' in task_map else None
task_transition_stats = transition_stats[
task] if task in transition_stats else None
# create transition parameters if training or decoding with crf/viterbi
task_crf = 'crf' in task_map and task_map[
'crf']
task_viterbi_decode = task_crf or 'viterbi' in task_map and task_map[
'viterbi']
transition_params = None
if task_viterbi_decode or task_crf:
# print("loading transition params", self.not_load_transition)
if hparams.train_with_crf:
transition_params = tf.get_variable(
"transitions", [
task_vocab_size,
task_vocab_size
])
else:
tf.logging.log(
tf.logging.INFO,
"Use default transition param")
tf.logging.log(
tf.logging.INFO,
"transition parameters:{}".
format(task_transition_stats))
transition_params = tf.constant(
task_transition_stats,
dtype=tf.float32
) #tf.get_variable("transitions", [task_vocab_size, task_vocab_size],
#initializer=tf.constant_initializer(task_transition_stats) if not self.not_load_transition else tf.constant_initializer(0),
#trainable=task_crf)
# if mode != ModeKeys.TRAIN:
# transition_params = tf.Print(transition_params, [tf.get_variable("transitions", [task_vocab_size, task_vocab_size])],
# "optimized transition?")
# transition_params = tf.cond(tf.equal(mode, ModeKeys.TRAIN),
# lambda: transition_params,
# lambda: tf.Print(transition_params, [
# tf.get_variable("transitions", [task_vocab_size, task_vocab_size])],
# "optimized transition?"))
# transition_params =
train_or_decode_str = "training" if task_crf else "decoding"
tf.logging.log(
tf.logging.INFO,
"Created transition params for %s %s"
% (train_or_decode_str, task))
output_fn_params = output_fns.get_params(
mode, self.model_config,
task_map['output_fn'], predictions, feats,
labels, current_input, task_labels,
task_vocab_size,
self.vocab.joint_label_lookup_maps,
tokens_to_keep, transition_params, hparams)
# print("debug <dispatch into {}>".format(task_map['output_fn']['name']))
task_outputs = output_fns.dispatch(
task_map['output_fn']['name'])(
**output_fn_params)
# print("debug <task_outputs>: ", task_outputs)
# want task_outputs to have:
# - predictions
# - loss
# - scores
# - probabilities
predictions[task] = task_outputs
# do the evaluation
for eval_name, eval_map in task_map[
'eval_fns'].items():
eval_fn_params = evaluation_fns.get_params(
task_outputs, eval_map, predictions,
feats, labels, task_labels,
self.vocab.reverse_maps,
tokens_to_keep)
if eval_name == 'parse_eval' and hparams.using_input_with_root:
eval_fn_params['has_root_token'] = True
eval_result = evaluation_fns.dispatch(
eval_map['name'])(**eval_fn_params)
eval_metric_ops[eval_name] = eval_result
# get the individual task loss and apply penalty
this_task_loss = task_outputs[
'loss'] * task_map['penalty']
# log this task's loss
items_to_log['%s_loss' % task] = this_task_loss
#outputing sub loss as well
for key in task_outputs.keys():
if key.startswith('loss'):
items_to_log['{}_{}'.format(
task, key)] = task_outputs[key]
# add this loss to the overall loss being minimized
# this_task_loss = tf.Print(this_task_loss, [this_task_loss], '{}_{}'.format(task, key))
loss += this_task_loss
# print("debug <accumulated loss>: ", loss)
# break # only take one loss
# set up moving average variables
assign_moving_averages_dep = tf.no_op()
if hparams.moving_average_decay > 0.:
moving_averager = tf.train.ExponentialMovingAverage(
hparams.moving_average_decay,
zero_debias=True,
num_updates=tf.train.get_global_step())
moving_average_op = moving_averager.apply(
train_utils.get_vars_for_moving_average(
hparams.average_norms))
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS,
moving_average_op)
# use moving averages of variables if evaluating
assign_moving_averages_dep = tf.cond(
tf.equal(mode, ModeKeys.TRAIN), lambda: tf.no_op(), lambda:
nn_utils.set_vars_to_moving_average(moving_averager))
# print("debug <finishing setting up moving avg variables>")
with tf.control_dependencies([assign_moving_averages_dep]):
items_to_log['loss'] = loss
# print("debug <final loss>: ", loss)
# get learning rate w/ decay
# todo dirty workaround
if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:
this_step_lr = train_utils.learning_rate(
hparams, tf.train.get_global_step())
items_to_log['lr'] = this_step_lr
# print("debug <items to log>: ", items_to_log)
# print("debug <eval_metric_content>: ", eval_metric_ops)
if hparams.optimizer == "lazyadam":
optimizer = LazyAdamOptimizer(
learning_rate=this_step_lr,
beta1=hparams.beta1,
beta2=hparams.beta2,
epsilon=hparams.epsilon,
use_nesterov=hparams.use_nesterov)
elif hparams.optimizer == "adam":
optimizer = tf.train.AdamOptimizer(
learning_rate=this_step_lr,
beta1=hparams.beta1,
beta2=hparams.beta2,
epsilon=hparams.epsilon)
else:
raise NotImplementedError(
"The specified optimizer is not implemented")
# loss = tf.Print(loss, [loss], "loss")
# # loss_no_nan = tf.cond(tf.reduce_any(tf.is_nan(loss)), lambda: tf.zeros_like(loss), lambda: loss)
# # loss_no_nan = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss)
# loss_no_nan = tf.where(tf.math.is_nan(loss), tf.zeros_like(loss), loss)
# loss_no_nan_printed = tf.Print(loss_no_nan, [loss_no_nan], "no nan loss")
# grad_and_var = optimizer.compute_gradients(loss_no_nan_printed)
# loss = tf.where(tf.math.is_nan(loss), tf.zeros_like(loss), loss)
# loss = tf.Print(loss, [loss], "loss")
grad_and_var = optimizer.compute_gradients(loss)
gradients, variables = zip(*grad_and_var)
# gradients_without_nan = [tf.cond(tf.reduce_any(tf.is_nan(item)), lambda: tf.zeros_like(item), item)for item in gradients]
# gradients_without_nan = gradients
gradients, gn = tf.clip_by_global_norm(
gradients, hparams.gradient_clip_norm)
# print([g is None for g in gradients])
# gn = gn[0]
zero_clipped_gradients = [
tf.clip_by_value(g, 0., 0.) if g is not None else g
for g in gradients
]
gradients_prev_inf_norm = [
tf.cond(
tf.logical_or(tf.math.is_inf(gn),
tf.math.is_nan(gn)), lambda: g_zeros,
lambda: g) if g is not None else None for g_zeros,
g in zip(zero_clipped_gradients, gradients)
]
# gn = tf.Print(gn, [gn], "global norm")
with tf.control_dependencies([gn]):
train_op = optimizer.apply_gradients(
zip(gradients_prev_inf_norm, variables),
global_step=tf.train.get_global_step())
# if hparams.debug and mode == tf.estimator.ModeKeys.TRAIN:
# gradients_to_print = [gradients[variables.index(var)] for var in nn_utils.gradient_to_watch]
# print(gradients_to_print)
# gradients[0] = tf.Print(gradients[0], gradients_to_print, "gradient for dependency label strength")
# train_op = optimizer.apply_gradients(zip(gradients, variables), global_step=tf.train.get_global_step())
# export_outputs = {'predict_output': tf.estimator.export.PredictOutput({'scores': scores, 'preds': preds})}
logging_hook = tf.train.LoggingTensorHook(items_to_log,
every_n_iter=100)
histogram_summary = [
summary
for name, summary in nn_utils.histogram_output.items()
]
summary_hook = tf.train.SummarySaverHook(
save_steps=500,
summary_op=[
tf.summary.scalar(k, items_to_log[k])
for k in items_to_log.keys()
] + histogram_summary)
flat_predictions = {
"%s_%s" % (k1, k2): v2
for k1, v1 in predictions.items() for k2, v2 in v1.items()
}
# print("debug <flat predictions>:", flat_predictions)
export_outputs = {
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
tf.estimator.export.PredictOutput(flat_predictions)
}
tf.logging.log(
tf.logging.INFO,
"Created model with %d trainable parameters" %
tf_utils.get_num_trainable_parameters())
# if hparams.cwr!= 'None':
# with tf.Session() as sess:
# sess.run(tf.global_variables_initializer(), feed_dict={cached_cwr_embeddings_placeholder: self.cwr_embedding})
if mode == tf.estimator.ModeKeys.TRAIN:
return tf.estimator.EstimatorSpec(
mode,
flat_predictions,
loss,
train_op,
eval_metric_ops,
training_hooks=[logging_hook, summary_hook],
export_outputs=export_outputs,
scaffold=scaffold if hparams.cwr != 'None' else None)
elif mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode,
flat_predictions,
loss,
train_op,
eval_metric_ops,
training_hooks=[logging_hook],
export_outputs=export_outputs,
scaffold=scaffold if hparams.cwr != 'None' else None)
elif mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(
mode,
flat_predictions,
loss,
tf.no_op(),
eval_metric_ops,
export_outputs=export_outputs,
scaffold=scaffold if hparams.cwr != 'None' else None)
theta = tf.stack([
tf.concat([
tf.stack([h * tf.math.cos(r), tf.math.sin(r)], axis=1),
tf.expand_dims(x, 1)
],
axis=1),
tf.concat([
tf.stack([-tf.math.sin(r), w * tf.math.cos(r)], axis=1),
tf.expand_dims(y, 1)
],
axis=1),
],
axis=1)
window = transformer(
tf.expand_dims(tf.reshape(input_layer, [-1, win_size, win_size]), 3),
theta, [win_size, win_size])[:, :, :, 0]
window = tf.clip_by_value(window, 0.0, 1.0)
with tf.variable_scope("vae"):
with tf.variable_scope("rsh"):
net = tf.reshape(window, [-1, win_size, win_size])
net = tf.reshape(net, [-1, win_size, win_size, 1])
rec_mean, rec_log_var = encoder(net, is_train, rec_hidden_units,
latent_dim)
with tf.variable_scope("rec_sample"):
standard_normal_sample = tf.random_normal(
[tf.shape(input_layer)[0], latent_dim])
rec_sample = rec_mean + 1 * standard_normal_sample * tf.sqrt(
tf.exp(rec_log_var))
def solar_flux(city,longitude, latitude, I_dif, I_dir, delta):
NB_dwe = len(city)
pi=3.14159
GMT=1.
nb_hour=np.linspace(1,8760,8760)
deg2rad = pi/180. #conversion degré en radian
rad2deg = 180./pi #conversion radian en degré
heure_23 = np.tile(np.linspace(0,23,24),365)
rho = 0.2
beta = 90
# Declinaison solaire, n est le numero de jour
n = np.around(nb_hour / 24 + 0.5)
n=np.r_[1,n]
n=n[0:8760]
declinaison=deg2rad * 23.45 * np.sin((deg2rad) * 360 * (284 + n) / 365.)
#Décalage à l'heure civile
B = ((n-1) * 360 / 365) * deg2rad
E = 229.2 * (0.000075 + 0.001868 * np.cos(B) - 0.032077 * np.sin(B) - 0.014615
* np.cos(2*B) - 0.04089 * np.sin(2*B)) # Equation du temps
Lst = 15. * GMT # GMT-1 (pour la france)
Lloc = longitude # longitude du lieu
#Calcul de l'heure solaire
ts = heure_23 + (4 * (Lst - Lloc) + E) / 60.
#Angle horaire (Temps solaire en angle)
omega = (ts - 12) * 15 * deg2rad
#Position apparente du soleil
###Angle du zénith (theta_z)###
phi = latitude * deg2rad # latitude
thetaz = np.cos(phi) * np.cos(declinaison) * np.cos(omega) + np.sin(phi) * np.sin(declinaison)
theta_z2 = np.arccos(thetaz)
theta_z2[theta_z2>pi/2]=pi/2 #lorsque theta_z>90°, le soleil est couché
theta_z = theta_z2 * rad2deg
### Azimute solaire (gamma_s)###
gamma = (np.cos(theta_z * deg2rad) * np.sin(phi) - np.sin(declinaison)) / (np.sin(theta_z
* deg2rad) *np.cos(phi))
gamma_s = np.sign(omega) * np.arccos(gamma) * rad2deg
### Angle d'incidence pour une surface orienté au sud (theta3_s)###
gamma0 = 0 #azimute de la surface
theta_s = np.cos(theta_z * deg2rad) * np.cos(beta*deg2rad) + np.sin(theta_z *
deg2rad) * np.sin(beta * deg2rad) * np.cos((gamma_s-gamma0)*deg2rad) #a verifier
theta_s[theta_s<0]=0
### Angle d'incidence pour une surface orienté au nord (theta3_n)###
gamma0 = 180 #azimute de la surface
theta_n = np.cos(theta_z * deg2rad) * np.cos(beta*deg2rad) + np.sin(theta_z *
deg2rad) * np.sin(beta * deg2rad) * np.cos((gamma_s-gamma0)*deg2rad)
theta_n[theta_n<0]=0
### Angle d'incidence pour une surface orienté à l'est (theta3_e)###
gamma0 = -90 #azimute de la surface
theta_e = np.cos(theta_z * deg2rad) * np.cos(beta*deg2rad) + np.sin(theta_z *
deg2rad) * np.sin(beta * deg2rad) * np.cos((gamma_s-gamma0)*deg2rad)
theta_e[theta_e<0]=0
### Angle d'incidence pour une surface orienté à l'ouest (theta3_o)###
gamma0 = 90 #azimute de la surface
theta_o = np.cos(theta_z * deg2rad) * np.cos(beta*deg2rad) + np.sin(theta_z *
deg2rad) * np.sin(beta * deg2rad) * np.cos((gamma_s-gamma0)*deg2rad)
theta_o[theta_o<0]=0
I_dir_s = I_dir * theta_s
I_dir_n = I_dir * theta_n
I_dir_e = I_dir * theta_e
I_dir_o = I_dir * theta_o
I_ref = rho * (I_dir * np.cos(theta_z)+I_dif) * (1 + np.cos(beta*deg2rad)) / 2.#Isotropique
I_dif_surf = I_dif * (1 + np.cos(beta * deg2rad)) / 2. #isotropique
I_ref[I_ref<0] = 0
#I_moy = (I_dir_s+I_dir_n+I_dir_e+I_dir_o) / 4. + I_dif_surf + I_ref
I_moy = np.zeros([8760*3600/delta,NB_dwe])
for i in range(0,NB_dwe):
temp = city['R_window_s'].values[i]*I_dir_s + city['R_window_n'].values[i]*I_dir_n + city['R_window_e'].values[i]*I_dir_e +\
city['R_window_w'].values[i]*I_dir_o + I_dif_surf + I_ref
I_moy[:,i] = tr.transformer(temp,3600,delta)
I_dir_n = tr.transformer(I_dir_n,3600,delta)
I_dir_s = tr.transformer(I_dir_s,3600,delta)
I_dir_e = tr.transformer(I_dir_e,3600,delta)
I_dir_o = tr.transformer(I_dir_o,3600,delta)
I_dif_surf = tr.transformer(I_dif_surf,3600,delta)
I_ref = tr.transformer(I_ref,3600,delta)
I_dif = tr.transformer(I_dif,3600,delta)
I_dir = tr.transformer(I_dir,3600,delta)
return I_dir_s, I_dir_n, I_dir_e, I_dir_o, I_ref, I_dif_surf, I_moy, I_dir, I_dif
def __call__(self, images, rnn2_outputs, is_training, reuse):
print('Spatial Transformer Network')
self.images = images
with tf.variable_scope('spatial_trans_2d_affine', reuse=reuse) as vs:
## 1. make the localisation network to [batch, 6] via Flatten and Dense.
# if self.theta_layer.outputs.get_shape().ndims > 2:
# self.theta_layer.outputs = flatten_reshape(self.theta_layer.outputs, 'flatten')
_image = self.images
with slim.arg_scope([slim.conv2d, slim.fully_connected],
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(),
weights_regularizer=slim.l2_regularizer(0.001)):
with slim.arg_scope([slim.batch_norm], decay=0.95, center=True, scale=True,
activation_fn=tf.nn.elu, updates_collections=None, is_training=is_training):
for i in range(len(self.n_filters)):
_image = slim.conv2d(_image, self.n_filters[i], 3, stride=1, padding='SAME',
scope='iloc_conv1_' + str(i))
_image = slim.batch_norm(_image, scope='iloc_bn1' + str(i))
_image = slim.conv2d(_image, self.n_filters[i], 3, stride=2, padding='SAME',
scope='iloc_conv2_' + str(i))
_image = slim.flatten(_image, scope='iloc_flatten')
_image = slim.fully_connected(_image, int(self.hidden/2), scope='iloc_fc_{}'.format(i+1))
_image = slim.batch_norm(_image, scope='iloc_bn_{}'.format(i+1))
_image = slim.fully_connected(_image, int(self.hidden/2), scope='iloc_fc_{}'.format(i+2))
_rnn2 = rnn2_outputs
with slim.arg_scope([slim.fully_connected],
weights_initializer=tf.contrib.layers.xavier_initializer(),
weights_regularizer=slim.l2_regularizer(0.001),
normalizer_fn=slim.batch_norm,
normalizer_params={'is_training': is_training}
):
with slim.arg_scope([slim.batch_norm], decay=0.95, center=True, scale=True,
activation_fn=tf.nn.elu, updates_collections=None, is_training=is_training):
_rnn2 = slim.fully_connected(_rnn2, int(self.hidden/2), scope='rloc_fc1')
_rnn2 = slim.batch_norm(_rnn2, scope='rloc_bn1')
_rnn2 = slim.fully_connected(_rnn2, int(self.hidden/2), scope='rloc_fc2')
self._theta = tf.concat([_image, _rnn2], axis=1) # concat above
#self.theta_layer.outputs
## 2. To initialize the network to the identity transform init.
# 2.1 W
n_in = int(self._theta.get_shape()[-1])
shape = (n_in, 6)
W = tf.get_variable(name='W', initializer=tf.zeros(shape)) #, dtype=D_TYPE)
# 2.2 b
# fixme:
identity = tf.constant(np.array([[self.glimpse_ratio, 0, 0], [0, self.glimpse_ratio, 0]]).astype('float32').flatten())
b = tf.get_variable(name='b', initializer=identity) #, dtype=D_TYPE)
# 2.3 transformation matrix
self.theta = tf.nn.tanh(tf.matmul(self._theta, W) + 1) / 2 + b
theta_constraint = tf.constant([1.0, 0, 1.0, 0, 1.0, 1.0])
self.theta = tf.multiply(self.theta, theta_constraint)
## 3. Spatial Transformer Sampling
# 3.1 transformation
self.transformer = transformer()
self.outputs = self.transformer(self.images, self.theta, out_size=self.out_size, reuse=reuse)
# 3.2 automatically set batch_size and channels
# e.g. [?, 40, 40, ?] --> [64, 40, 40, 1] or [64, 20, 20, 4]/ Hao Dong
#
fixed_batch_size = self.images.get_shape().with_rank_at_least(1)[0]
if fixed_batch_size.value:
batch_size = fixed_batch_size.value
else:
from tensorflow.python.ops import array_ops
batch_size = array_ops.shape(self.images)[0]
size = self.images.get_shape().as_list()
n_channels = self.images.get_shape().as_list()[-1]
self.outputs = tf.reshape(self.outputs, shape=[batch_size, self.out_size[0], self.out_size[1], n_channels])
return self.outputs, self.theta
def run_training(data_train, data_test, queries, query_summary, Tags, concepts,
concept_embeeding, model_save_dir, test_mode):
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
max_f1 = MAX_F1
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
# placeholders
features_holder = tf.placeholder(tf.float32,
shape=(hp.bc * hp.gpu_num, hp.seq_len,
D_FEATURE))
labels_holder = tf.placeholder(tf.float32,
shape=(hp.bc * hp.gpu_num, hp.seq_len,
D_OUTPUT))
scores_src_holder = tf.placeholder(tf.float32,
shape=(hp.bc * hp.gpu_num,
hp.seq_len + CONCEPT_NUM))
scores_tgt_holder = tf.placeholder(tf.float32,
shape=(hp.bc * hp.gpu_num,
hp.seq_len))
txt_emb_holder = tf.placeholder(tf.float32,
shape=(hp.bc * hp.gpu_num, CONCEPT_NUM,
D_TXT_EMB))
img_emb_holder = tf.placeholder(tf.float32,
shape=(hp.bc * hp.gpu_num, CONCEPT_NUM,
D_IMG_EMB))
dropout_holder = tf.placeholder(tf.float32, shape=())
training_holder = tf.placeholder(tf.bool, shape=())
# training operations
lr = noam_scheme(hp.lr_noam, global_step, hp.warmup)
opt_train = tf.train.AdamOptimizer(lr)
# graph building
tower_grads_train = []
logits_list = []
loss_list = []
for gpu_index in range(hp.gpu_num):
with tf.device('/gpu:%d' % gpu_index):
features = features_holder[gpu_index * hp.bc:(gpu_index + 1) *
hp.bc]
labels = labels_holder[gpu_index * hp.bc:(gpu_index + 1) *
hp.bc]
scores_src = scores_src_holder[gpu_index *
hp.bc:(gpu_index + 1) * hp.bc]
scores_tgt = scores_tgt_holder[gpu_index *
hp.bc:(gpu_index + 1) * hp.bc]
txt_emb = txt_emb_holder[gpu_index * hp.bc:(gpu_index + 1) *
hp.bc]
img_emb = img_emb_holder[gpu_index * hp.bc:(gpu_index + 1) *
hp.bc]
# predict concept distribution
logits = transformer(features, labels, scores_src, scores_tgt,
txt_emb, img_emb, dropout_holder,
training_holder,
hp) # 输入的shot在所有concept上的相关性分布
logits_list.append(logits)
loss = tower_loss(logits, labels)
varlist = tf.trainable_variables() # 全部训练
grads_train = opt_train.compute_gradients(loss, varlist)
thresh = GRAD_THRESHOLD # 梯度截断 防止爆炸
grads_train_cap = [(tf.clip_by_value(grad, -thresh,
thresh), var)
for grad, var in grads_train]
tower_grads_train.append(grads_train_cap)
loss_list.append(loss)
grads_t = average_gradients(tower_grads_train)
train_op = opt_train.apply_gradients(grads_t, global_step=global_step)
if test_mode == 1:
train_op = tf.no_op()
# session
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# load model
saver_overall = tf.train.Saver(max_to_keep=100)
if LOAD_CKPT_MODEL:
logging.info(' Ckpt Model Restoring: ' + CKPT_MODEL_PATH)
saver_overall.restore(sess, CKPT_MODEL_PATH)
logging.info(' Ckpt Model Resrtored !')
# train & test preparation
train_scheme = train_scheme_build(data_train, hp.seq_len)
test_scheme, test_vids = test_scheme_build(data_test, hp.seq_len)
epoch_step = math.ceil(len(train_scheme) / (hp.gpu_num * hp.bc))
max_test_step = math.ceil(len(test_scheme) / (hp.gpu_num * hp.bc))
# concept embedding processing
txt_emb_b = []
img_emb_b = []
for c in concepts:
txt_emb_b.append(concept_embeeding[c]['txt'])
img_emb_b.append(concept_embeeding[c]['img'])
txt_emb_b = np.array(txt_emb_b).reshape([1, CONCEPT_NUM, D_TXT_EMB])
img_emb_b = np.array(img_emb_b).reshape([1, CONCEPT_NUM, D_IMG_EMB])
txt_emb_b = np.tile(
txt_emb_b, [hp.gpu_num * hp.bc, 1, 1]) # (bc*gpu_num)*48*d_txt
img_emb_b = np.tile(img_emb_b, [hp.gpu_num * hp.bc, 1, 1])
# begin training
ob_loss = []
timepoint = time.time()
for step in range(hp.maxstep):
features_b, labels_b, scores_b = get_batch_train(
data_train, train_scheme, step, hp.gpu_num, hp.bc, hp.seq_len)
scores_src_b = np.hstack(
(scores_b, np.ones((hp.gpu_num * hp.bc,
CONCEPT_NUM)))) # encoder中开放所有concept节点
scores_tgt_b = scores_b
observe = sess.run([train_op] + loss_list + logits_list +
[global_step, lr],
feed_dict={
features_holder: features_b,
labels_holder: labels_b,
scores_src_holder: scores_src_b,
scores_tgt_holder: scores_tgt_b,
txt_emb_holder: txt_emb_b,
img_emb_holder: img_emb_b,
dropout_holder: hp.dropout,
training_holder: True
})
loss_batch = np.array(observe[1:1 + hp.gpu_num])
ob_loss.append(loss_batch) # 卡0和卡1返回的是来自同一个batch的两部分loss，求平均
# save checkpoint & evaluate
epoch = step / epoch_step
if step % epoch_step == 0 or (step + 1) == hp.maxstep:
if step == 0 and test_mode == 0:
continue
duration = time.time() - timepoint
timepoint = time.time()
loss_array = np.array(ob_loss)
ob_loss.clear()
logging.info(' Step %d: %.3f sec' % (step, duration))
logging.info(' Evaluate: ' + str(step) + ' Epoch: ' +
str(epoch))
logging.info(' Average Loss: ' + str(np.mean(loss_array)) +
' Min Loss: ' + str(np.min(loss_array)) +
' Max Loss: ' + str(np.max(loss_array)))
if not int(epoch) % hp.eval_epoch == 0:
continue # 增大测试间隔
# 按顺序预测测试集中每个视频的每个分段，全部预测后在每个视频内部排序，计算指标
pred_scores = [] # 每个batch输出的预测得分
for test_step in range(max_test_step):
features_b, labels_b, scores_b = get_batch_test(
data_test, test_scheme, test_step, hp.gpu_num, hp.bc,
hp.seq_len)
scores_src_b = np.hstack(
(scores_b, np.ones(
(hp.gpu_num * hp.bc,
CONCEPT_NUM)))) # encoder中开放所有concept节点
scores_tgt_b = scores_b
logits_temp_list = sess.run(logits_list,
feed_dict={
features_holder:
features_b,
labels_holder: labels_b,
scores_src_holder:
scores_src_b,
scores_tgt_holder:
scores_tgt_b,
txt_emb_holder: txt_emb_b,
img_emb_holder: img_emb_b,
dropout_holder: hp.dropout,
training_holder: False
})
for preds in logits_temp_list:
pred_scores.append(preds.reshape((-1, D_OUTPUT)))
p, r, f = evaluation(pred_scores, queries, query_summary, Tags,
test_vids, concepts)
logging.info('Precision: %.3f, Recall: %.3f, F1: %.3f' %
(p, r, f))
random.shuffle(train_scheme)
if test_mode == 1:
return
# save model
if step > MIN_TRAIN_STEPS - PRESTEPS and f >= max_f1:
if f > max_f1:
max_f1 = f
model_path = model_save_dir + 'Model'
elif f == max_f1:
model_path = model_save_dir + 'S%d-E%d-L%.6f-F%.3f' % (
step, epoch, np.mean(loss_array), f)
saver_overall.save(sess, model_path)
logging.info('Model Saved: ' + model_path + '\n')
if step % 3000 == 0 and step > 0:
model_path = model_save_dir + 'S%d-E%d' % (step + PRESTEPS,
epoch)
# saver_overall.save(sess, model_path)
logging.info('Model Saved: ' + str(step + PRESTEPS))
# saving final model
model_path = model_save_dir + 'S%d' % (hp.maxstep + PRESTEPS)
# saver_overall.save(sess, model_path)
logging.info('Model Saved: ' + str(hp.maxstep + PRESTEPS))
# Prefetch the dataset
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
print(dataset)
# Hyper-parameters
NUM_LAYERS = 2
D_MODEL = 256
NUM_HEADS = 16
UNITS = 512
DROPOUT = 0.2
# Create the model
model = transformer.transformer(vocab_size=VOCAB_SIZE,
num_layers=NUM_LAYERS,
units=UNITS,
d_model=D_MODEL,
num_heads=NUM_HEADS,
dropout=DROPOUT)
# Define the loss function using Sparse Categorical Cross Entropy
def loss_function(y_true, y_pred):
y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True,
reduction='none')(
y_true, y_pred)
# Apply the mask correctly
mask = tf.cast(tf.not_equal(y_true, 0), tf.float32)
loss = tf.multiply(loss, mask)
import logs
# Logging
logger = logging.getLogger("__main__")
load_dotenv()
token = os.getenv("TESTTOKEN")
guildid = os.getenv("TESTID")
guildid = int(guildid)
intents = discord.Intents.default()
intents.members = True
bot = Bot(command_prefix=".", intents=intents)
transformer = transformer.transformer("CV_Tennis_Roster.xlsx")
data = data.data("CV_Tennis_Roster.csv")
blacklisttxt = open("blacklist.txt", "r")
blacklist = blacklisttxt.read().split(" ")
blacklisttxt.close()
@bot.event
async def on_ready():
global lockout
lockout = {}
global token_list
token_list = []
list_guild_no_cmd = list(
filter(
# epochs = 50
dataset = tf.data.Dataset.from_tensor_slices(({
'inputs': questions,
'dec_inputs': answers[:, :-1]
}, {
'outputs': answers[:, 1:]
}))
dataset = dataset.cache()
dataset = dataset.shuffle(buffer_size)
dataset = dataset.batch(batch_size)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
tf.keras.backend.clear_session()
model = transformer(vocab_size=vocab_size + 2,
num_layers=num_layers,
units=units,
d_model=d_model,
num_heads=num_heads,
dropout=dropout)
learning_rate = CustomSchedule(d_model=d_model)
optimizer = tf.keras.optimizers.Adam(learning_rate,
beta_1=0.9,
beta_2=0.98,
epsilon=1e-9)
model.compile(optimizer=optimizer, loss=loss_function, metrics=[accuracy])
# train_model(dataset, epochs = 1)
theta_recon = tf.stack([
tf.concat([
tf.stack([1.0 / s, tf.zeros_like(s)], axis=1),
tf.expand_dims(-x / s, 1)
],
axis=1),
tf.concat([
tf.stack([tf.zeros_like(s), 1.0 / s], axis=1),
tf.expand_dims(-y / s, 1)
],
axis=1),
],
axis=1)
window_recon = transformer(tf.expand_dims(input_layer, 3), theta_recon,
[pic_size, pic_size])[:, :, :, 0]
window_recon = tf.reshape(tf.clip_by_value(window_recon, 0.0, 255),
[-1, pic_size, pic_size])
data_type = 2
if data_type == 1:
#MNIST dataset
(x_train, y_train), (x_test, y_test) = mnist.load_data()
if data_type == 2:
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
with tf.variable_scope("st_forward"):
theta = tf.stack([
tf.concat(
[tf.stack([s, tf.zeros_like(s)], axis=1),
tf.expand_dims(x, 1)],
axis=1),
tf.concat(
[tf.stack([tf.zeros_like(s), s], axis=1),
tf.expand_dims(y, 1)],
axis=1),
],
axis=1)
window = transformer(
tf.expand_dims(tf.reshape(input_layer, [-1, pic_size, pic_size]), 3),
theta, [win_size, win_size])[:, :, :, 0]
window = tf.clip_by_value(window, 0.0, 1.0)
with tf.variable_scope("st_bypart"):
theta_recon = tf.stack([
tf.concat([
tf.stack([1.0 / s, tf.zeros_like(s)], axis=1),
tf.expand_dims(-x / s, 1)
],
axis=1),
tf.concat([
tf.stack([tf.zeros_like(s), 1.0 / s], axis=1),
tf.expand_dims(-y / s, 1)
],
axis=1),
total_rating += rated ratings_count += 1 if(rated > 3): rated = 1 else: rated = -1 trainer.reward(recommended_item, context, rated) c += 1 total = 10000 if( c > total): break if c % 100 == 0: print '\n\n ' + "Evaluated %d/%d lines." % (c, total) print "Avg. Recommended Rating = %f" % (float(total_rating) / ratings_count) avg_ratings.append(float(total_rating) / ratings_count) print '' print '\n\n ' + "Evaluated %d/%d lines." % (c, total) print "Avg. Recommended Rating = %f" % (float(total_rating) / ratings_count) avg_ratings.append(float(total_rating) / ratings_count) print avg_ratings trainer = ucb() logger = logservice() db = memoryDB.memoryDB() transformer = transformer.transformer(db) signal.signal(signal.SIGINT, signal_handler) run_test(logger, trainer,transformer)
def call(self, inputs):
return transformer(inputs[0], inputs[1])
def main(arv):
def plot_results(model_name="vae_mnist", index=0):
import os
import matplotlib.pyplot as plt
if not os.path.exists(model_name):
os.makedirs(model_name)
filename = os.path.join(model_name, "digits_over_latent.png")
# display a 30x30 2D manifold of digits
n = 10
digit_size = 32
figure = np.zeros((digit_size * n, digit_size * n, 3))
# linearly spaced coordinates corresponding to the 2D plot
# of digit classes in the latent space
grid_x = np.linspace(-4 * 4, 4 * 4, n)
grid_y = np.linspace(-4 * 4, 4 * 4, n)[::-1]
for i, yi in enumerate(grid_y):
for j, xi in enumerate(grid_x):
z_sample = np.zeros([1, latent_dim])
z_sample[0, 0] = xi
z_sample[0, 1] = yi
#z_sample = np.array([[xi, yi]])
x_decoded = sess.run(rec,
feed_dict={
shf_sample: z_sample,
is_train: False
})
#x_decoded = decoder.predict(z_sample)
digit = x_decoded[0] #.reshape(digit_size, digit_size,3)
figure[i * digit_size:(i + 1) * digit_size,
j * digit_size:(j + 1) * digit_size, :] = digit
plt.figure(figsize=(10, 10))
start_range = digit_size // 2
end_range = n * digit_size + start_range + 1
pixel_range = np.arange(start_range, end_range, digit_size)
sample_range_x = np.round(grid_x, 1)
sample_range_y = np.round(grid_y, 1)
plt.xticks(pixel_range, sample_range_x)
plt.yticks(pixel_range, sample_range_y)
plt.xlabel("z[0]")
plt.ylabel("z[1]")
plt.imshow(figure, cmap='Greys')
plt.savefig(filename)
filename = os.path.join(model_name, "compair%02d.png" % (index))
shff = all_shuff[142:260]
nrml = all_images[142:260]
num_rows = 5
num_cols = 3
num_images = num_rows * num_cols
plt.figure(figsize=(3 * 2 * num_cols, 2 * num_rows))
for i in range(num_images):
j = 3 * i
any_image = shff[j]
any_image0 = nrml[j, :, :, 0]
any_image1 = nrml[j, :, :, 1]
any_image2 = nrml[j, :, :, 2]
x_rolled,x_decoded,x_shuf,error,x_encoded= sess.run([window,rec2,rec,meansq,shf_mean],\
feed_dict={input_shuff:[any_image],
input_digit0:[any_image0],
input_digit1:[any_image1],
input_digit2:[any_image2],
output_true:[any_image],
is_train:False})
x_tt = nrml[j] #.reshape(pic_size, pic_size)
x_dec = x_decoded[0] #.reshape(pic_size, pic_size)
#print(x_encoded.shape)
sar = [str(int(a * 10) / 10) for a in x_encoded[0]]
if len(sar) > 5:
sar = sar[0:5]
ax = plt.subplot(num_rows, 3 * num_cols, 3 * i + 1)
plt.imshow(x_rolled[0], cmap='Greys')
#plt.xlabel(error)
plt.xlabel('z = [' + ", ".join(sar) + ']')
plt.xticks([])
plt.yticks([])
ax = plt.subplot(num_rows, 3 * num_cols, 3 * i + 2)
plt.imshow(x_dec, cmap='Greys')
plt.xticks([])
plt.yticks([])
ax = plt.subplot(num_rows, 3 * num_cols, 3 * i + 3)
plt.imshow(x_shuf[0], cmap='Greys')
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.savefig(filename)
#plt.figure()
#ax = plt.subplot(1, 2, 1)
#plt.imshow(x_tt , cmap='Greys')
#plt.xticks([])
#plt.yticks([])
#ax = plt.subplot(1, 2, 2)
#plt.imshow(x_dec, cmap='Greys')
#plt.xticks([])
#plt.yticks([])
filename = os.path.join(model_name, "style.png")
plt.figure(figsize=(3 * 2 * num_cols, 2 * num_rows))
glob_a, local_a = sess.run([nrm_sample,shf_sample],\
feed_dict={input_shuff:[shff[0]],
input_digit0:[nrml[0,:,:,0]],
input_digit1:[nrml[0,:,:,1]],
input_digit2:[nrml[0,:,:,2]],
output_true:[shff[0]],
is_train:False})
for i in range(num_images):
j = 3 * i
any_image = shff[j]
any_image0 = nrml[j, :, :, 0]
any_image1 = nrml[j, :, :, 1]
any_image2 = nrml[j, :, :, 2]
origi,hig,wei,rol,xx,yy, glob_b, local_b = sess.run([window,h,w,r,x,y, nrm_sample,shf_sample],\
feed_dict={input_shuff:[any_image],
input_digit0:[any_image0],
input_digit1:[any_image1],
input_digit2:[any_image2],
output_true:[any_image],
is_train:False})
ab_img = sess.run([rec2],
feed_dict={
nrm_sample: glob_a,
shf_sample: local_b,
is_train: False
})
ba_img = sess.run([rec2],
feed_dict={
nrm_sample: glob_b,
shf_sample: local_a,
is_train: False
})
ax = plt.subplot(num_rows, 3 * num_cols, 3 * i + 1)
plt.imshow(origi[0], cmap='Greys')
plt.xlabel('(%.2f, %.2f) %.2f (%.2f, %.2f))' %
(hig, wei, rol, xx, yy))
plt.xticks([])
plt.yticks([])
ax = plt.subplot(num_rows, 3 * num_cols, 3 * i + 2)
plt.imshow(ab_img[0][0], cmap='Greys')
plt.xticks([])
plt.yticks([])
ax = plt.subplot(num_rows, 3 * num_cols, 3 * i + 3)
plt.imshow(ba_img[0][0], cmap='Greys')
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.savefig(filename)
plt.close('all')
#plt.show()
data_type = 4
if data_type == 3:
train = sio.loadmat('32x32/train_32x32.mat')
img_train = np.array(train['X'])
#y_test = np.array(train['y'])
color_channel = 3
tot_images = 60000 # total number of images
elif data_type == 4:
current_dir = os.getcwd()
file_dir = os.path.join(current_dir, '32x32/A')
images = []
for each in os.listdir(file_dir):
img = cv2.imread(os.path.join(file_dir, each))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
images.append(np.array(img))
img_train = np.array(images) #/255
#print(img_train.shape)
#win_size = 32
color_channel = 3
tot_images = 1500 # total number of images
#exit()
# Deciding how many nodes wach layer should have
pre_transf_units = (8, 16, 32)
rec_hidden_units = (32, 16) #(512, 256)
vae_generative_units = (16, 32) #(256, 512)
hidden_units = 64
latent_dim = 40
pic_size = 32
nnn = img_train.shape
if data_type == 3:
all_images = np.zeros((nnn[3], pic_size, pic_size, color_channel))
all_shuff = np.zeros((nnn[3], pic_size, pic_size, color_channel))
for i in range(nnn[3]):
all_images[i] = img_train[:, :, :, i].astype('float32') / 255.0
aaa = np.zeros((32, 32, 3))
for i1 in range(4):
for j1 in range(4):
px = (3 * i1 + 1) % 4
py = (3 * j1 + 2) % 4
aaa[i1 * 8 + 0:i1 * 8 + 8, j1 * 8 + 0:j1 * 8 +
8, :] = all_images[i, px * 8 + 0:px * 8 + 8,
py * 8 + 0:py * 8 + 8, :]
all_shuff[i] = aaa
elif data_type == 4:
all_images = np.zeros((nnn[0], pic_size, pic_size, color_channel))
all_shuff = np.zeros((nnn[0], pic_size, pic_size, color_channel))
for i in range(nnn[0]):
all_images[i] = img_train[i, :, :, :].astype('float32') / 255.0
aaa = np.zeros((32, 32, 3))
for i1 in range(4):
for j1 in range(4):
px = (3 * i1 + 1) % 4
py = (3 * j1 + 2) % 4
aaa[i1 * 8 + 0:i1 * 8 + 8, j1 * 8 + 0:j1 * 8 +
8, :] = all_images[i, px * 8 + 0:px * 8 + 8,
py * 8 + 0:py * 8 + 8, :]
all_shuff[i] = aaa
#print("all = ",all_images.shape)
#print("shuf = ",all_shuff.shape)
#print(all_images[0])
#plt.imshow(all_images[1])
#plt.show()
#exit()
with tf.variable_scope("input_layer"):
input_shuff = tf.placeholder(
'float32',
shape=[None, pic_size, pic_size, color_channel],
name="input_shuff")
input_digit0 = tf.placeholder('float32',
shape=[None, pic_size, pic_size],
name="input_normal0")
input_digit1 = tf.placeholder('float32',
shape=[None, pic_size, pic_size],
name="input_normal1")
input_digit2 = tf.placeholder('float32',
shape=[None, pic_size, pic_size],
name="input_normal2")
is_train = tf.placeholder(tf.bool, name='is_train')
print("input", input_digit0.shape)
input_digit = tf.concat([
tf.expand_dims(input_digit0, axis=3),
tf.expand_dims(input_digit1, axis=3),
tf.expand_dims(input_digit2, axis=3)
],
axis=3)
#print("input",input_digit.shape)
with tf.variable_scope("Pre-transofrm"):
net = tf.reshape(input_digit, shape=[-1, pic_size, pic_size, 3])
with slim.arg_scope(
[slim.conv2d, slim.fully_connected],
normalizer_fn=slim.batch_norm,
activation_fn=lambda x: tf.nn.leaky_relu(x, alpha=0.1),
normalizer_params={"is_training": is_train}):
for i in range(len(pre_transf_units)):
net = slim.conv2d(net,
pre_transf_units[i], [3, 3],
scope="conv%d" % (i * 2 + 1))
net = slim.conv2d(net,
pre_transf_units[i], [3, 3],
stride=2,
scope="conv%d" % (i * 2 + 2))
net = slim.flatten(net)
rnn_out = slim.fully_connected(net, 64)
with tf.variable_scope("scale"):
with tf.variable_scope("mean"):
with tf.variable_scope("hidden") as scope:
hidden = slim.fully_connected(rnn_out,
hidden_units * 2,
scope=scope)
with tf.variable_scope("output") as scope:
scale_mean = slim.fully_connected(hidden,
2,
activation_fn=None,
scope=scope)
scale = tf.nn.sigmoid(scale_mean) * 0.25 + .85
h, w = scale[:, 0], scale[:, 1]
#print(h.shape)
with tf.variable_scope("shift"):
with tf.variable_scope("mean"):
with tf.variable_scope("hidden") as scope:
hidden = slim.fully_connected(rnn_out,
hidden_units * 2,
scope=scope)
with tf.variable_scope("output") as scope:
shift_mean = slim.fully_connected(hidden,
2,
activation_fn=None,
scope=scope)
shift = tf.nn.tanh(shift_mean) * 0.35
x, y = shift[:, 0], shift[:, 1]
with tf.variable_scope("roll"):
with tf.variable_scope("mean"):
with tf.variable_scope("hidden") as scope:
hidden = slim.fully_connected(rnn_out,
hidden_units,
scope=scope)
with tf.variable_scope("output") as scope:
roll_mean = slim.fully_connected(hidden,
1,
activation_fn=None,
scope=scope)
roll = tf.nn.tanh(roll_mean)
r = roll[:, 0]
with tf.variable_scope("Transformer"):
theta = tf.stack([
tf.concat([
tf.stack(
[h * tf.math.cos(r), tf.math.sin(r)], axis=1),
tf.expand_dims(x, 1)
],
axis=1),
tf.concat([
tf.stack([-tf.math.sin(r), w * tf.math.cos(r)], axis=1),
tf.expand_dims(y, 1)
],
axis=1),
],
axis=1)
#print(input_digit.shape)
#print(tf.reshape(input_digit[:,:,:,0], [-1, pic_size, pic_size,3]).shape)
window0 = transformer(
tf.expand_dims(tf.reshape(input_digit0, [-1, pic_size, pic_size]),
3), theta, [pic_size, pic_size])[:, :, :, 0]
window1 = transformer(
tf.expand_dims(tf.reshape(input_digit1, [-1, pic_size, pic_size]),
3), theta, [pic_size, pic_size])[:, :, :, 0]
window2 = transformer(
tf.expand_dims(tf.reshape(input_digit2, [-1, pic_size, pic_size]),
3), theta, [pic_size, pic_size])[:, :, :, 0]
window = tf.concat([
tf.expand_dims(window0, 3),
tf.expand_dims(window1, 3),
tf.expand_dims(window2, 3)
],
axis=3)
window = tf.clip_by_value(window, 0.0, 1.0)
# VAE model
with tf.variable_scope("VAE_disent"):
with tf.variable_scope("shuff"):
net = input_shuff #tf.reshape(net,[-1,win_size,win_size,1])
with tf.variable_scope("encoder"):
shf_mean, shf_log_var = encoder(net, is_train,
rec_hidden_units, latent_dim)
with tf.variable_scope("sampling"):
shf_standard_sample = tf.random_normal(
[tf.shape(input_shuff)[0], latent_dim])
shf_sample = shf_mean + 1 * shf_standard_sample * tf.sqrt(
tf.exp(shf_log_var))
with tf.variable_scope("decode"):
rec = decoder(shf_sample, is_train, [3, 8, 8, 16],
vae_generative_units, latent_dim)
with tf.variable_scope("normal"):
net2 = window
with tf.variable_scope("encoder"):
nrm_mean, nrm_log_var = encoder(net2, is_train,
rec_hidden_units,
latent_dim * 2)
with tf.variable_scope("sampling"):
nrm_standard_sample = tf.random_normal(
[tf.shape(input_digit)[0], latent_dim * 2])
nrm_sample = nrm_mean + 1 * nrm_standard_sample * tf.sqrt(
tf.exp(nrm_log_var))
pack_sample = tf.concat([nrm_sample, shf_sample], axis=1)
with tf.variable_scope("decode"):
rec2 = decoder(pack_sample, is_train, [3, 8, 8, 16],
vae_generative_units, latent_dim * 2)
#rec = tf.reshape(rec,[-1,win_size*win_size*color_channel])
# output_true shall have the original image for error calculations
output_true = tf.placeholder('float32',
[None, pic_size, pic_size, color_channel],
name="Truth")
def gaussian_log_likelihood(x, mean, var, eps=1e-8):
# compute log P(x) for diagonal Guassian
# -1/2 log( (2pi)^k sig_1 * sig_2 * ... * sig_k ) - sum_i 1/2sig_i^2 (x_i - m_i)^2
bb = tf.square(x - mean)
bb /= (var + eps)
return -0.5 * tf.reduce_sum(tf.log(2. * np.pi * var + eps) + bb,
axis=1)
with tf.variable_scope("loss_function"):
# define our cost function
with tf.variable_scope("recon_loss"):
meansq = tf.reduce_mean(tf.square(rec - output_true))
meansq *= pic_size * pic_size
meansq2 = tf.reduce_mean(
tf.square(rec2 - window) *
tf.to_float(tf.math.greater(window, 1e-3)))
meansq2 *= pic_size * pic_size
binarcs = -tf.reduce_mean(output_true * tf.log(rec + 10e-10) +
(1.0 - output_true) *
tf.log(1.0 - rec + 10e-10))
with tf.variable_scope("kl_loss"):
vae_kl = 0.5 * tf.reduce_sum(
0.0 - shf_log_var - 1.0 + tf.exp(shf_log_var) +
tf.square(shf_mean - 0.0), 1)
vae_kl = tf.reduce_mean(vae_kl)
vae_kl2 = tf.reduce_mean(
- gaussian_log_likelihood(shf_sample, 0.0, 1.0, eps=0.0) \
+ gaussian_log_likelihood(shf_sample, shf_mean, (tf.exp(shf_log_var)) )
)
vae_kl3 = 0.5 * tf.reduce_sum(
0.0 - nrm_log_var - 1.0 + tf.exp(nrm_log_var) +
tf.square(nrm_mean - 0.0), 1)
vae_kl3 = tf.reduce_mean(vae_kl3)
vae_loss = meansq * .25 + meansq2 + vae_kl + vae_kl3
#vae_loss = binarcs*0.0001-tf.reduce_mean(tf.square(window))
learn_rate = 0.001 # how fast the model should learn
slimopt = slim.learning.create_train_op(vae_loss,
tf.train.AdamOptimizer(learn_rate))
# initialising stuff and starting the session
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
saver = tf.train.Saver()
writer = tf.summary.FileWriter("demo/3")
writer.add_graph(sess.graph)
summary = tf.summary.merge([
tf.summary.scalar("loss_total", vae_loss),
tf.summary.scalar("mean_sq", meansq),
tf.summary.scalar("binary_cross", binarcs),
tf.summary.image("recon", tf.reshape(rec2,
[-1, pic_size, pic_size, 3])),
tf.summary.image("recon_shuf",
tf.reshape(rec, [-1, pic_size, pic_size, 3])),
tf.summary.image("original",
tf.reshape(input_shuff, [-1, pic_size, pic_size, 3])),
#tf.summary.image("window", tf.reshape(, [-1, win_size, win_size, 1])),
])
# defining batch size, number of epochs and learning rate
batch_size = 250 # how many images to use together for training
hm_epochs = 5000 # how many times to go through the entire dataset
# running the model for a 1000 epochs taking 100 images in batches
# total improvement is printed out after each epoch
kl = 0
for epoch in range(hm_epochs):
epoch_loss = 0 # initializing error as 0
for i in range(int(tot_images / batch_size)):
epoch_x = all_shuff[i * batch_size:(i + 1) * batch_size]
epoch_xx = all_images[i * batch_size:(i + 1) * batch_size]
#print("epoch_x",epoch_x.shape)
epoch_x0 = epoch_xx[:, :, :, 0]
epoch_x1 = epoch_xx[:, :, :, 1]
epoch_x2 = epoch_xx[:, :, :, 2]
_,c = sess.run([slimopt,vae_loss],feed_dict={input_shuff:epoch_x,\
input_digit0:epoch_x0,
input_digit1:epoch_x1,
input_digit2:epoch_x2,
output_true:epoch_x,
is_train:True})
epoch_loss += c
epoch_x = all_shuff[110:130]
epoch_xx = all_images[110:130]
epoch_x0 = epoch_xx[:, :, :, 0]
epoch_x1 = epoch_xx[:, :, :, 1]
epoch_x2 = epoch_xx[:, :, :, 2]
summ = sess.run(summary, feed_dict={input_shuff: epoch_x, \
input_digit0:epoch_x0,
input_digit1:epoch_x1,
input_digit2:epoch_x2,
output_true: epoch_x,
is_train:False})
writer.add_summary(summ, epoch)
if epoch % 100 == 0:
print('Epoch', epoch, '/', hm_epochs, 'loss:', epoch_loss)
if epoch % 100 == 0:
plot_results(model_name="vae_test", index=epoch)
print('Epoch', epoch + 1, '/', hm_epochs, 'loss:', epoch_loss)
plot_results(model_name="vae_test")
def model_fn(self, features, mode):
# todo can estimators handle dropout for us or do we need to do it on our own?
hparams = self.hparams(mode)
with tf.variable_scope("LISA", reuse=tf.AUTO_REUSE):
batch_shape = tf.shape(features)
batch_size = batch_shape[0]
batch_seq_len = batch_shape[1]
layer_config = self.model_config['layers']
sa_hidden_size = layer_config['head_dim'] * layer_config[
'num_heads']
feats = {
f: features[:, :, idx]
for f, idx in self.feature_idx_map.items()
}
# todo this assumes that word_type is always passed in
words = feats['word_type']
# for masking out padding tokens
tokens_to_keep = tf.where(tf.equal(words, constants.PAD_VALUE),
tf.zeros([batch_size, batch_seq_len]),
tf.ones([batch_size, batch_seq_len]))
# Extract named features from monolithic "features" input
feats = {
f: tf.multiply(tf.cast(tokens_to_keep, tf.int32), v)
for f, v in feats.items()
}
# Extract named labels from monolithic "features" input, and mask them
# todo fix masking -- is it even necessary?
labels = {}
for l, idx in self.label_idx_map.items():
these_labels = features[:, :, idx[0]:idx[1]] if idx[
1] != -1 else features[:, :, idx[0]:]
these_labels_masked = tf.multiply(
these_labels,
tf.cast(tf.expand_dims(tokens_to_keep, -1), tf.int32))
# check if we need to mask another dimension
if idx[1] == -1:
last_dim = tf.shape(these_labels)[2]
this_mask = tf.where(
tf.equal(these_labels_masked, constants.PAD_VALUE),
tf.zeros([batch_size, batch_seq_len, last_dim],
dtype=tf.int32),
tf.ones([batch_size, batch_seq_len, last_dim],
dtype=tf.int32))
these_labels_masked = tf.multiply(these_labels_masked,
this_mask)
else:
these_labels_masked = tf.squeeze(these_labels_masked, -1)
labels[l] = these_labels_masked
# load transition parameters
transition_stats = util.load_transition_params(
self.task_config, self.vocab)
# Create embeddings tables, loading pre-trained if specified
embeddings = {}
for embedding_name, embedding_map in self.model_config[
'embeddings'].items():
embedding_dim = embedding_map['embedding_dim']
if 'pretrained_embeddings' in embedding_map:
input_pretrained_embeddings = embedding_map[
'pretrained_embeddings']
include_oov = True
embedding_table = self.get_embedding_table(
embedding_name,
embedding_dim,
include_oov,
pretrained_fname=input_pretrained_embeddings)
else:
num_embeddings = self.vocab.vocab_names_sizes[
embedding_name]
include_oov = self.vocab.oovs[embedding_name]
embedding_table = self.get_embedding_table(
embedding_name,
embedding_dim,
include_oov,
num_embeddings=num_embeddings)
embeddings[embedding_name] = embedding_table
tf.logging.log(tf.logging.INFO,
"Created embeddings for '%s'." % embedding_name)
# Set up model inputs
inputs_list = []
for input_name in self.model_config['inputs']:
input_values = feats[input_name]
input_embedding_lookup = tf.nn.embedding_lookup(
embeddings[input_name], input_values)
inputs_list.append(input_embedding_lookup)
tf.logging.log(tf.logging.INFO,
"Added %s to inputs list." % input_name)
current_input = tf.concat(inputs_list, axis=2)
current_input = tf.nn.dropout(current_input, hparams.input_dropout)
with tf.variable_scope('project_input'):
current_input = nn_utils.MLP(current_input,
sa_hidden_size,
n_splits=1)
predictions = {}
eval_metric_ops = {}
export_outputs = {}
loss = tf.constant(0.)
items_to_log = {}
num_layers = max(self.task_config.keys()) + 1
tf.logging.log(
tf.logging.INFO,
"Creating transformer model with %d layers" % num_layers)
with tf.variable_scope('transformer'):
current_input = transformer.add_timing_signal_1d(current_input)
for i in range(num_layers):
with tf.variable_scope('layer%d' % i):
special_attn = []
special_values = []
if i in self.attention_config:
this_layer_attn_config = self.attention_config[i]
if 'attention_fns' in this_layer_attn_config:
for attn_fn, attn_fn_map in this_layer_attn_config[
'attention_fns'].items():
attention_fn_params = attention_fns.get_params(
mode, attn_fn_map, predictions, feats,
labels)
this_special_attn = attention_fns.dispatch(
attn_fn_map['name'])(
**attention_fn_params)
special_attn.append(this_special_attn)
if 'value_fns' in this_layer_attn_config:
for value_fn, value_fn_map in this_layer_attn_config[
'value_fns'].items():
value_fn_params = value_fns.get_params(
mode, value_fn_map, predictions, feats,
labels, embeddings)
this_special_values = value_fns.dispatch(
value_fn_map['name'])(
**value_fn_params)
special_values.append(this_special_values)
current_input = transformer.transformer(
current_input, tokens_to_keep,
layer_config['head_dim'],
layer_config['num_heads'], hparams.attn_dropout,
hparams.ff_dropout, hparams.prepost_dropout,
layer_config['ff_hidden_size'], special_attn,
special_values)
if i in self.task_config:
# if normalization is done in layer_preprocess, then it should also be done
# on the output, since the output can grow very large, being the sum of
# a whole stack of unnormalized layer outputs.
current_input = nn_utils.layer_norm(current_input)
# todo test a list of tasks for each layer
for task, task_map in self.task_config[i].items():
task_labels = labels[task]
task_vocab_size = self.vocab.vocab_names_sizes[
task] if task in self.vocab.vocab_names_sizes else -1
# Set up CRF / Viterbi transition params if specified
with tf.variable_scope(
"crf"
): # to share parameters, change scope here
# transition_stats_file = task_map['transition_stats'] if 'transition_stats' in task_map else None
task_transition_stats = transition_stats[
task] if task in transition_stats else None
# create transition parameters if training or decoding with crf/viterbi
task_crf = 'crf' in task_map and task_map[
'crf']
task_viterbi_decode = task_crf or 'viterbi' in task_map and task_map[
'viterbi']
transition_params = None
if task_viterbi_decode or task_crf:
transition_params = tf.get_variable(
"transitions",
[task_vocab_size, task_vocab_size],
initializer=tf.
constant_initializer(
task_transition_stats),
trainable=task_crf)
train_or_decode_str = "training" if task_crf else "decoding"
tf.logging.log(
tf.logging.INFO,
"Created transition params for %s %s"
% (train_or_decode_str, task))
output_fn_params = output_fns.get_params(
mode, self.model_config,
task_map['output_fn'], predictions, feats,
labels, current_input, task_labels,
task_vocab_size,
self.vocab.joint_label_lookup_maps,
tokens_to_keep, transition_params, hparams)
task_outputs = output_fns.dispatch(
task_map['output_fn']['name'])(
**output_fn_params)
# want task_outputs to have:
# - predictions
# - loss
# - scores
# - probabilities
predictions[task] = task_outputs
# do the evaluation
for eval_name, eval_map in task_map[
'eval_fns'].items():
eval_fn_params = evaluation_fns.get_params(
task_outputs, eval_map, predictions,
feats, labels, task_labels,
self.vocab.reverse_maps,
tokens_to_keep)
eval_result = evaluation_fns.dispatch(
eval_map['name'])(**eval_fn_params)
eval_metric_ops[eval_name] = eval_result
# get the individual task loss and apply penalty
this_task_loss = task_outputs[
'loss'] * task_map['penalty']
# log this task's loss
items_to_log['%s_loss' % task] = this_task_loss
# add this loss to the overall loss being minimized
loss += this_task_loss
# set up moving average variables
assign_moving_averages_dep = tf.no_op()
if hparams.moving_average_decay > 0.:
moving_averager = tf.train.ExponentialMovingAverage(
hparams.moving_average_decay,
zero_debias=True,
num_updates=tf.train.get_global_step())
moving_average_op = moving_averager.apply(
train_utils.get_vars_for_moving_average(
hparams.average_norms))
# tf.logging.log(tf.logging.INFO,
# "Using moving average for variables: %s" % str([v.name for v in tf.trainable_variables()])
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS,
moving_average_op)
# use moving averages of variables if evaluating
assign_moving_averages_dep = tf.cond(
tf.equal(mode, ModeKeys.TRAIN), lambda: tf.no_op(), lambda:
nn_utils.set_vars_to_moving_average(moving_averager))
with tf.control_dependencies([assign_moving_averages_dep]):
items_to_log['loss'] = loss
# get learning rate w/ decay
this_step_lr = train_utils.learning_rate(
hparams, tf.train.get_global_step())
items_to_log['lr'] = this_step_lr
# optimizer = tf.contrib.opt.NadamOptimizer(learning_rate=this_step_lr, beta1=hparams.beta1,
# beta2=hparams.beta2, epsilon=hparams.epsilon)
optimizer = LazyAdamOptimizer(
learning_rate=this_step_lr,
beta1=hparams.beta1,
beta2=hparams.beta2,
epsilon=hparams.epsilon,
use_nesterov=hparams.use_nesterov)
gradients, variables = zip(*optimizer.compute_gradients(loss))
gradients, _ = tf.clip_by_global_norm(
gradients, hparams.gradient_clip_norm)
train_op = optimizer.apply_gradients(
zip(gradients, variables),
global_step=tf.train.get_global_step())
# export_outputs = {'predict_output': tf.estimator.export.PredictOutput({'scores': scores, 'preds': preds})}
logging_hook = tf.train.LoggingTensorHook(items_to_log,
every_n_iter=20)
# need to flatten the dict of predictions to make Estimator happy
flat_predictions = {
"%s_%s" % (k1, k2): v2
for k1, v1 in predictions.items() for k2, v2 in v1.items()
}
export_outputs = {
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
tf.estimator.export.PredictOutput(flat_predictions)
}
tf.logging.log(
tf.logging.INFO,
"Created model with %d trainable parameters" %
tf_utils.get_num_trainable_parameters())
return tf.estimator.EstimatorSpec(
mode,
flat_predictions,
loss,
train_op,
eval_metric_ops,
training_hooks=[logging_hook],
export_outputs=export_outputs)
class smartestClass: param1 = 0 param2 = 0 param3 = "" param4 = 0 def __init__(self, param1, param2, param3, param4): self.param1 = param1 self.param2 = param2 self.param3 = param3 self.param4 = param4 def getFeatures(self): feats = dict() feats['param1'] = self.param1 feats['param2'] = self.param2 feats['param3'] = self.param3 feats['param4'] = str(self.param4) return feats t = transformer.transformer(None) c = t.transform(naiveClass(1,2,"Male")) print "Naive Class = \t\t" + str(c) db = memoryDB.memoryDB() t = transformer.transformer(db) c = t.transform(smartestClass(2.,2.,"Male",2)) c = t.transform(smartestClass(2.,2.,"Female",3)) c = t.transform(smartestClass(1.,1.,"Female",3)) print "Smart Class = \t\t" + str(c)
initial = np.array([[0.7, -0.7, 0], [0.7, 0.7, 0]]) # rotation
initial = np.array([[2.0, 0, 0], [0, 2.0, 0]]) # zooming out
initial = initial.astype('float32')
initial = initial.flatten()
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
# the first layer
h_fc_loc1 = tf.nn.tanh(tf.matmul(x, W_fc_loc1) + b_fc_loc1)
keep_prob = tf.placeholder(tf.float32)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
# the second layer
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)
out_size = (40, 40)
h_trans = transformer(x_tensor, h_fc_loc2, out_size)
""" the first convolutional layer """
filter_size = 3
n_filters_1 = 16
W_conv1 = weight_variable([filter_size, filter_size, 1, n_filters_1]) # (3, 3, 1, 16)
b_conv1 = bias_variable([n_filters_1]) # (16)
# %% Now we can build a graph which does the first layer of convolution:
# we define our stride as batch x height x width x channels
# instead of pooling, we use strides of 2 and more layers
# with smaller filters.
h_conv1 = tf.nn.relu(
tf.nn.conv2d(input=h_trans,
filter=W_conv1,
strides=[1, 2, 2, 1],
