def prepare_all():
count_questions_len()
gen_answers_id_mapping()
gen_question_word_set()
gen_word_vector_mapping()
# gen_word_vector_mapping_glove()
gen_question_word_id_vec()
gen_data('train')
gen_data('val')
def _train(self, network, iterations):
bob_decrypt_error, eve_decrypt_error, alice_decrypt_error = 0.0 , 0.0 , 0.0
bs = self.batch_size
# Train Eve for two minibatches to give it a slight computational edge
if network == 'eve':
bs *= 2
for i in range(iterations):
msg_in_val, secret_val, key_val= gen_data(n=bs, msg_len=self.msg_len, secret_len=self.secret_len, key_len=self.random_seed_len)
if network == 'bob':
_, decrypt_err_alice, decrypt_err,alice,bob,eve = self.sess.run([self.bob_optimizer, self.decrypt_err_alice, self.decrypt_err_bob,self.alice_output,self.bob_output,self.eve_output],
feed_dict={self.msg: msg_in_val, self.secret: secret_val, self.seed: key_val})
# bob_decrypt_error = min(bob_decrypt_error, decrypt_err)
bob_decrypt_error = bob_decrypt_error + decrypt_err
alice_decrypt_error = alice_decrypt_error + decrypt_err_alice
elif network == 'eve':
_, decrypt_err,alice,bob,eve = self.sess.run([self.eve_optimizer, self.decrypt_err_eve,self.alice_output,self.bob_output,self.eve_output],
feed_dict={self.msg: msg_in_val, self.secret: secret_val, self.seed: key_val})
#eve_decrypt_error = min(eve_decrypt_error, decrypt_err)
eve_decrypt_error = eve_decrypt_error + decrypt_err
print msg_in_val[0], secret_val[0], alice[0], bob[0], eve[0]
return bob_decrypt_error/iterations, eve_decrypt_error/iterations , alice_decrypt_error/iterations
def _train(self, network, iterations):
bob_decrypt_error, eve_decrypt_error = 1., 1.
bs = self.batch_size
# Train Eve for two minibatches to give it a slight computational edge
if network == 'eve':
bs *= 2
for i in range(iterations):
msg_in_val, key_val = gen_data(n=bs)
if network == 'bob':
_, decrypt_err = self.sess.run(
[self.bob_optimizer, self.decrypt_err_bob],
feed_dict={
self.msg: msg_in_val,
self.key: key_val
})
bob_decrypt_error = min(bob_decrypt_error, decrypt_err)
elif network == 'eve':
_, decrypt_err = self.sess.run(
[self.eve_optimizer, self.decrypt_err_eve],
feed_dict={
self.msg: msg_in_val,
self.key: key_val
})
eve_decrypt_error = min(eve_decrypt_error, decrypt_err)
return bob_decrypt_error, eve_decrypt_error
def test(predict, data, language, model_name, seq_len,
long_letter_reverse_mapping, transliteration, trans_to_index,
char_to_index, index_to_trans, index_to_char):
sentence_in = ""
sentence_real = ""
sentence_out = ""
p = 0
turned = False
while not turned:
x, y, non_valids, p, turned = utils.gen_data(p,
seq_len,
1,
data,
transliteration,
trans_to_index,
char_to_index,
is_train=False)
sentence_in += utils.one_hot_matrix_to_sentence(
x, index_to_trans).replace(u'\u2001', '').replace(u'\u2000', '')
real_without_signs = utils.one_hot_matrix_to_sentence(
y, index_to_char).replace(u'\u2001', '').replace(u'\u2000', '')
ind = 0
real = ""
for c in real_without_signs:
if c == '#' and ind < len(non_valids):
real += non_valids[ind]
ind += 1
else:
real += c
sentence_real += real
guess = utils.one_hot_matrix_to_sentence(
predict(x), index_to_char).replace(u'\u2001',
'').replace(u'\u2000', '')
ind = 0
final_guess = ""
for c in guess:
if c == '#' and ind < len(non_valids):
final_guess += non_valids[ind]
ind += 1
else:
final_guess += c
sentence_out += final_guess
print(str(100.0 * p / len(data)) + "% done ", end='\r')
for letter in long_letter_reverse_mapping:
sentence_real = sentence_real.replace(
letter, long_letter_reverse_mapping[letter])
sentence_out = sentence_out.replace(
letter, long_letter_reverse_mapping[letter])
print("Computing editdistance and writing to -> " + 'languages/' +
language + '/results.' + model_name.split('/')[-1])
fl = codecs.open('languages/' + language + '/results.' +
model_name.split('/')[-1],
'w',
encoding='utf-8')
fl.write(sentence_in + '\n' + sentence_real + '\n' + sentence_out + '\n')
fl.write(
str(editdistance.eval(sentence_real, sentence_out)) + ' / ' +
str(len(sentence_real)))
def main(args):
# Phase 1: Assemble the graph
# Step 1: read in data from the .xls file
theta = (3, 1)
x_data, y_data = gen_data(100, *theta)
# Step 2: create placeholders for input X (number of fire) and label Y
# (number of theft)
X = tf.placeholder(tf.float32, shape=[None, 1], name="X")
Y = tf.placeholder(tf.float32, shape=[None, 1], name="Y")
# Step 3: create weight and bias, initialized to 0
w = tf.Variable(tf.zeros([1, 1]), name='w')
b = tf.Variable(tf.zeros([1, 1]), name='b')
# Step 4: predict Y (number of theft) from the number of fire
with tf.name_scope("Model"):
Y_predicted = tf.add(tf.matmul(X, w), b, name='Y_predicted')
# Step 5: use the square error as the loss function
with tf.name_scope("Loss"):
loss = tf.reduce_mean(tf.square(Y - Y_predicted, name='loss'))
# Step 6: using gradient descent with learning rate of 0.01 to minimize
# loss
with tf.name_scope("SDG"):
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=0.001).minimize(loss)
# Create operation to initialize all variables
init = tf.global_variables_initializer()
# Create a saver
saver = tf.train.Saver()
# Phase 2: Train our model
with tf.Session() as sess:
# Step 7: initialize the necessary variables, in this case, w and b
sess.run(init)
# Step 8: train the model
for i in range(500): # run 500 epochs
# Session runs optimizer to minimize loss and fetch the value of
# loss
_, loss_, = sess.run([optimizer, loss],
feed_dict={
X: x_data,
Y: y_data
})
if i % 10 == 0:
print("Epoch {0}: {1}".format(i, loss_))
# Step 9: output the values of w and b
w_value, b_value = sess.run([w, b])
print("w_hat: %0.2f, b_hat: %0.2f" % (w_value, b_value))
print("w: %0.2f, b: %0.2f" % theta)
saver.save(sess, os.path.join(args['PATH'], 'model'))
def prediction(input_data, candidate_answers):
# 生成词表（word）
t = open(model_dir + '/vocab_word', 'rb')
vocab_word = pickle.load(t)
t.close()
print("========================word===========================")
print('dec_vocab_size: ', vocab_word.n_words_for_decoder)
print('vocab_size: ', vocab_word.n_words)
print('max_word_length: ', max(map(lambda x: len(x),
vocab_word.word2index)))
# smn
embedding = nn.Embedding(vocab_word.n_words, embed_size, padding_idx=0)
smn = SMN(embed_size,
hidden_size,
compression_hidden_size,
max_num_utterance,
max_length,
bidirectional=bidirectional,
num_layers=num_layers,
dropout_p=dropout_p)
embedding = torch.nn.DataParallel(embedding).to(device_cuda)
smn = torch.nn.DataParallel(smn).to(device_cuda)
embedding.load_state_dict(
torch.load(model_dir + '/embedding', map_location='cpu'))
smn.load_state_dict(torch.load(model_dir + '/smn', map_location='cpu'))
input_data_with_kb = list(map(lambda x: sentence_with_kb(x), input_data))
input_data_with_kb = list(
map(lambda x: word_tokenizer(x), input_data_with_kb))
pred_probs = []
for i in range(len(input_data)):
# score
candidate_response = candidate_answers[i]
candidate_utterances = [input_data_with_kb[i]
] * len(candidate_response)
examples = gen_data(vocab_word, candidate_utterances,
candidate_response, 1, max_length,
max_num_utterance)
print(candidate_utterances)
print(candidate_response)
print(examples[0][1])
print(examples[0][2])
utterances = torch.tensor(examples[0],
dtype=torch.long,
device=device_cuda)
response = torch.tensor(examples[1],
dtype=torch.long,
device=device_cuda)
probs = inference(utterances, response, embedding, smn)
pred_probs.append(probs)
return pred_probs
def test_(self):
# Loss Functions
self.decrypt_err_eve = tf.reduce_mean(tf.square(self.secret - self.eve_output))
self.decrypt_err_bob = tf.reduce_mean(tf.square(self.secret - self.bob_output))
self.decrypt_err_alice = tf.reduce_mean(tf.square(self.msg - self.alice_output))
# Train Eve for two minibatches to give it a slight computational edge
no_of_examples = 20
alice_test_error = []
bob_test_error = []
eve_test_error = []
for i in range(no_of_examples):
#alice_decrypt_error, bob_decrypt_error, eve_decrypt_error = 0.0, 0.0, 0.0
t1 = datetime.now()
bs = self.batch_size
msg_in_val, secret_val, key_val = gen_data(n=bs, msg_len=self.msg_len, secret_len=self.secret_len, key_len=self.random_seed_len)
alice = self.sess.run(self.alice_output,
feed_dict={self.msg: msg_in_val, self.secret: secret_val, self.seed: key_val})
#eve_decrypt_error = min(eve_decrypt_error, decrypt_err)
t2 = datetime.now()
# eve_decrypt_error = eve_decrypt_error + decrypt_err_eve
# bob_decrypt_error = bob_decrypt_error + decrypt_err_bob
# alice_decrypt_error = alice_decrypt_error + decrypt_err_alice
bob = self.sess.run(self.bob_output,
feed_dict={self.alice_output:alice, self.seed: key_val})
t3 = datetime.now()
# print msg_in_val[0], key_val[0], alice[0],bob[0],eve[0]
bob_test_error.append((t3-t2).microseconds*1.0 /bs)
#eve_test_error.append(eve_decrypt_error)
alice_test_error.append( (t2-t1).microseconds*1.0 /bs )
#visualize_image(convToInt(msg_in_val),self.file_name,bs,steg=False)
#visualize_image(convToInt(alice),self.file_name,bs,steg=True)
print bob_test_error
print alice_test_error
print np.mean(bob_test_error), np.std(bob_test_error)
#print np.mean(eve_test_error), np.std(eve_test_error)
print np.mean(alice_test_error), np.std(alice_test_error)
def gen_data_fun(key):
"""
根据关键字选择数据生成方式并返回
:param key:
:return:
"""
global global_price, global_unit, global_tax
global_price = float(global_price)
global_tax = float(global_tax)
if "YEAR/MONTH/DAY" == key:
return utils.gen_data()
elif "12_10_LEN" == key:
return utils.gen_len_digit([12, 10])
elif "8_LEN" == key:
return utils.gen_len_digit([8])
elif "ENTERPRISE" == key:
return utils.gen_enterprise()
elif "15_18_20_LEN" == key:
return utils.gen_len_digit([15, 18, 20])
elif "11_LEN" == key:
return utils.gen_len_digit([11])
elif "16_17_19_LEN" == key:
return utils.gen_len_digit([16, 17, 19])
elif "NONE" == key:
return ""
elif "PERSON_NAME" == key:
return utils.gen_person_name()
elif "PROVINCE" == key:
return utils.gen_province()
elif "GOOD_NAME" == key:
good, global_unit = utils.gen_good_name_and_unit()
return good
elif "UNIT" == key:
return global_unit
elif "NUMS" == key:
if "Kg" == global_unit:
global_price = float("{:.2f}".format(0.1 + random.random() * 5))
else:
global_price = random.randint(1, 20)
return "{:.2f}".format(global_price)
elif "PRICE_TAX" == key:
tax_one = float("{:.2f}".format(0.5 + random.random()))
global_price *= tax_one
return "{:.2f}".format(tax_one)
elif "PRICE" == key:
return "{:.2f}".format(global_price)
elif "RATE" == key:
global_tax = float("{:.2f}".format(0.1 + random.random() * 10.0))
return "{:.2f}".format(global_tax)
elif "TAX_SUM" == key:
return "{:.2f}".format(global_price * global_tax / 100)
def percentAccuracy(filename, ns, times): global zones, data with open(filename, 'wb') as csvfile: spamwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow(["NUM", "PERCENT ACCURACY"]) for n in ns: print n for i in range(times): print "-", i zones, data = utils.gen_data(n, 1000) acc, _ = getCrossCorrelation(True) spamwriter.writerow([n, acc*100])
def main():
print("Testing KD Tree...")
test_times = 100
run_time_1 = run_time_2 = 0
for _ in range(test_times):
# 随机生成数据
low = 0
high = 100
n_rows = 1000
n_cols = 2
X = gen_data(low, high, n_rows, n_cols)
y = gen_data(low, high, n_rows)
Xi = gen_data(low, high, n_cols)
# 创建Kd树
tree = KDTree()
tree.build_tree(X, y)
# Kd树查找
start = time()
nd = tree.nearest_neighbour_search(Xi)
run_time_1 += time() - start
ret1 = get_eu_dist(Xi, nd.split[0])
# 普通线性查找
start = time()
row = exhausted_search(X, Xi)
run_time_2 += time() - start
ret2 = get_eu_dist(Xi, row)
# 比较结果
assert ret1 == ret2, "target:%s\nrestult1:%s\nrestult2:%s\ntree:\n%s" % (
Xi, nd, row, tree)
print("%d tests passed!" % test_times)
print("KD Tree Search %.2f s" % run_time_1)
print("Exhausted search %.2f s" % run_time_2)
def main():
args = utils.get_args()
print("Prepare dataset...")
mnist = input_data.read_data_sets("mnist/", one_hot = True)
with tf.Graph().as_default(), tf.Session() as session:
autoencoder = Autoencoder(
784, args.hid_shape, args.lat_shape,
optimizer = tf.train.AdagradOptimizer(args.lr),
batch_size = args.batch_size,
dropout = args.dropout)
session.run(tf.initialize_all_variables())
if args.save_model or args.load_model:
saver = tf.train.Saver()
if args.load_model:
try:
saver.restore(session, utils.SAVER_FILE)
except ValueError:
print("Cant find model file")
sys.exit(1)
if args.make_imgs:
index = 0
print("Prepare images directory...")
utils.prepare_image_folder()
example = utils.get_example(args.digit, mnist.test)
print("Start training...")
for epoch in range(args.epoches):
for i, batch in enumerate(utils.gen_data(args.batch_size, mnist.train.images)):
autoencoder.fit_on_batch(session, batch)
if (i+1) % args.log_after == 0:
test_cost = autoencoder.evaluate(session, mnist.test.images)
print("Test error = {0:.4f} on {1} batch in {2} epoch".format(test_cost, i+1, epoch+1))
if args.make_imgs:
path = os.path.join(utils.IMG_FOLDER, "{0:03}.png".format(index))
autoencoded = autoencoder.encode_decode(session, example.reshape(1, 784))
utils.save_image(autoencoded.reshape((28, 28)), path)
index += 1
if args.save_model:
saver.save(session, utils.SAVER_FILE)
print("Model saved")
def run_recurrent(
# Min/max sequence length
min_length=50,
max_length=55,
# Number of units in the hidden (recurrent) layern
n_hidden=100,
# Number of training sequences in each batch
batch_size=100,
# Optimization learning rate
learning_rate=.001,
# All gradients above this will be clipped
grad_clipping=100.0,
# How often should we check the output?
epoch_size=100,
# Number of epochs to train the net
n_epochs=10,
model_name='single_layer_rnn'):
#############
# Load Data #
#############
print("Loading data...")
# We'll use this "validation set" to periodically check progress
X_val, y_val, mask_val = gen_data(min_length=min_length,
max_length=max_length,
batch_size=batch_size)
############################################
# allocate symbolic variables for the data #
############################################
target_var = T.vector('targets')
####################################################
# BUILD MODEL (The model is a function in Lasagne) #
####################################################
print('... building the model')
model = None
if model_name == 'single_layer_rnn':
model = single_layer_rnn(shape=(batch_size, max_length, 2),
n_hidden=n_hidden,
grad_clipping=grad_clipping)
elif model_name == 'two_layer_rnn':
model = two_layer_rnn(shape=(batch_size, max_length, 2),
n_hidden=n_hidden,
grad_clipping=grad_clipping)
elif model_name == 'single_layer_bidirectional_rnn':
model = single_layer_bidirectional_rnn(shape=(batch_size, max_length,
2),
n_hidden=n_hidden,
grad_clipping=grad_clipping)
elif model_name == 'two_layer_bidirectional_rnn':
model = two_layer_bidirectional_rnn(shape=(batch_size, max_length, 2),
n_hidden=n_hidden,
grad_clipping=grad_clipping)
elif model_name == 'single_layer_lstm':
model = single_layer_lstm(shape=(batch_size, max_length, 2),
n_hidden=n_hidden,
grad_clipping=grad_clipping)
elif model_name == 'single_layer_bidirectional_lstm':
model = single_layer_bidirectional_lstm(shape=(batch_size, max_length,
2),
n_hidden=n_hidden,
grad_clipping=grad_clipping)
#####################
# Training Function #
#####################
# lasagne.layers.get_output produces a variable for the output of the net
network_output = lasagne.layers.get_output(model.l_out)
# The network output will have shape (batch_size, 1); let's flatten to get a
# 1-dimensional vector of predicted values
prediction = network_output.flatten()
# Our cost will be mean-squared error
loss = T.mean((prediction - target_var)**2)
# Retrieve all parameters from the network
params = lasagne.layers.get_all_params(model.l_out)
updates = lasagne.updates.adagrad(loss,
params,
learning_rate=learning_rate)
# Theano functions for training and computing loss
train_model = theano.function(
inputs=[model.l_in.input_var, target_var, model.l_mask.input_var],
outputs=loss,
updates=updates,
name='train')
##################################
# Validation & Testing Functions #
##################################
validate_model = theano.function(
inputs=[model.l_in.input_var, target_var, model.l_mask.input_var],
outputs=loss,
name='validate')
###############
# TRAIN MODEL #
###############
print('... Starting training')
try:
for epoch in range(n_epochs):
for _ in range(epoch_size):
X, y, m = gen_data(min_length=min_length,
max_length=max_length,
batch_size=batch_size)
train_model(X, y, m)
val_err = validate_model(X_val, y_val, mask_val)
print("Epoch {} validation error = {}".format(epoch, val_err))
except KeyboardInterrupt:
pass
def run_train():
data = {
'trainAnswers': [],
'devAnswers': [],
'trainQuestions': [],
'devQuestions': []
}
data['trainAnswers'].extend(
map(
lambda x: x.split(' '),
open('opennmt-kb-char-' + str(dev_id) + '/train.txt.tgt',
encoding="utf-8").read().strip().split('\n')))
data['trainQuestions'].extend(
map(
lambda x: x.split(' '),
open('opennmt-kb-char-' + str(dev_id) + '/train.txt.src',
encoding="utf-8").read().strip().split('\n')))
data['devAnswers'].extend(
map(
lambda x: x.split(' '),
open('opennmt-kb-char-' + str(dev_id) + '/val.txt.tgt',
encoding="utf-8").read().strip().split('\n')))
data['devQuestions'].extend(
map(
lambda x: x.split(' '),
open('opennmt-kb-char-' + str(dev_id) + '/val.txt.src',
encoding="utf-8").read().strip().split('\n')))
# for debug
'''
data['trainAnswers'] = data['trainAnswers'][:5]
data['trainQuestions'] = data['trainQuestions'][:5]
data['devAnswers'] = [list(x) for x in data['trainAnswers']]
data['devQuestions'] = [list(x) for x in data['trainQuestions']]
'''
# 生成词表（word）
if os.path.exists(model_dir + '/vocab_word'):
t = open(model_dir + '/vocab_word', 'rb')
vocab_word = pickle.load(t)
t.close()
else:
vocab_word = prepare_vocabulary(data, cut=cut)
t = open(model_dir + '/vocab_word', 'wb')
pickle.dump(vocab_word, t)
t.close()
print("========================word===========================")
print('dec_vocab_size: ', vocab_word.n_words_for_decoder)
print('vocab_size: ', vocab_word.n_words)
print('max_word_length: ', max(map(lambda x: len(x),
vocab_word.word2index)))
# 生成数据（截断，生成负例）
if os.path.exists(model_dir + '/data'):
t = open(model_dir + '/data', 'rb')
train_examples, dev_examples = pickle.load(t)
t.close()
else:
train_examples = gen_data(vocab_word, data['trainQuestions'],
data['trainAnswers'], 1, max_length,
max_num_utterance)
dev_examples = gen_data(vocab_word, data['devQuestions'],
data['devAnswers'], 10, max_length,
max_num_utterance)
t = open(model_dir + '/data', 'wb')
pickle.dump((train_examples, dev_examples), t)
t.close()
print(train_examples[0][1])
print(train_examples[0][2])
print(dev_examples[0][1])
print(dev_examples[0][2])
print("========================dataset===========================")
print('train: ', len(train_examples[0]), len(train_examples[1]),
len(train_examples[2]))
print('dev: ', len(dev_examples[0]), len(dev_examples[1]),
len(dev_examples[2]))
embed = init_embedding(embed_size, vocab_word.n_words,
vocab_word.word2index, True)
embedding = nn.Embedding(vocab_word.n_words, embed_size,
padding_idx=0).from_pretrained(embed,
freeze=False)
dam = DAM(embed_size, max_num_utterance, max_length, max_stacks)
embedding = torch.nn.DataParallel(embedding).to(device_cuda)
dam = torch.nn.DataParallel(dam).to(device_cuda)
if os.path.isfile(model_dir + '/embedding'):
embedding.load_state_dict(torch.load(model_dir + '/embedding'))
if os.path.isfile(model_dir + '/dam'):
dam.load_state_dict(torch.load(model_dir + '/dam'))
optimizer = optim.Adam([{
"params": embedding.parameters()
}, {
"params": dam.parameters()
}],
lr=lr,
amsgrad=True)
if os.path.isfile(model_dir + '/optimizer'):
optimizer.load_state_dict(torch.load(model_dir + '/optimizer'))
trainIters(vocab_word,
embedding,
dam,
optimizer,
train_examples,
dev_examples,
n_epochs,
lr,
batch_size,
infer_batch_size,
print_every=1)
sparsity=s,
seed=s).ravel()
if len(np.nonzero(subset)[0]) == 0:
eta[i1, nidx, rep, i3, :] = np.nan
eta2[i1, nidx, rep, i3] = np.nan
continue
else:
eta2[i1, nidx, rep,
i3] = calc_irrep_const(sigma_rep,
np.nonzero(subset)[0])
for rep2 in range(nreps2):
X, _, _, _, _ = gen_data(2000,
p,
covariance=sigma_rep,
beta=subset.ravel())
# Normalize X
X = StandardScaler().fit_transform(X)
C = 1 / n_ * X.T @ X
eta[i1, nidx, rep, i3,
rep2] = calc_irrep_const(C,
np.nonzero(subset)[0])
if comm.rank == 0:
print(time.time() - t0)
print('%d/%d' % (i1 + 1, len(task_chunk[comm.rank])))
# Gather and save results
rho = Gatherv_rows(rho, comm, root=0)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle.fluid as fluid
from nets import mlp
from utils import gen_data
input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
cost = mlp(input_x, input_y)
optimizer = fluid.optimizer.SGD(learning_rate=0.01)
optimizer.minimize(cost)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
step = 1001
for i in range(step):
cost_val = exe.run(feed=gen_data(),
fetch_list=[cost.name])
print("step%d cost=%f" % (i, cost_val[0]))
"""Perform animation step."""
x, y, z, count = data
particles.set_data(x, y)
particles_3d.set_data(x, y)
particles_3d.set_3d_properties(z)
title.set_text("Epochs: {}".format(count))
return particles, particles_3d, title
if __name__ == "__main__":
out_mode = get_out_mode()
for f in INFO.keys():
print("\n" + f.__name__.capitalize() + " function")
swarm = Swarm([(INFO[f]["x_min"], INFO[f]["x_max"]),
(INFO[f]["y_min"], INFO[f]["y_max"])], f, 1e-4)
if out_mode == ANIMATION:
display(f)
fig, title, particles, particles_3d = init(f)
anim = animate(fig, update, lambda: gen_data(swarm),
INFO[f]["fps"])
# save(anim, f.__name__, INFO[f]["fps"], ".mp4")
elif out_mode == TEXT:
swarm.minimize()
print()
def gen_data_(params, subcomm, subrank):
''' Use the seeds provided from the arg file to generate regression design and data'''
seed = params['seed']
if subrank == 0:
# Generate covariance
sigma = gen_covariance(params['n_features'],
params['cov_params']['correlation'],
params['cov_params']['block_size'],
params['cov_params']['L'],
params['cov_params']['t'])
# Sparsify the beta - seed with the block size
beta = sparsify_beta(params['betadict']['beta'], params['cov_params']['block_size'],
params['sparsity'], seed=params['cov_params']['block_size'])
else:
sigma = None
beta = None
sigma = Bcast_from_root(sigma, subcomm)
beta = Bcast_from_root(beta, subcomm)
params['sigma'] = sigma
params['betas'] = beta
# If all betas end up zero for this sparsity level, output a warning and skip
# this iteration (Make sure all ranks are instructed to continue!)
if np.count_nonzero(beta) == 0:
print('Warning, all betas were 0!')
print(params)
if subrank == 0:
# Generate data
X, X_test, y, y_test, ss = gen_data(params['n_samples'], params['n_features'],
params['kappa'], sigma, beta, seed)
# Standardize
X = StandardScaler().fit_transform(X)
X_test = StandardScaler().fit_transform(X_test)
y -= np.mean(y)
y_test -= np.mean(y_test)
else:
X = None
X_test = None
y = None
y_test = None
ss = None
X = Bcast_from_root(X, subcomm)
X_test = Bcast_from_root(X_test, subcomm)
y = Bcast_from_root(y, subcomm)
y_test = Bcast_from_root(y_test, subcomm)
ss = Bcast_from_root(ss, subcomm)
params['ss'] = ss
return X, X_test, y, y_test, params
def __call__(self, task_tuple):
cov_param_idx = task_tuple[0]
rep = task_tuple[1]
algorithm = task_tuple[2]
n_features = self.n_features
n_samples = self.n_samples
beta = gen_beta2(n_features,
n_features,
sparsity=1,
betawidth=-1,
seed=1234)
cov_param = self.cov_params[cov_param_idx]
sigma = gen_covariance(n_features, cov_param['correlation'],
cov_param['block_size'], cov_param['L'],
cov_param['t'])
beta_ = sparsify_beta(beta,
cov_param['block_size'],
sparsity=0.25,
seed=cov_param['block_size'])
# Follow the procedure of generating beta with a fixed betaseed at the getgo
# and then sparsifying as one goes on. Is this reproducible subsequently?
t0 = time.time()
X, X_test, y, y_test, ss = gen_data(n_samples,
n_features,
kappa=5,
covariance=sigma,
beta=beta_)
# Standardize
X = StandardScaler().fit_transform(X)
y -= np.mean(y)
if algorithm == 0:
lasso = LassoCV(fit_intercept=False, cv=5)
lasso.fit(X, y.ravel())
beta_hat = lasso.coef_
elif algorithm == 1:
uoi = UoI_Lasso(fit_intercept=False, estimation_score='r2')
uoi.fit(X, y)
beta_hat = uoi.coef_
elif algorithm == 2:
scad = PycassoCV(penalty='scad',
fit_intercept=False,
nfolds=5,
n_alphas=100)
scad.fit(X, y)
beta_hat = scad.coef_
elif algorithm == 3:
mcp = PycassoCV(penalty='mcp',
fit_intercept=False,
nfolds=5,
n_alphas=100)
mcp.fit(X, y)
beta_hat = mcp.coef_
self.beta.append(beta_)
self.beta_hat.append(beta_hat)
self.task_signature.append((cov_param_idx, rep, algorithm))
print('call successful, algorithm %d took %f seconds' %
(algorithm, time.time() - t0))
from paddle.fluid.incubate.fleet.base import role_maker
input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
cost = mlp(input_x, input_y)
optimizer = fluid.optimizer.Adagrad(learning_rate=0.01)
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
optimizer = fleet.distributed_optimizer(optimizer)
optimizer.minimize(cost)
if fleet.is_server():
fleet.init_server()
fleet.run_server()
elif fleet.is_worker():
fleet.init_worker()
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
step = 1001
for i in range(step):
cost_val = exe.run(program=fleet.main_program,
feed=gen_data(),
fetch_list=[cost.name])
print("worker_index: %d, step%d cost = %f" %
(fleet.worker_index(), i, cost_val[0]))
fleet.stop_worker()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--hdim', default=512, type=int)
parser.add_argument('--grad_clip', default=100, type=int)
parser.add_argument('--lr', default=0.01, type=float)
parser.add_argument('--batch_size', default=50, type=int)
parser.add_argument('--num_epochs', default=10, type=int)
parser.add_argument('--seq_len', default=60, type=int)
parser.add_argument('--depth', default=1, type=int)
parser.add_argument('--model', default=None)
parser.add_argument('--model_name_prefix', default='model')
parser.add_argument('--language', default='hy-AM')
parser.add_argument('--start_from', default=0, type=float)
args = parser.parse_args()
print("Loading Files")
(char_to_index, index_to_char, vocab_size, trans_to_index, index_to_trans,
trans_vocab_size) = utils.load_vocabulary(language=args.language)
(train_text, val_text,
trans) = utils.load_language_data(language=args.language)
data_size = len(train_text)
print("Building Network ...")
(output_layer, train, cost) = utils.define_model(args.hdim,
args.depth,
args.lr,
args.grad_clip,
trans_vocab_size,
vocab_size,
is_train=True)
if args.model:
f = np.load('languages/' + args.language + '/models/' + args.model)
param_values = [np.float32(f[i]) for i in range(len(f))]
lasagne.layers.set_all_param_values(output_layer, param_values)
print("Training ...")
p = int(len(train_text) * args.start_from) + 1
step_cnt = 0
avg_cost = 0
it = 0
while it < args.num_epochs:
avg_cost = 0
date_at_beginning = datetime.now()
non_native_skipped = 0
for _ in range(PRINT_FREQ):
x, y, p, turned, non_native_sequences = utils.gen_data(
p, args.seq_len, args.batch_size, train_text, trans,
trans_to_index, char_to_index)
if turned:
it += 1
avg_cost += train(x, np.reshape(y, (-1, vocab_size)))
non_native_skipped += non_native_sequences
date_after = datetime.now()
print("Epoch {} average loss = {} Time {} sec. Nonnatives skipped {}".
format(1.0 * it + 1.0 * p / data_size, avg_cost / PRINT_FREQ,
(date_after - date_at_beginning).total_seconds(),
non_native_skipped))
step_cnt += 1
if True: #step_cnt * args.batch_size > 100000:
print('computing validation loss...')
val_turned = False
val_p = 0
val_steps = 0.
val_cost = 0.
while not val_turned:
x, y, val_p, val_turned, non_native = utils.gen_data(
val_p, args.seq_len, args.batch_size, val_text, trans,
trans_to_index, char_to_index)
val_steps += 1
val_cost += cost(x, np.reshape(y, (-1, vocab_size)))
print('validation loss is ' + str(val_cost / val_steps))
file_name = 'languages/' + args.language + '/models/' + args.model_name_prefix + '.hdim' + str(
args.hdim) + '.depth' + str(args.depth) + '.seq_len' + str(
args.seq_len) + '.bs' + str(
args.batch_size) + '.epoch' + str(
1.0 * it + 1.0 * p / data_size) + '.loss' + str(
avg_cost / PRINT_FREQ) + '.npz'
print("saving to -> " + file_name)
np.save(file_name,
lasagne.layers.get_all_param_values(output_layer))
step_cnt = 0
def getProbs(zn, tn): global zones, data zones, data = utils.gen_data(zn, tn) R = getCrossCorrelation() cov_thr = getCovar(R) return (cov_thr[0], getTransitionProb(cov_thr))
# check_get floor
brick.control_brick(ord("s"), field)
can_down = utils.check_valid(field, brick)
brick.revert()
if not can_down:
field.add_brick(brick)
brick.create_new_brick()
# clear one row when full
field.check_and_clear_rows()
# render screen
rendered = utils.create_screen(field, brick)
window.addstr(0, 0, rendered)
def move_set(keys_list):
global window
move(keys_list[0])
for key in keys_list[1:]:
window.getch()
move(key)
while True:
window.getch()
keys_list = utils.get_moveset(field, brick)
utils.gen_data(field, brick, keys_list)
move_set(keys_list)
import time
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from utils import gen_data, iterModel, MetaClassifier, Sampling
EPOCHS = 3 # arbitrary number of times that the flow will work through the entire dataset.
for i in tqdm(range(0, EPOCHS)):
# ------------------------------ I) Load & Process Data ----------------------------------- #
# A) Train Set: generate train dataset fro challenge data
PATH_TRAIN = pjoin("Data", "dataset-challenge.xlsx")
data_dict = gen_data(PATH_TRAIN, 'train')
X, y = [data_dict['X'], data_dict['y']]
# B) Test Set: generate test dataset from the challenge data (optional)
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15)
#train_dict = {'X_train': X_train, 'y_train':y_train}
#test_dict = {'X_test': X_test, 'y_test':y_test}
# C) Evaluation Set: generate evaluation dataset
PATH_EVAL = pjoin("Data", "dataset-evaluation.xlsx")
eval_dict = gen_data(PATH_EVAL, 'eval')
X_eval = eval_dict['X']
train_dict = {'X_train': X, 'y_train': y}
test_dict = {}
def test(self):
# Loss Functions
self.decrypt_err_eve = tf.reduce_mean(tf.square(self.secret - self.eve_output))
self.decrypt_err_bob = tf.reduce_mean(tf.square(self.secret - self.bob_output))
self.decrypt_err_alice = tf.reduce_mean(tf.square(self.msg - self.alice_output))
# Train Eve for two minibatches to give it a slight computational edge
no_of_examples = 5
alice_test_error = []
bob_test_error = []
eve_test_error = []
for i in range(no_of_examples):
alice_decrypt_error, bob_decrypt_error, eve_decrypt_error = 0.0, 0.0, 0.0
bs = self.batch_size
msg_in_val, secret_val, key_val = gen_data(n=bs, msg_len=self.msg_len, secret_len=self.secret_len, key_len=self.random_seed_len)
decrypt_err_alice, decrypt_err_bob, decrypt_err_eve,alice,bob,eve = self.sess.run([self.decrypt_err_alice, self.decrypt_err_bob, self.decrypt_err_eve,self.alice_output,self.bob_output,self.eve_output],
feed_dict={self.msg: msg_in_val, self.secret: secret_val, self.seed: key_val})
#eve_decrypt_error = min(eve_decrypt_error, decrypt_err)
eve_decrypt_error = eve_decrypt_error + decrypt_err_eve
bob_decrypt_error = bob_decrypt_error + decrypt_err_bob
alice_decrypt_error = alice_decrypt_error + decrypt_err_alice
print msg_in_val[0], key_val[0], alice[0],bob[0],eve[0]
bob_test_error.append(bob_decrypt_error)
eve_test_error.append(eve_decrypt_error)
alice_test_error.append(alice_decrypt_error)
visualize_image(convToInt(msg_in_val),self.file_name,bs,steg=False)
visualize_image(convToInt(alice),self.file_name,bs,steg=True)
print "here"
f = open(self.file_name+".out.txt" ,'w')
print "here2"
for i in bob_test_error:
f.write(str(i)+",")
f.write("\n")
for i in eve_test_error:
f.write(str(i)+",")
f.write("\n")
for i in alice_test_error:
f.write(str(i)+",")
f.write("\n")
f.write(str(np.mean(bob_test_error)) +", "+str(np.std(bob_test_error)))
f.write("\n")
f.write(str(np.mean(eve_test_error)) + ", " + str(np.std(eve_test_error)))
f.write("\n")
f.write(str(np.mean(alice_test_error)) + ", " + str(np.std(alice_test_error)))
f.write("\n")
f.close()
print np.mean(bob_test_error), np.std(bob_test_error)
print np.mean(eve_test_error), np.std(eve_test_error)
print np.mean(alice_test_error), np.std(alice_test_error)
return bob_decrypt_error, eve_decrypt_error, alice_decrypt_error
# 释放内存 否则会耗尽资源
del x_train
del x_test
del y_train
del y_test
del his
del score
del model
# 真正释放
gc.collect()
# 释放Keras
K.clear_session()
print '\nEnding Round ' + str(cnt) + ' at ' + time.strftime(
"%Y-%m-%d %H:%M:%S", time.localtime())
if __name__ == "__main__":
original_stdout = utils.before('resnet_training_console_output')
try:
X, label = utils.gen_data(
'../../data/whole/sharpened/whole_5_classes.h5')
train_resnet('../model/resnet', X, label, './')
# gen_resnet_model()
except Exception, e:
print e
traceback.print_exc()
raise
finally:
utils.after(original_stdout)
INFO = { # parameters for the animation
"quarto_stato": {
"fps": 10
},
"arduino": {
"fps": 20
}
}
if __name__ == "__main__":
out_mode = get_out_mode()
for name in INFO.keys():
print("\n" + name + ".png")
target = mpimg.imread("images/" + name + "/target.png")
model = mpimg.imread("images/" + name + "/model.png")
swarm = Swarm([(0, target.shape[1] - model.shape[1]), (0, target.shape[0] - model.shape[0])],
lambda x, y: dissimilarity(x, y, model, target), 0.5)
if out_mode == ANIMATION:
display(model, target, name)
fig, title, boxes = init(model, target, name)
anim = animate(fig, update, lambda: gen_data(swarm), INFO[name]["fps"])
# save(anim, name + "/" + name, INFO[name]["fps"], ".mp4")
elif out_mode == TEXT:
swarm.minimize()
print()
