def key_schedule(key):
"""
From the given key, generate 16 subkeys to be used in each round.
Uses constants PC_1 and PC_2.
Page 20 of the FIPS doc and:
https://en.wikipedia.org/wiki/Data_encryption_standard#Key_schedule
"""
keys = []
key = utils.str_to_bits(key)
# Apply PC_1 on the key
# effectively converting a 64 bit key into 56 bits
key = utils.permute(key, C.PC_1)
# Split it into two halves
c, d = utils.nsplit(key, 28)
for i in range(16):
# Shift both halves by a specified amount
c = utils.lshift(c, C.SHIFT[i])
d = utils.lshift(d, C.SHIFT[i])
# Apply PC_2 to get a round key of 48 bits
k = utils.permute(c + d, C.PC_2)
keys.append(k)
return keys
def capture(self, input_R):
if (type(input_R) == pd.Series):
input_R = input_R.tolist()
self.output_T = pd.Series(utils.bind(self.ngram + [input_R]))
i = self.ngram_size - 2
while (i > 0):
self.ngram[i] = utils.permute(self.ngram[i - 1])
i -= 1
self.ngram[0] = utils.permute(input_R)
def analyzeScores(node, scores, ibdsamples, publicnodes):
result = utils.JStruct()
highscores = dict(
(k, s) for k, s in scores.items() if sum(s) >= sum(scores[node]))
result.num_cands = len(highscores)
siblinggroups = set(
frozenset(k.dad.children & k.mom.children) for k in highscores)
groupreps = utils.permute(
[node if node in group else list(group)[0] for group in siblinggroups])
groupreps = filter(
lambda r: nonmatchScore(node, r, publicnodes, scores, ibdsamples) >
0.1, groupreps)
result.num_groups_unfiltered = len(siblinggroups)
result.num_groups = len(groupreps)
#FIXME: inefficient
#result.distances = [smallestDistance(node, rep) for rep in groupreps]
try:
result.generations = map(lambda n: n.generation, groupreps)
except AttributeError:
pass
result.num_relatives = sum(1 for r in common.relationMap(node)
if r.ispublic)
result.num_matching_relatives = len(scores[node])
result.cand_num_rel_hist = map(lambda n: len(scores[n]),
groupreps) | pHist()
return result
def crossOver(chrnum, topsegs, botsegs):
"""given two sets of segments corresponding to two homologous chromosomes,
produce segments for result of crossover"""
length = chromelengths[chrnum]
morgan = 1e8
topsegs, botsegs = utils.permute([topsegs, botsegs])
cur_chiasma = 0
chiasmas = []
while True:
cur_chiasma += int(morgan * (-math.log(random.random())))
if cur_chiasma >= length:
break
chiasmas.append(cur_chiasma)
chiasmas.append(length) # one fake chiasma
if len(chiasmas) == 1:
return topsegs
outsegs = []
insegs = [tup + (0,) for tup in topsegs] + [tup + (1,) for tup in botsegs]
for (which, left, right, parity) in insegs:
for idx in xrange(parity, len(chiasmas), 2):
chi = chiasmas[idx]
if chi < left:
continue
prevchi = chiasmas[idx - 1] if idx > 0 else 0
if right < prevchi:
break
outsegs.append((which, max(left, prevchi), min(right, chi)))
return outsegs
def crossOver(chrnum, topsegs, botsegs):
"""given two sets of segments corresponding to two homologous chromosomes,
produce segments for result of crossover"""
length = chromelengths[chrnum]
morgan = 1e8
topsegs, botsegs = utils.permute([topsegs, botsegs])
cur_chiasma = 0
chiasmas = []
while True:
cur_chiasma += int(morgan * (-math.log(random.random())))
if cur_chiasma >= length:
break
chiasmas.append(cur_chiasma)
chiasmas.append(length) # one fake chiasma
if len(chiasmas) == 1:
return topsegs
outsegs = []
insegs = [tup + (0, )
for tup in topsegs] + [tup + (1, ) for tup in botsegs]
for (which, left, right, parity) in insegs:
for idx in xrange(parity, len(chiasmas), 2):
chi = chiasmas[idx]
if chi < left:
continue
prevchi = chiasmas[idx - 1] if idx > 0 else 0
if right < prevchi:
break
outsegs.append((which, max(left, prevchi), min(right, chi)))
return outsegs
def count(self, sequences):
for sequence in sequences:
for s in permute(sequence.split(' ')):
for nouns in s:
noun = ''.join(nouns)
if (noun not in counter):
self.counter[noun] = naver_search(noun)
def sample_site_from_copies(sigma, Ne, L, copies, ps=None):
if ps is None:
ps = ps_from_copies(sigma, Ne, L, copies)
k = inverse_cdf_sample(range(L + 1), ps, normalized=False)
return "".join(
permute(["A" for _ in range(L - k)] +
[random.choice("CGT") for _ in range(k)]))
def ppo_step(self, num_value_iters, states, actions, indices, returns, advantages, fixed_action_logprobs, fixed_index_logprobs, lr_mult, lr, clip_epsilon, l2_reg):
clip_epsilon = clip_epsilon * lr_mult
"""update critic"""
values_target = Variable(u.cuda_if_needed(returns, self.args)) # (mb, 1)
for k in range(num_value_iters):
values_pred = self.valuefn(Variable(states)) # (mb, 1)
value_loss = (values_pred - values_target).pow(2).mean()
# weight decay
for param in self.valuefn.parameters():
value_loss += param.pow(2).sum() * l2_reg
self.value_optimizer.zero_grad()
value_loss.backward()
self.value_optimizer.step()
"""update policy"""
advantages_var = Variable(u.cuda_if_needed(advantages, self.args)).view(-1) # (mb)
########################################
perm_idx, sorted_actions = u.sort_decr(actions)
inverse_perm_idx = u.invert_permutation(perm_idx)
group_idx, group_actions = u.group_by_element(sorted_actions)
# permute everything by action type
states_ap, actions_ap, indices_ap = map(lambda x: u.permute(x, perm_idx), [states, actions, indices])
# group everything by action type
states_ag, actions_ag, indices_ag = map(lambda x: u.group_by_indices(x, group_idx), [states_ap, actions_ap, indices_ap])
action_logprobs, index_logprobs = [], []
for grp in xrange(len(group_idx)):
states_grp = torch.stack(states_ag[grp]) # (g, grp_length, indim)
actions_grp = torch.LongTensor(np.stack(actions_ag[grp])) # (g)
indices_grp = torch.LongTensor(np.stack(indices_ag[grp])) # (g)
actions_grp = u.cuda_if_needed(actions_grp, self.args)
indices_grp = u.cuda_if_needed(indices_grp, self.args)
alp, ilp = self.policy.get_log_prob(Variable(states_grp), Variable(actions_grp), Variable(indices_grp))
action_logprobs.append(alp)
index_logprobs.append(ilp)
action_logprobs = torch.cat(action_logprobs)
index_logprobs = torch.cat(index_logprobs)
# unpermute
inverse_perm_idx = u.cuda_if_needed(torch.LongTensor(inverse_perm_idx), self.args)
action_logprobs = action_logprobs[inverse_perm_idx]
index_logprobs = index_logprobs[inverse_perm_idx]
########################################
ratio = torch.exp(action_logprobs + index_logprobs - Variable(fixed_action_logprobs) - Variable(fixed_index_logprobs))
surr1 = ratio * advantages_var # (mb)
surr2 = torch.clamp(ratio, 1.0 - clip_epsilon, 1.0 + clip_epsilon) * advantages_var # (mb)
policy_surr = -torch.min(surr1, surr2).mean()
self.policy_optimizer.zero_grad()
policy_surr.backward()
torch.nn.utils.clip_grad_norm(self.policy.parameters(), 40)
self.policy_optimizer.step()
def sample_site(sigma, mu, Ne, L):
phats = [phat(k, sigma, mu, Ne, L) for k in range(L + 1)]
# Z = sum(phats)
# ps = [ph/Z for ph in phats]
k = inverse_cdf_sample(range(L + 1), phats, normalized=False)
return "".join(
permute(["A" for _ in range(L - k)] +
[random.choice("CGT") for _ in range(k)]))
def prefix(s):
ndb = []
print("\nPermuting Bits...")
perm_bits = permute(s)
m = len(perm_bits)
print("Generating Negative Database...")
for i in range(m - 1, 1, -1):
x = list(create_sequence_of_symbol(m))
x[i] = to_symbol(comp(perm_bits[i]))
j = random.randint(0, i - 1)
while True:
k = random.randint(0, i - 1)
if j != k:
break
x[j] = to_symbol(perm_bits[j])
x[k] = to_symbol(perm_bits[k])
ndb.append(inv_permute(x))
x = list(create_sequence_of_symbol(m))
x[0] = to_symbol(perm_bits[0])
x[1] = to_symbol(comp(perm_bits[1]))
j = random.randint(2, m - 1)
x[j] = '0'
ndb.append(inv_permute(x))
x[j] = '1'
ndb.append(inv_permute(x))
x = list(create_sequence_of_symbol(m))
x[0] = to_symbol(comp(perm_bits[0]))
j = random.randint(1, m - 1)
while True:
k = random.randint(1, m - 1)
if j != k:
break
x[j] = '0'
x[k] = '0'
ndb.append(inv_permute(x))
x[k] = '1'
ndb.append(inv_permute(x))
x[k] = '*'
while True:
k = random.randint(1, m - 1)
if j != k:
break
x[j] = '1'
x[k] = '0'
ndb.append(inv_permute(x))
x[k] = '1'
ndb.append(inv_permute(x))
print("Negative Database Generated")
return ndb
def feistel(R, K):
"""
Feistel (F) function which operates on right half block and a subkey.
Page 16 of FIPS.
https://en.wikipedia.org/wiki/Data_encryption_standard#The_Feistel_(F)_function
"""
# Apply a permutation that expands the 32 bit half block to 48 bits
T = utils.permute(R, C.EXPAND)
# XOR the expanded value with given subkey
T = utils.xor(T, K)
# Apply SBoxes to a 48 bit input and return 32 bit output.
T = substitute(T)
# Finally, apply the direct permutation P
T = utils.permute(T, C.P)
return T
def test_order_invariance(self):
"""
Loads all cluster sections in the test config in all possible orders
(i.e. c1,c2,c3, c3,c1,c2, etc.) and test that the results are the same
"""
cfg = self.config
orig = cfg.clusters
cfg.clusters = None
sections = cfg._get_sections('cluster')
for perm in utils.permute(sections):
new = cfg._load_cluster_sections(perm)
assert new == orig
def improve_policy_ppo(self):
optim_epochs = self.args.ppo_optim_epochs # can anneal this
minibatch_size = self.args.ppo_minibatch_size
num_value_iters = self.args.ppo_value_iters
clip_epsilon = self.args.ppo_clip
gamma = self.args.gamma
tau = 0.95
l2_reg = 1e-3
batch = self.replay_buffer.sample()
all_states, all_actions, all_indices, all_fixed_action_logprobs, all_fixed_index_logprobs, all_values, all_rewards, all_masks, perm_idx, group_idx = self.unpack_ppo_batch(batch)
all_advantages, all_returns = self.estimate_advantages(all_rewards, all_masks, all_values, gamma, tau) # (b, 1) (b, 1)
# permute everything by length
states_p, actions_p, indices_p, returns_p, advantages_p, fixed_action_logprobs_p, fixed_index_logprobs_p = map(
lambda x: u.permute(x, perm_idx), [all_states, all_actions, all_indices, all_returns, all_advantages, all_fixed_action_logprobs, all_fixed_index_logprobs])
# group everything by length
states_g, actions_g, indices_g, returns_g, advantages_g, fixed_action_logprobs_g, fixed_index_logprobs_g = map(
lambda x: u.group_by_indices(x, group_idx), [states_p, actions_p, indices_p, returns_p, advantages_p, fixed_action_logprobs_p, fixed_index_logprobs_p])
for j in range(optim_epochs):
for grp in range(len(group_idx)):
states = torch.cat(states_g[grp], dim=0) # FloatTensor (g, grp_length, indim)
actions = torch.cat(actions_g[grp]) # LongTensor (g)
indices = torch.cat(indices_g[grp]) # LongTensor (g)
returns = torch.cat(returns_g[grp]) # FloatTensor (g)
advantages = torch.cat(advantages_g[grp]) # FloatTensor (g)
fixed_action_logprobs = u.cuda_if_needed(torch.FloatTensor(fixed_action_logprobs_g[grp]), self.args) # FloatTensor (g)
fixed_index_logprobs = u.cuda_if_needed(torch.FloatTensor(fixed_index_logprobs_g[grp]), self.args) # FloatTensor (g)
for x in [states, actions, indices, returns, advantages, fixed_action_logprobs, fixed_index_logprobs]:
assert not isinstance(x, torch.autograd.variable.Variable)
perm = np.random.permutation(range(states.shape[0]))
perm = u.cuda_if_needed(torch.LongTensor(perm), self.args)
states, actions, indices, returns, advantages, fixed_action_logprobs, fixed_index_logprobs = \
states[perm], actions[perm], indices[perm], returns[perm], advantages[perm], fixed_action_logprobs[perm], fixed_index_logprobs[perm]
optim_iter_num = int(np.ceil(states.shape[0] / float(minibatch_size)))
for i in range(optim_iter_num):
ind = slice(i * minibatch_size, min((i + 1) * minibatch_size, states.shape[0]))
states_b, actions_b, indices_b, advantages_b, returns_b, fixed_action_logprobs_b, fixed_index_logprobs_b = \
states[ind], actions[ind], indices[ind], advantages[ind], returns[ind], fixed_action_logprobs[ind], fixed_index_logprobs[ind]
self.ppo_step(num_value_iters, states_b, actions_b, indices_b, returns_b, advantages_b, fixed_action_logprobs_b, fixed_index_logprobs_b,
1, self.args.plr, clip_epsilon, l2_reg)
def analyze_motif(motif, trials=1000):
cols = transpose(motif)
L = len(cols)
ps = []
for col1, col2 in (choose2(cols)):
actual_mi = dna_mi(col1,col2)
perm_mis = [dna_mi(col1,permute(col2)) for i in xrange(trials)]
p = percentile(actual_mi, perm_mis)
#print p
ps.append(p)
q = fdr(ps)
correlated_pairs = [(i,j) for (i,j),p in zip(choose2(range(L)),ps) if p < q]
num_correlated = len(correlated_pairs)
print "correlated column pairs:", num_correlated, "%1.2f" % ((num_correlated)/choose(L,2))
return correlated_pairs
def mixmax_count(limit=5):
word = [chr(n) for n in range(65, 65 + limit)]
ordered = [chr(n) for n in range(65, 65 + limit)]
max_count = 0
valid_words = []
words = utils.permute(word)
for w in words:
count = mix_count(w, ordered)
if count > max_count:
max_count = count
valid_words = [w]
elif count == max_count:
valid_words.append(w)
return sorted(["".join(word) for word in valid_words])
def score(self, sequences, weight):
result = []
for sequence in sequences:
scr_max = 0
temp = []
for s in permute(sequence.split(' ')):
score = 1
for nouns in s:
noun = ''.join(nouns)
score *= self.counter[noun] * (weight**(len(s))) - 0.01
if (scr_max < score):
scr_max = score
temp = s
for t in temp:
result.append(''.join(t))
return ' '.join(result)
def analyze_motif(motif, trials=1000):
cols = transpose(motif)
L = len(cols)
ps = []
for col1, col2 in (choose2(cols)):
actual_mi = dna_mi(col1, col2)
perm_mis = [dna_mi(col1, permute(col2)) for i in xrange(trials)]
p = percentile(actual_mi, perm_mis)
#print p
ps.append(p)
q = fdr(ps)
correlated_pairs = [(i, j) for (i, j), p in zip(choose2(range(L)), ps)
if p < q]
num_correlated = len(correlated_pairs)
print "correlated column pairs:", num_correlated, "%1.2f" % (
(num_correlated) / choose(L, 2))
return correlated_pairs
def fitModel(self,trX,trY,snapshot_rate=1,path=None,epochs=30,batch_size=50,learning_rate_decay=0.97,decay_after=10):
X_minibatch=[]
Y_minibatch=[]
num_examples = trX.shape[1]
batches_in_one_epoch = int(num_examples / batch_size)
loss_values = []
for epoch in range(epochs):
t0 = time.time()
permutation = permute(num_examples)
if self.X.ndim > 2:
trX = trX[:,permutation,:]
else:
trX = trX[:,permutation]
if self.Y.ndim > 1:
trY = trY[:,permutation]
else:
trY = trY[permutation]
for j in range(batches_in_one_epoch):
if self.X.ndim > 2:
X_minibatch = trX[:,j*batch_size:(j+1)*batch_size,:]
else:
X_minibatch = trX[:,j*batch_size:(j+1)*batch_size]
if self.Y.ndim > 1:
Y_minibatch = trY[:,j*batch_size:(j+1)*batch_size]
else:
Y_minibatch = trY[j*batch_size:(j+1)*batch_size]
loss = self.train(X_minibatch,Y_minibatch)
loss_values.append(loss)
print "epoch={0} loss={1}".format(epoch,loss)
if path and epoch % snapshot_rate == 0:
print 'saving snapshot checkpoint.{0}'.format(epoch)
save(self,"{0}checkpoint.{1}".format(path,epoch))
f = open('{0}logfile'.format(path),'w')
for v in loss_values:
f.write('{0}\n'.format(v))
f.close()
t1 = time.time()
print 'Epoch took {0} seconds'.format(t1-t0)
def analyzeScores(node, scores, ibdsamples, publicnodes):
result = utils.JStruct()
highscores = dict((k, s) for k, s in scores.items() if sum(s) >= sum(scores[node]))
result.num_cands = len(highscores)
siblinggroups = set(frozenset(k.dad.children & k.mom.children) for k in highscores)
groupreps = utils.permute(
[node if node in group else list(group)[0] for group in siblinggroups])
groupreps = filter(lambda r: nonmatchScore(node, r, publicnodes, scores, ibdsamples) > 0.1, groupreps)
result.num_groups_unfiltered = len(siblinggroups)
result.num_groups = len(groupreps)
#FIXME: inefficient
#result.distances = [smallestDistance(node, rep) for rep in groupreps]
try:
result.generations = map(lambda n:n.generation, groupreps)
except AttributeError:
pass
result.num_relatives = sum(1 for r in common.relationMap(node) if r.ispublic)
result.num_matching_relatives = len(scores[node])
result.cand_num_rel_hist = map(lambda n: len(scores[n]), groupreps) | pHist()
return result
def __init__(self, n, m=None, output=False):
if m is None:
m = n
self.n = n
self.m = m
poss = []
poss2 = []
for s in three_parts(n,m,output=output):
poss.append(s)
if s[0] != s[1]:
poss.append(permute(s,(1,0,2)))
if s[0] != s[2]:
poss.append(permute(s,(2,1,0)))
if s[1] != s[2]:
poss.append(permute(s,(0,2,1)))
if s[1] != s[2] or s[1] != s[0]:
poss.append(permute(s,(1,2,0)))
poss.append(permute(s,(2,0,1)))
poss2.append(s)
if s[0] != s[1]:
poss2.append(permute(s,(1,0,2)))
self.poss = poss
self.poss_short = poss2
def __init__(self, n, m=None, output=False):
if m is None:
m = n
self.n = n
self.m = m
poss = []
poss2 = []
for s in three_parts(n, m, output=output):
poss.append(s)
if s[0] != s[1]:
poss.append(permute(s, (1, 0, 2)))
if s[0] != s[2]:
poss.append(permute(s, (2, 1, 0)))
if s[1] != s[2]:
poss.append(permute(s, (0, 2, 1)))
if s[1] != s[2] or s[1] != s[0]:
poss.append(permute(s, (1, 2, 0)))
poss.append(permute(s, (2, 0, 1)))
poss2.append(s)
if s[0] != s[1]:
poss2.append(permute(s, (1, 0, 2)))
self.poss = poss
self.poss_short = poss2
def makePairsRandom(nodes):
males, females = splitSexes(nodes)
numpairs = min(len(males), len(females))
return zip(utils.permute(males)[:numpairs], utils.permute(females)[:numpairs])
def ringify_col(col):
perm = {a: b for (a, b) in zip("ACGT", permute("ACGT"))}
return [perm[c] for c in col]
def __init__(self, config):
# Configuration
self.version = config.version
# Data config
self.batch_size = config.batch_size # batch size
self.max_length = config.graph_dimension # input sequence length (number of cities)
self.dimension = config.dimension # dimension of a city (coordinates)
# Network config
self.input_embed = config.input_embed # dimension of embedding space
self.num_neurons = config.num_neurons # dimension of hidden states (encoder)
self.num_stacks = config.num_stacks # encoder num stacks
self.num_heads = config.num_heads # encoder num heads
self.query_dim = config.query_dim # decoder query space dimension
self.num_units = config.num_units # dimension of attention product space (decoder and critic)
self.num_neurons_critic = config.num_neurons_critic # critic n-1 layer num neurons
self.initializer = tf.contrib.layers.xavier_initializer(
) # variables initializer
# Training config (actor and critic)
self.global_step = tf.Variable(0, trainable=False,
name="global_step") # actor global step
self.global_step2 = tf.Variable(
0, trainable=False, name="global_step2") # critic global step
self.init_B = config.init_B # critic initial baseline
self.lr_start = config.lr_start # initial learning rate
self.lr_decay_step = config.lr_decay_step # learning rate decay step
self.lr_decay_rate = config.lr_decay_rate # learning rate decay rate
self.is_training = config.is_training # switch to False if test mode
self.C = config.C
self.temperature = tf.placeholder("float", 1) # config.temperature
# instance variables to save
self.logits1, self.logits2, self.idx, self.encoded_ref, \
self.inter_city_distances, self.permutations, self.log_prob_mean, self.entropies_mean, \
self.reward, self.predictions, self.loss, self.loss_2, \
self.trn_op1, self.trn_op2, self.trn_op3, self.v_g, self.v = (None for _ in range(17))
# Tensor block holding the input sequences [Batch Size, Sequence Length, Features]
self.input_ = tf.placeholder(tf.float32,
[None, self.max_length, self.dimension],
name="input_coordinates")
if 'graph' in self.version:
self.graph_structure = tf.placeholder(
tf.float32, [None, self.max_length, self.max_length],
name='adj_matrix')
else:
self.graph_structure = None
if self.is_training:
self.optimal_tour = tf.placeholder(tf.int32,
[None, self.max_length],
name="optimal_tour")
self.next = permute(self.optimal_tour, 1)
self.next_next = permute(self.optimal_tour, 2)
else:
self.current = tf.placeholder(tf.int32, [None],
name="current_position")
self.previous = tf.placeholder(tf.int32, [None],
name="previous_position")
self.previous2 = tf.placeholder(tf.int32, [None],
name="previous2_position")
with tf.variable_scope("actor"):
self.encode_decode()
if self.is_training:
with tf.variable_scope("environment"):
self.build_reward()
with tf.variable_scope("optimizer"):
self.build_optim()
self.merged = tf.summary.merge_all()
def des(text, key, typ="encrypt"):
"""
The DES Encryption routine.
"""
# Clip Key
if len(key) < 8:
raise ValueError("key should be atleast 8 Bytes long.")
elif len(key) > 8:
print("Key is more than 8 Bytes long; taking first 8 Bytes.")
key = key[:8]
if len(text) % 8 != 0:
print(len(text))
raise ValueError("text length should be a multiple of 8.")
# TODO Add padding to data
text = utils.str_to_bits(text)
# Generate round keys
subkeys = key_schedule(key)
# This will store the result of encryption
cipher = []
# Since each block is encrypted independently of other blocks, this is ECB mode
# TODO Add support for other encryption modes: CBC etc.
# Store outputs of each round
# (used to verify that encryption & decryption are inverses of each other)
round_out = []
# Split input text into
for block in utils.nsplit(text, 64):
# Apply initial permutation to the block
block = utils.permute(block, C.IP)
# Split it into two halves
L, R = utils.nsplit(block, 32)
print("%2d - L%2d: %s R%2d: %s" %
(0, 0, utils.bits_to_hex(L), 0, utils.bits_to_hex(R)))
round_out.append((utils.bits_to_hex(L), utils.bits_to_hex(R)))
# The 16 rounds
for i in range(16):
# Only the keys change during encryption
if typ == "encrypt":
K = subkeys[i]
else:
K = subkeys[15 - i]
T = feistel(R, K)
T = utils.xor(L, T)
L = R
R = T
print("%2d - L%2d: %s R%2d: %s K%2d: %s" %
(i + 1, i + 1, utils.bits_to_hex(L), i + 1,
utils.bits_to_hex(R), i + 1, utils.bits_to_hex(K)))
round_out.append((utils.bits_to_hex(L), utils.bits_to_hex(R)))
# Apply the inverse initial permutation
cipher += utils.permute(R + L, C.IP_i)
cipher_text = utils.bits_to_str(cipher)
return cipher_text, round_out
def mate(self):
"""produce segments based on crossing over parents'"""
for chrnum in xrange(1, 23):
self.segments[chrnum] = utils.permute(
(crossOver(chrnum, *self.lparent.segments[chrnum]),
crossOver(chrnum, *self.rparent.segments[chrnum])))
def sample_motif_from_mismatches(cs,L):
return ["".join(permute(['A'] * (L - c) + [random.choice("CGT") for i in range(c)])) for c in cs]
import utils
"""
The Fibonacci sequence is defined by the recurrence relation:
Fn = Fn-1 + Fn-2, where F1 is 1 and F2 is 1
It turns out that F541, which contains 113 digits, is the first Fibonacci number for which the last nine digits are 1-9 pandigital
(contain all the digits 1 to 9, but not necessarily in order). And F2749, which contains 575 digits, is the first Fibonacci number
for which the first nine digits are 1-9 pandigital.
Given that Fk is the first Fibonacci number for which the first nine digits AND the last nine digits are 1-9 pandigital, find k.
"""
digits = range(1, 10)
pandigitals = set([utils.join(n) for n in utils.permute(range(1, 10))])
def firstAndLastTenPandigital(n):
"""return whether the first and last ten digits are pandigital"""
if utils.digit_slice(n, -9) in pandigitals:
if utils.digit_slice(n, 9) in pandigitals:
return True
return False
last, curr, index = 1, 1, 2
while True:
last, curr, index = curr, last + curr, index + 1 # fib increment
if index > 100 and firstAndLastTenPandigital(curr):
print(index) # output: 329468 in 23 seconds
break
def sample_col_from_count(count):
col = concat([[base] * n for base, n in zip(permute("ACGT"), count)])
random.shuffle(col)
return col
The number, 1406357289, is a 0 to 9 pandigital number because it is made up of each of the digits 0 to 9 in some order, but it also has a rather interesting sub-string divisibility property.
Let d1 be the 1st digit, d2 be the 2nd digit, and so on. In this way, we note the following:
d2d3d4=406 is divisible by 2
d3d4d5=063 is divisible by 3
d4d5d6=635 is divisible by 5
d5d6d7=357 is divisible by 7
d6d7d8=572 is divisible by 11
d7d8d9=728 is divisible by 13
d8d9d10=289 is divisible by 17
Find the sum of all 0 to 9 pandigital numbers with this property.
"""
firstPrimes = [2, 3, 5, 7, 11, 13, 17]
def divisibleByFirstPrimes(digits):
"""determine whether the substrings are divisible by the first 7 primes"""
for i, p in enumerate(firstPrimes):
if utils.join(digits[i + 1:i + 4]) % p != 0:
return False
return True
primePandigitalsSum = 0
for pan in utils.permute(range(0, 10)):
if divisibleByFirstPrimes(pan):
primePandigitalsSum += utils.join(pan)
print(primePandigitalsSum)
# output: 16695334890
import utils
"""
Pandigital prime
Problem 41
We shall say that an n-digit number is pandigital if it makes use of all the digits 1 to n exactly once.
For example, 2143 is a 4-digit pandigital and is also prime.
What is the largest n-digit pandigital prime that exists?
"""
# permute and sort with the largest numbers first
for i in range(9, 0, -1):
PERMS = [utils.join(x) for x in utils.permute(range(1, i))]
PERMS = sorted(PERMS, reverse=True)
for p in PERMS:
if utils.is_prime(p):
print(p)
raise StopIteration
def encode_decode(self):
embedding = embed_seq(
input_seq=self.input_,
from_=self.dimension,
to_=self.input_embed,
is_training=self.is_training,
BN=True,
initializer=tf.contrib.layers.xavier_initializer())
encoding = encode_seq(input_seq=embedding,
graph_tensor=self.graph_structure,
input_dim=self.input_embed,
num_stacks=self.num_stacks,
num_heads=self.num_heads,
num_neurons=self.num_neurons,
is_training=self.is_training,
version=self.version)
n_hidden = encoding.get_shape().as_list()[2] # input_embed
if not self.is_training:
encoding = tf.tile(encoding, [self.batch_size, 1, 1])
with tf.variable_scope("Decoder"):
W_ref = tf.get_variable("W_ref", [1, n_hidden, self.num_units],
initializer=self.initializer)
encoded_ref = tf.nn.conv1d(
encoding, W_ref, 1,
"VALID") # [Batch size, seq_length, n_hidden]
self.encoded_ref = tf.expand_dims(encoded_ref, 1)
# initialize weights for pointer
W_q = tf.get_variable("W_q", [self.query_dim, self.num_units],
initializer=self.initializer)
self.v = tf.get_variable("v", [self.num_units],
initializer=self.initializer)
# self.v_g = tf.get_variable("v_g", [1], initializer=self.initializer)
# initialize weights for query
W_1 = tf.get_variable("W_1", [n_hidden, self.query_dim],
initializer=tf.initializers.he_normal())
W_2 = tf.get_variable("W_2", [n_hidden, self.query_dim],
initializer=tf.initializers.he_normal())
W_3 = tf.get_variable("W_3", [n_hidden, self.query_dim],
initializer=tf.initializers.he_normal())
if self.is_training:
# initialize query vectors
indices_tours = vectorize_indices(self.optimal_tour,
self.batch_size,
for_sl=True)
tour_embeddings = tf.gather_nd(encoding, indices_tours)
tour_embeddings = tf.reshape(tour_embeddings,
[-1, self.max_length, n_hidden])
present_city = tf.tensordot(tour_embeddings, W_1, axes=1)
previous_city = tf.tensordot(tour_embeddings, W_2, axes=1)
previous_city = permute(previous_city, -1)
previous2_city = tf.tensordot(tour_embeddings, W_3, axes=1)
previous2_city = permute(previous2_city, -2)
# query = tf.nn.relu(present_city + previous_city + previous2_city)
query = tf.nn.relu(present_city + previous_city)
self.logits1 = pointer_for_sl(encoded_ref=self.encoded_ref,
query=query,
W_q=W_q,
v=self.v,
C=self.C,
temperature=self.temperature)
prob = distributions.Categorical(
self.logits1) # logits = masked_scores
self.idx = prob.sample()
idx_ = vectorize_indices(self.idx,
self.batch_size,
for_sl=True)
next_city = tf.gather_nd(encoding,
idx_) # update trajectory (state)
next_city = tf.reshape(next_city,
[-1, self.max_length, n_hidden])
present_city = tf.tensordot(next_city, W_1, axes=1)
previous_city = tf.tensordot(tour_embeddings, W_2, axes=1)
previous2_city = tf.tensordot(tour_embeddings, W_3, axes=1)
previous2_city = permute(previous2_city, -1)
# query2 = tf.nn.relu(present_city + previous_city + previous2_city)
query2 = tf.nn.relu(present_city + previous_city)
self.logits2 = pointer_for_sl(encoded_ref=self.encoded_ref,
query=query2,
W_q=W_q,
v=self.v,
C=self.C,
temperature=self.temperature)
self.log_prob_mean = tf.reduce_mean(prob.log_prob(self.idx))
self.entropies_mean = tf.reduce_mean(prob.entropy())
tf.summary.scalar('log_prob_mean', self.log_prob_mean)
tf.summary.scalar('entropies_mean', self.entropies_mean)
else:
indices_current = vectorize_indices(self.current,
self.batch_size)
encoding_current = tf.gather_nd(encoding, indices_current)
present_city = tf.tensordot(encoding_current, W_1, axes=1)
indices_previous = vectorize_indices(self.previous,
self.batch_size)
encoding_previous = tf.gather_nd(encoding, indices_previous)
previous_city = tf.tensordot(encoding_previous, W_2, axes=1)
indices_previous2 = vectorize_indices(self.previous2,
self.batch_size)
encoding_previous2 = tf.gather_nd(encoding, indices_previous2)
previous2_city = tf.tensordot(encoding_previous2, W_3, axes=1)
# query = tf.nn.relu(present_city + previous_city + previous2_city)
query = tf.nn.relu(present_city + previous_city)
self.logits1 = pointer_for_sl(encoded_ref=self.encoded_ref,
query=query,
W_q=W_q,
v=self.v,
C=self.C,
temperature=self.temperature,
training=False)
def sample_site(sigma,mu,Ne,L):
phats = [phat(k,sigma,mu,Ne,L) for k in range(L+1)]
# Z = sum(phats)
# ps = [ph/Z for ph in phats]
k = inverse_cdf_sample(range(L+1), phats,normalized=False)
return "".join(permute(["A" for _ in range(L-k)] + [random.choice("CGT") for _ in range(k)]))
def sample_col_from_count(count):
return permute(
concat([[base] * n for base, n in zip(permute("ACGT"), count)]))
def mate(self):
"""produce segments based on crossing over parents'"""
for chrnum in xrange(1, 23):
self.segments[chrnum] = utils.permute(
(crossOver(chrnum, *self.lparent.segments[chrnum]), crossOver(chrnum, *self.rparent.segments[chrnum]))
)
def sortDatasetByMass(spectra):
"""
Orders all data points by m/z for fast image generation.
Does so in a memory-efficient fashion.
Parameters
----------
spectra: iterator over (index, mzs, intensities) tuples
Returns
-------
ret : (ndarray, ndarray, ndarray)
Arrays containing ordered m/z values, intensities and
spectrum indices.
"""
mzs_list = None
count_list = None
indices = []
reps = []
mzs_dtype = intensities_dtype = None
def typecode(numpy_array):
dtype = numpy_array.dtype
if dtype == np.float64:
return 'd'
elif dtype == np.float32:
return 'f'
else:
raise "unsupported data type"
for index, mzs, intensities in spectra:
indices.append(index)
reps.append(len(mzs))
if not mzs_list:
mzs_dtype = mzs.dtype
intensities_dtype = intensities.dtype
mzs_list = array.array(typecode(mzs))
count_list = array.array(typecode(intensities))
mzs_list.extend(mzs)
count_list.extend(intensities)
# convert array.arrays into numpy arrays (without copying)
mzs_list = np.frombuffer(mzs_list, mzs_dtype)
count_list = np.frombuffer(count_list, intensities_dtype)
# np.int64 branch will probably never be tested...
order_dtype = np.int64 if len(mzs_list) >= 2**31 else np.int32
order = np.arange(len(mzs_list), dtype=order_dtype)
# sort mzs_list and set the permutation accordingly
utils.keysort(mzs_list, order)
# rearrange intensities and indices as well
indices = np.array(indices, np.int32)
reps = np.array(reps, np.uint64)
utils.permute(count_list, order)
utils.permuterepeat(reps, indices, order)
idx_list = order
return indices, mzs_list, count_list, idx_list
import utils
from sets import Set
"""
Pandigital products
Problem 32
We shall say that an n-digit number is pandigital if it makes use of all the digits 1 to n exactly once;
for example, the 5-digit number, 15234, is 1 through 5 pandigital.
7245 is unusual, 39 x 186 = 7254, multiplicand, multiplier and product is 1 thru 9 pandigital
HINT: Some products can be obtained in more than one way so be sure to only include it once in your sum.
"""
PRODUCTS = Set()
PANS = utils.permute(range(1, 10))
for p in PANS:
for c in range(1, 6): # multiplicand
for d in range(c + 1, 7): # product
multiplicand = utils.join(p[0:c])
multiplier = utils.join(p[c:d])
product = utils.join(p[d:])
if multiplicand * multiplier == product:
PRODUCTS.add(product)
print reduce(lambda x, y: x + y, PRODUCTS)
def sample_site_from_copies(sigma,Ne,L,copies,ps=None):
if ps is None:
ps = ps_from_copies(sigma, Ne, L, copies)
k = inverse_cdf_sample(range(L+1), ps,normalized=False)
return "".join(permute(["A" for _ in range(L-k)] + [random.choice("CGT") for _ in range(k)]))
def sample_motif_from_mismatches(cs, L):
return [
"".join(
permute(['A'] * (L - c) + [random.choice("CGT")
for i in range(c)])) for c in cs
]