def run(self, T=10, T_min=0.04, n_sweeps=1000): temperatures = self.generate_temperatures(T, T_min) if isinstance( T, int) else T self.cost = self.cost(self.solution) if self.plot: energies = [self.cost] hammings = [hamming_distance(self.solution, self.config)] # swaps = np.arange(0, len(self.solution)) swaps = [i for i, val in enumerate(self.solution)] for T in temperatures: start = time.time() accept_probs = self.get_accept_probs(T) for sweep in xrange(n_sweeps): self.shuffle(swaps) for swap in swaps: new_sol, new_cost = self.update_solution(swap) diff = int(self.cost - new_cost) if diff >= 0 or self.random() < accept_probs[diff]: self.solution = new_sol self.cost = new_cost if self.plot: energies.append(self.cost) hammings.append( hamming_distance(self.solution, self.config)) print T, ': ', 'Current best: ', self.cost print 'timing', time.time() - start if self.plot: self.save_plot(energies, T, hammings) return (self.solution, self.cost)
def reassign_template_counts(self, msa_info, new_alleles, debug=False): # XXX need to update family_groups here if len(new_alleles) == 0: return if debug: print ' template new' print ' size snps snps assigned', if self.reco_info is not None: print ' true', print '' dbg_print = debug # don't print all the tiny clusters templates = {newfo['template-gene'] : newfo['gene'] for newfo in new_alleles.values()} self.adjusted_glcounts = {} for clusterfo in sorted(msa_info, key=lambda cfo: len(cfo['seqfos']), reverse=True): sorted_glcounts, true_sorted_glcounts = self.get_glcounts(clusterfo) # it would be nice to not re-call this for the clusters we already called it on above for gene, counts in sorted_glcounts: # <gene> is the one assigned by sw before allele clustering if debug and len(clusterfo['seqfos']) < 5: if dbg_print: print ' not printing clusters smaller than 5' dbg_print = False if gene not in self.adjusted_glcounts: # add it before we decide whether to switch it, so a template gene with zero counts will be in there with zero counts self.adjusted_glcounts[gene] = 0 if gene in templates: # if this was a template for a new allele, we have to decide whether to apportion some or all of the sequences in this cluster to that new allele template_gene = gene template_cpos = utils.cdn_pos(self.glfo, self.region, template_gene) cons_seq = clusterfo['cons_seq'] template_seq = self.glfo['seqs'][self.region][template_gene] new_allele_seq = new_alleles[templates[template_gene]]['seq'] compare_len = min([template_cpos, len(cons_seq), len(template_seq), len(new_allele_seq)]) # NOTE this doesn't account for indels, i.e. the template and consensus sequences are in general different lengths, but that's ok, it'll just inflate the hamming distance for sequences that differ from consensus by indels, and all we care is finding the one that doesn't have any indels n_template_snps = utils.hamming_distance(cons_seq[:compare_len], template_seq[:compare_len]) n_new_snps = utils.hamming_distance(cons_seq[:compare_len], new_allele_seq[:compare_len]) if debug and dbg_print: print ' %5d %3d %3d' % (len(clusterfo['seqfos']), n_template_snps, n_new_snps), if n_new_snps < n_template_snps: # reassign to the new allele gene = templates[template_gene] if gene not in self.adjusted_glcounts: # add it before we decide whether to switch it, so a template gene with zero counts will be in there with zero counts self.adjusted_glcounts[gene] = 0 if debug and dbg_print: print ' %s' % utils.color_gene(gene, width=15), if self.reco_info is not None: true_gene = true_sorted_glcounts[0][0] # NOTE this is the most *common* simulated gene in the cluster, not necessarily the one corresponding to these particular sequences... but clusters with new alleles should generally be dominated by one gene, so oh, well if true_gene == gene: print ' %s' % utils.color('green', 'ok'), else: print ' %s' % utils.color_gene(true_gene, width=15), print '' self.adjusted_glcounts[gene] += counts if debug: print ' final counts:' for gene, counts in sorted(self.adjusted_glcounts.items(), key=operator.itemgetter(1), reverse=True): print ' %4d %s' % (counts, utils.color_gene(gene))
def finalize(self, sorted_gene_counts, debug=False): # NOTE <sorted_gene_counts> is usually/always floats instead of integers assert not self.finalized easycounts = {gene : counts for gene, counts in sorted_gene_counts} total_counts = sum([counts for counts in easycounts.values()]) self.genes_to_keep = set() if debug: print ' removing least likely genes (%.1f total counts)' % total_counts print ' %-20s %5s (%s) removed genes (counts)' % ('genes to keep', 'counts', 'snps'), def count_str(cnt): if cnt < 10.: return '%.1f' % cnt else: return '%.0f' % cnt class_counts = self.separate_into_classes(sorted_gene_counts, easycounts) for iclass in range(len(class_counts)): gclass = class_counts[iclass] n_from_this_class = 0 for ig in range(len(gclass)): gfo = gclass[ig] if self.args.n_max_total_alleles is not None and len(self.genes_to_keep) >= self.args.n_max_total_alleles: # command line can specify the total number of alleles break if float(gfo['counts']) / total_counts < self.args.min_allele_prevalence_fraction: # always skip everybody that's super uncommon pass elif ig == 0: # keep the first one from this class self.genes_to_keep.add(gfo['gene']) n_from_this_class += 1 elif utils.hamming_distance(gclass[0]['seq'], gclass[ig]['seq']) == 0: # don't keep it if it's indistinguishable from the most common one (the matches are probably mostly really the best one) pass # don't keep it elif n_from_this_class < self.args.n_alleles_per_gene: # always keep the most common <self.args.n_alleles_per_gene> in each class self.genes_to_keep.add(gfo['gene']) n_from_this_class += 1 else: pass # don't keep it if debug and gfo['gene'] in self.genes_to_keep: snpstr = ' ' if ig == 0 else '(%d)' % utils.hamming_distance(gclass[0]['seq'], gfo['seq']) print '\n %-s %7s %-3s' % (utils.color_gene(gfo['gene'], width=20), count_str(gfo['counts']), snpstr), if debug: if n_from_this_class == 0: print '\n %-s %7s %-3s' % (utils.color('blue', 'none', width=20, padside='right'), '-', ''), removedfo = [gfo for gfo in gclass if gfo['gene'] not in self.genes_to_keep] if len(removedfo) > 0: removal_strs = ['%s (%s)' % (utils.color_gene(gfo['gene']), count_str(gfo['counts'])) for gfo in removedfo] print ' %s' % ' '.join(removal_strs), if debug: print '' self.genes_to_remove = set(self.glfo['seqs'][self.region]) - self.genes_to_keep print ' keeping %d / %d %s gene%s' % (len(self.genes_to_keep), len(self.glfo['seqs'][self.region]), self.region, utils.plural(len(self.genes_to_keep))) # print ' removing %d %s genes: %d with no matches, %d with unconvincing matches' % (len(self.genes_to_remove), self.region, len(set(self.glfo['seqs'][self.region]) - set(easycounts)), len(set(easycounts) - self.genes_to_keep)) self.finalized = True
def nao_valid(queue, model): pa = utils.AvgrageMeter() hs = utils.AvgrageMeter() with torch.no_grad(): model.eval() for step, sample in enumerate(queue): encoder_input = sample['encoder_input'] encoder_target = sample['encoder_target'] decoder_target = sample['decoder_target'] encoder_input = encoder_input.cuda() encoder_target = encoder_target.cuda() decoder_target = decoder_target.cuda() predict_value, logits, arch = model(encoder_input) n = encoder_input.size(0) pairwise_acc = utils.pairwise_accuracy( encoder_target.data.squeeze().tolist(), predict_value.data.squeeze().tolist()) hamming_dis = utils.hamming_distance( decoder_target.data.squeeze().tolist(), arch.data.squeeze().tolist()) pa.update(pairwise_acc, n) hs.update(hamming_dis, n) return pa.avg, hs.avg
def naive_hdist_or_none(line1, line2): if line1['cdr3_length'] != line2['cdr3_length']: return None hdist = utils.hamming_distance(naive_cdr3(line1), naive_cdr3(line2)) if hdist > args.max_cdr3_distance: return None return hdist
def keep_this_gene(self, this_gene, pcounter, easycounts, debug=False): assert self.region == 'v' # conserved codon stuff below will have to be changed for j glseqs = self.glfo['seqs'][self.region] this_seq = glseqs[this_gene][:self.codon_positions[this_gene] + 3] # only compare up to the conserved cysteine # don't keep it if it's pretty close to a gene we already have n_close_genes = 0 nearest_gene, nearest_hdist = None, None for kgene in self.genes_to_keep: kseq = glseqs[kgene][:self.codon_positions[kgene] + 3] if len(kseq) != len(this_seq): continue hdist = utils.hamming_distance(kseq, this_seq) if nearest_hdist is None or hdist < nearest_hdist: nearest_hdist = hdist nearest_gene = kgene if hdist < self.args.n_max_snps - 1: n_close_genes += 1 if easycounts[this_gene] < self.alfinder.n_total_min: # if we hardly ever saw it, there's no good reason to believe it wasn't the result of just mutational wandering self.dbg_strings[this_gene] = 'not enough counts (%d < %d)' % (easycounts[this_gene], self.alfinder.n_total_min) return False self.dbg_strings[this_gene] = 'nearest gene %s %s' % (nearest_gene, nearest_hdist) return True
def nao_valid(queue, model): inputs = [] targets = [] predictions = [] archs = [] with torch.no_grad(): model.eval() for step, sample in enumerate(queue): encoder_input = sample['encoder_input'] encoder_target = sample['encoder_target'] decoder_target = sample['decoder_target'] encoder_input = encoder_input.cuda() encoder_target = encoder_target.cuda() decoder_target = decoder_target.cuda() predict_value, logits, arch = model(encoder_input) n = encoder_input.size(0) inputs += encoder_input.data.squeeze().tolist() targets += encoder_target.data.squeeze().tolist() predictions += predict_value.data.squeeze().tolist() archs += arch.data.squeeze().tolist() pa = utils.pairwise_accuracy(targets, predictions) hd = utils.hamming_distance(inputs, archs) return pa, hd
def test_hamming_distance_between_base_case_strings_as_specified_in_problem_spec( self): from utils import hamming_distance from data_utils import convert_string_to_binary_string self.assertEqual(37,hamming_distance(convert_string_to_binary_string('this is a test'),\ convert_string_to_binary_string('wokka wokka!!!')))
def rate_key_size(key_size, ciphertext): dist = 0 for block_1, block_2 in zip(chunks(ciphertext, key_size), chunks(ciphertext, key_size)[1:]): dist += hamming_distance(block_1, block_2) dist /= len(ciphertext) / key_size normalized = dist / key_size return normalized
def find(self, NAP): crude_hash = utils.d_hash(NAP, hash_size=8) fine_hash = utils.d_hash(NAP, hash_size=16) clean_key, overlap_key = self._digitize(NAP) unsorted = self.NAP_intervals[clean_key] if clean_key in self.NAP_intervals else [] + self.NAP_intervals[overlap_key] if overlap_key in self.NAP_intervals else [] return sorted(unsorted, key = lambda x: utils.hamming_distance(fine_hash, x.fine_hash))[0] if len(unsorted) else [], crude_hash, fine_hash, clean_key, overlap_key
def too_close_to_already_added_gene(self, new_seq, new_alleles, debug=False): for added_name, added_info in new_alleles.items(): n_snps = utils.hamming_distance(added_info['seq'], new_seq, align=True) if n_snps < self.min_n_snps or n_snps < self.args.n_max_snps: if debug: print 'too close (%d snp%s) to gene we just added %s' % (n_snps, utils.plural(n_snps), utils.color_gene(added_name)) return True return False
def choose_seqs_to_remove( chain_ids, max_hdist=4, tdbg=False): # choose one of <chain_ids> to eliminate # look for pairs with the same locus that ids_to_remove = set(u for u in chain_ids if getloc(u) == '?') if tdbg and len( ids_to_remove ) > 0: # i think this actually can't happen a.t.m. TODO maybe remove it print ' removed %d with missing annotations' % len( ids_to_remove) dbgstr = [] n_equivalent = 0 for tpair in itertools.combinations(chain_ids, 2): if len(set(getloc(u) for u in tpair)) > 1: continue if len(set(len(gval(u, 'seqs')) for u in tpair)) > 1: continue hdist = utils.hamming_distance(*[gval(u, 'seqs') for u in tpair]) if tdbg: dbgstr.append( utils.color('blue' if hdist == 0 else 'yellow', '%d' % hdist)) if hdist <= max_hdist: # TODO would be nice to be able to combine their sequences, but I think propagating the resulting annotation modifications would be hard # print ' identical sequence overlap, choosing longer one' better_id, worse_id = sorted( tpair, key=lambda q: utils.ambig_frac(gval(q, 'seqs')) ) # TODO if we're tossing one with hdist > 0, maybe should take the lower-shm one if they're the same length? ids_to_remove.add(worse_id) n_equivalent += 1 if tdbg and len(dbgstr) > 0: print ' %d pair%s equivalent with hdists %s' % ( n_equivalent, utils.plural(n_equivalent), ' '.join(dbgstr)) # remove unproductive dbgstr = [] unproductive_ids = [] for uid in chain_ids: if not utils.is_functional( all_antns[uid], all_antns[uid]['unique_ids'].index(uid)): unproductive_ids.append(uid) if tdbg: dbgstr.append( utils.is_functional_dbg_str( all_antns[uid], all_antns[uid]['unique_ids'].index(uid), sep='+')) # unproductive_ids = [u for u in chain_ids if not utils.is_functional(all_antns[u], all_antns[u]['unique_ids'].index(u))] # this way is only one line, which may or may not be nicer if tdbg and len(unproductive_ids) > 0: print ' %d unproductive %s' % (len(unproductive_ids), ', '.join(dbgstr)) ids_to_remove |= set(unproductive_ids) return ids_to_remove
def build_v_gene_set(glfo, introns): total_d_counts = {} refseqs = {} for d_gene, counts in introns.items(): total_d_counts[d_gene] = sum(counts.values()) for d_gene, _ in sorted(total_d_counts.items(), key=operator.itemgetter(1), reverse=True): counts = introns[d_gene] # first decide on the reference sequences refseq, column_counts = None, None for seq in sorted(counts, key=len, reverse=True): if refseq is None: # first one, i.e. the longest refseq = seq column_counts = [{n : 0 for n in utils.nukes} for i in range(len(refseq))] ioffset = len(refseq) - len(seq) partial_refseq = refseq[ioffset:] assert len(partial_refseq) == len(seq) for ibase in range(ioffset, len(refseq)): column_counts[ibase][seq[ibase - ioffset]] += counts[seq] refseqs[d_gene] = [] for basecounts in column_counts: most_common_base = sorted(basecounts.items(), key=operator.itemgetter(1), reverse=True)[0][0] refseqs[d_gene].append(most_common_base) refseqs[d_gene] = ''.join(refseqs[d_gene]) n_ok = 0 mutecounts = {} for seq in sorted(counts, key=len, reverse=True): # print ' %3d %150s' % (count, seq) partial_refseq = refseqs[d_gene][len(refseqs[d_gene]) - len(seq):] if seq == partial_refseq: n_ok += counts[seq] else: # utils.color_mutants(partial_refseq, seq, print_result=True, extra_str=' ') n_mutes = utils.hamming_distance(partial_refseq, seq) if n_mutes not in mutecounts: mutecounts[n_mutes] = 0 mutecounts[n_mutes] += counts[seq] print ' %s %4d / %-4d ok' % (utils.color_gene(d_gene, width=10), n_ok, n_ok + sum(mutecounts.values())), if len(mutecounts) > 0: print '(mean of %.1f mutations among the other %d' % (numpy.average(mutecounts.keys(), weights=mutecounts.values()), sum(mutecounts.values())), print '' # add the intronic v genes to glfo for d_gene, refseq in refseqs.items(): glfo['seqs']['v'][utils.generate_dummy_v(d_gene)] = refseq glfo['cyst-positions'][utils.generate_dummy_v(d_gene)] = len(refseq) - 3 # write a glfo dir with everything glutils.write_glfo(outdir + '/germlines/imgt-and-intronic', glfo, debug=True) # remove the original v genes, and write a glfo dir with just the intronic ones glutils.remove_genes(glfo, [g for g in glfo['seqs']['v'] if 'xDx' not in g], debug=True) glutils.write_glfo(outdir + '/germlines/intronic', glfo, debug=True)
def test_barcodes(self, barcode_set): ''' validate stored barcodes to check if they are at leasts ${min_distance} nucleotide apart ''' for bc1 in barcode_set: for bc2 in barcode_set: if bc1 != bc2: assert hamming_distance(bc1, bc2) >= self.min_distance, \ 'Bad barcodes %s and %s' %(bc1,bc2) print('All barcode had at least %i nucleotide distance apart' % self.min_distance, file=sys.stderr)
def c6(): str1 = "this is a test" str2 = "wokka wokka!!!" result = utils.hamming_distance(str1, str2) expect(37.0, result) filename = "data/6.txt" ciphertext = utils.get_ciphertext(filename) result = cryptopals.break_repeating_key_xor(ciphertext) print print result[1] print result[0]
def guess_inflection(self, lemma, best_node): ''' :lemma: the lemma to inflect :best_node: the deepest node logically compatible with the lemma. :return: an inflected form using the nearest-neighbor at the :best_node: (using Hamming distance). ''' options = sorted(best_node.switch_statement.vocab, key=lambda it: hamming_distance(lemma, it[0])) closest_lemma, closest_inflected = options[0][:-1] suffix_of_closest = closest_inflected[len(closest_lemma):] return f'{lemma}{suffix_of_closest}'
def approx_pattern_matching(pattern: str, genome: str, d: int) -> List[int]: """Find locations of k-mer (pattern') in the genome with hamming_distance(pattern, pattern') <= d >>> approx_pattern_matching("ATTCTGGA", "CGCCCGAATCCAGAACGCATTCCCATATTTCGGGACCACTGGCCTCCACGGTACGGACGTCAATCAAAT", 3) [6, 7, 26, 27] """ intervals = sliding_window(genome, len(pattern)) return [ i for i, interval in enumerate(intervals) if hamming_distance(pattern, interval) <= d ]
def too_close_to_existing_glfo_gene(self, clusterfo, new_seq, template_seq, template_cpos, template_gene, debug=False): if len(new_seq[:template_cpos]) != len(template_seq[:template_cpos]): # TODO update this to use the new n_snps from the aligned template/new seqs return False mean_j_mutations = numpy.mean([self.all_j_mutations[seqfo['name']] for seqfo in clusterfo['seqfos']]) # TODO <self.all_j_mutations> uses everybody in the cluster, rather than just the representative. It'd be nice to synchronize this with other things # TODO should probably update this to do the same thing (with min([])) as up in decide_whether_to_remove_template_genes(), or just use the new <align> option to utils.hamming_distance (although that would be slower, and this seems to work ok) pre_cpos_snps = utils.hamming_distance(new_seq[:template_cpos], template_seq[:template_cpos]) factor = 1.75 if pre_cpos_snps < self.min_n_snps or pre_cpos_snps < factor * mean_j_mutations: # i.e. we keep if it's *further* than factor * <number of j mutations> from the closest existing allele (should presumably rescale by some factor to go from j --> v, but it seems like the factor's near to 1.) if debug: print 'too close to existing glfo gene %s (%d snp%s < %.2f = %.2f * %.1f mean j mutation%s)' % (utils.color_gene(template_gene), pre_cpos_snps, utils.plural(pre_cpos_snps), factor * mean_j_mutations, factor, mean_j_mutations, utils.plural(mean_j_mutations)) return True return False
def learn(self, NAP, wav_file, segment_idxs): best_match, crude_hash, fine_hash, clean_key, overlap_key = self.find(NAP) if not len(best_match): audio_id = self.audio_id_counter print 'New audio_id {}, never before heard length {}'.format(audio_id, NAP.shape[0]) self.audio_id_counter += 1 else: if utils.hamming_distance(crude_hash, best_match.crude_hash) < AUDIO_HAMMERTIME: audio_id = best_match.audio_id print 'Similar to audio_id {}, hamming distance {}'.format(audio_id, utils.hamming_distance(crude_hash, best_match.crude_hash)) else: audio_id = self.audio_id_counter self.audio_id_counter += 1 print 'New audio_id {}, hamming distance {} from audio_id {}'.format(audio_id, utils.hamming_distance(crude_hash, best_match.crude_hash), best_match.audio_id) audio_segment = AudioSegment(audio_id, crude_hash, fine_hash, wav_file, segment_idxs) self._insert(self.NAP_intervals, clean_key, audio_segment) self._insert(self.NAP_intervals, overlap_key, audio_segment) self._insert(self.audio_ids, audio_id, audio_segment) return audio_id
def hamming_to_true_naive(self, true_line, line, restrict_to_region=''): true_naive_seq = true_line['naive_seq'] inferred_naive_seq = line['naive_seq'] if len(true_naive_seq) != len(inferred_naive_seq): raise Exception('different length true and inferred naive seqs for %s\n %s\n %s' % (' '.join(line['unique_ids']), true_line['naive_seq'], line['naive_seq'])) if restrict_to_region != '': # NOTE very similar to utils.get_n_muted(), except, we want to use the true bounds for both true and naive sequences if restrict_to_region in utils.regions: bounds = true_line['regional_bounds'][restrict_to_region] elif restrict_to_region == 'cdr3': bounds = (true_line['codon_positions']['v'], true_line['codon_positions']['j'] + 3) else: assert False true_naive_seq = true_naive_seq[bounds[0] : bounds[1]] inferred_naive_seq = inferred_naive_seq[bounds[0] : bounds[1]] return utils.hamming_distance(true_naive_seq, inferred_naive_seq)
def hamming_to_true_naive(self, true_line, line, restrict_to_region=''): true_naive_seq = true_line['naive_seq'] inferred_naive_seq = line['naive_seq'] if len(true_naive_seq) != len(inferred_naive_seq): raise Exception('different length true and inferred naive seqs for %s\n %s\n %s' % (' '.join(line['unique_ids']), true_line['naive_seq'], line['naive_seq'])) if restrict_to_region != '': # NOTE very similar to utils.get_n_muted(), except, we want to use the true bounds for both true and naive sequences if restrict_to_region in utils.regions: bounds = true_line['regional_bounds'][restrict_to_region] elif restrict_to_region == 'cdr3': bounds = (true_line['codon_positions']['v'], true_line['codon_positions']['j'] + 3) else: print 'invalid regional restriction %s' % restrict_to_region true_naive_seq = true_naive_seq[bounds[0] : bounds[1]] inferred_naive_seq = inferred_naive_seq[bounds[0] : bounds[1]] return utils.hamming_distance(true_naive_seq, inferred_naive_seq)
def hamming_to_true_naive(self, true_line, line, restrict_to_region=''): true_naive_seq, inferred_naive_seq = self.harmonize_naive_seq_lengths( true_line, line) if restrict_to_region != '': # NOTE very similar to utils.get_n_muted(), except, we want to use the true bounds for both true and naive sequences if restrict_to_region in utils.regions: bounds = true_line['regional_bounds'][restrict_to_region] elif restrict_to_region == 'cdr3': bounds = (true_line['codon_positions']['v'], true_line['codon_positions']['j'] + 3) else: print 'invalid regional restriction %s' % restrict_to_region if restrict_to_region == 'v': # NOTE this is kind of hackey, especially treating v differently to d and j, but it kind of makes sense -- v is fundamentally different in that germline v is a real source of diversity, so it makes sense to isolate the v germline accuracy from the boundary-call accuracy like this bounds = ( bounds[0], max(bounds[0], bounds[1] - self.v_3p_exclusion) ) # most of the boundary uncertainty is in the last three bases true_naive_seq = true_naive_seq[bounds[0]:bounds[1]] inferred_naive_seq = inferred_naive_seq[bounds[0]:bounds[1]] return utils.hamming_distance(true_naive_seq, inferred_naive_seq)
def nao_valid(queue, model): pa = utils.AvgMeter() hs = utils.AvgMeter() mse = utils.AvgMeter() for step, sample in enumerate(queue): encoder_input = sample['encoder_input'] encoder_target = sample['encoder_target'] decoder_target = sample['decoder_target'] predict_value, logits, arch = model(encoder_input) n = encoder_input.shape[0] pairwise_acc = utils.pairwise_accuracy(encoder_target.data.squeeze().tolist(), predict_value.data.squeeze().tolist()) hamming_dis = utils.hamming_distance(decoder_target.data.squeeze().tolist(), arch.data.squeeze().tolist()) mse.update(keras.losses.MSE(encoder_target.data.squeeze(), predict_value.data.squeeze()), n) pa.update(pairwise_acc, n) hs.update(hamming_dis, n) return mse.avg, pa.avg, hs.avg
def solve_(self, problem: CSProblem) -> Tuple[str, dict]: m, n, strings, alphabet = problem.m, problem.n, problem.strings, problem.alphabet i, j = max(combinations(range(n), 2), key=lambda coords: utils.hamming_distance( strings[coords[0]], strings[coords[1]])) si = strings[i] sj = strings[j] P = utils.P(si, sj) Q = utils.Q(si, sj) k = len(P) # Number of positions that they disagree on epsilon = self.config['epsilon'] sigma = self.config['sigma'] solve_func = solve_by_lp_relaxation lp_used = True decision_measure = (6 * math.log(sigma * m)) // (epsilon**2) if k <= decision_measure: solve_func = solve_by_force lp_used = False # print(f'Decision: |P| = {k}', file=sys.stderr) # print(f'Measure = {decision_measure}', file=sys.stderr) # print(f'Meaning {"FORCE" if not lp_used else "LP"}', file=sys.stderr) ss = solve_func(P, Q, alphabet, m, n, strings, si) if ss is None: # print('not stonks?') return si, {'lp_used': lp_used, 'orig': True} new_sol, new_sol_metric = ss # print(f'Found sol: {new_sol}') if new_sol_metric < k: return new_sol, {'lp_used': lp_used, 'orig': False} return si, {'lp_used': lp_used, 'orig': True}
def separate_into_classes(self, sorted_gene_counts, easycounts): class_counts = [] for gene, counts in sorted_gene_counts: seq = self.glfo['seqs'][self.region][gene][:self.codon_positions[gene] + 3] add_new_class = True for gclass in class_counts: for gfo in gclass: if len(gfo['seq']) != len(seq): continue hdist = utils.hamming_distance(gfo['seq'], seq) if hdist < self.args.n_max_snps - 1: # if this gene is close to any gene in the class, add it to this class add_new_class = False class_counts[class_counts.index(gclass)].append({'gene' : gene, 'counts' : counts, 'seq' : seq}) break if not add_new_class: break if add_new_class: class_counts.append([{'gene' : gene, 'counts' : counts, 'seq' : seq}, ]) return class_counts
#!/usr/bin/env python import utils utils.test(utils.hamming_distance(utils.str_to_bin('this is a test'), utils.str_to_bin('wokka wokka!!!')), 37) with open('files/6.txt') as f: text = f.read().decode('base64') bin_text = utils.str_to_bin(text) bin_text_arr = bin_text.split() min_distance = 0 sizes = [] keysizes = [] for keysize in range(2, 41): # first KEYSIZE worth of bytes, and the second KEYSIZE worth of bytes info = [bin_text_arr[i:i+keysize] for i in range(0, len(bin_text_arr), keysize)] total_normalized_distance = 0 # average the hamming distance between sets of bytes for a, b in zip(info[::2], info[1::2]): edit_distance = utils.hamming_distance(''.join(a), ''.join(b)) # Normalize this result by dividing by KEYSIZE. total_normalized_distance += float(edit_distance) / float(keysize) avg_distance = float(total_normalized_distance) / float(len(info))
def finalize(self, pcounter, swfo, debug=False): assert not self.finalized sorted_gene_counts = [(deps[0], counts) for deps, counts in sorted(pcounter.counts[self.region + '_gene'].items(), key=operator.itemgetter(1), reverse=True)] easycounts = {gene : counts for gene, counts in sorted_gene_counts} total_counts = sum([counts for counts in easycounts.values()]) self.genes_to_keep = set() if debug: print ' removing least likely alleles (%d total counts)' % total_counts print ' %-20s %5s (%s) removed counts removed genes' % ('genes to keep', 'counts', 'snps'), class_counts = self.separate_into_classes(sorted_gene_counts, easycounts) for iclass in range(len(class_counts)): gclass = class_counts[iclass] n_from_this_class = 0 for ig in range(len(gclass)): gfo = gclass[ig] if self.args.n_max_total_alleles is not None and len(self.genes_to_keep) >= self.args.n_max_total_alleles: # command line can specify the total number of alleles break if float(gfo['counts']) / total_counts < self.args.min_allele_prevalence_fraction: # always skip everybody that's super uncommon pass elif ig == 0: # keep the first one from this class self.genes_to_keep.add(gfo['gene']) n_from_this_class += 1 elif utils.hamming_distance(gclass[0]['seq'], gclass[ig]['seq']) == 0: # don't keep it if it's indistinguishable from the most common one (the matches are probably mostly really the best one) pass # don't keep it elif n_from_this_class < self.args.n_alleles_per_gene: # always keep the most common <self.args.n_alleles_per_gene> in each class self.genes_to_keep.add(gfo['gene']) n_from_this_class += 1 else: pass # don't keep it if debug and gfo['gene'] in self.genes_to_keep: snpstr = ' ' if ig == 0 else '(%d)' % utils.hamming_distance(gclass[0]['seq'], gfo['seq']) print '\n %-s %7d %-3s' % (utils.color_gene(gfo['gene'], width=20), gfo['counts'], snpstr), if debug: if n_from_this_class == 0: print '\n %-s %7s %-3s' % (utils.color('blue', 'none', width=20, padside='right'), '-', ''), removedfo = [gfo for gfo in gclass if gfo['gene'] not in self.genes_to_keep] if len(removedfo) > 0: print ' %5d %s' % (sum([gfo['counts'] for gfo in removedfo]), ' '.join([utils.color_gene(gfo['gene']) for gfo in removedfo])), if debug: print '' # for igene in range(len(sorted_gene_counts)): # gene, counts = sorted_gene_counts[igene] # if igene == 0: # always keep the first one # self.dbg_strings[gene] = 'first gene' # self.genes_to_keep.add(gene) # continue # if self.keep_this_gene(gene, pcounter, easycounts, debug=debug): # self.genes_to_keep.add(gene) # print ' keeping:' # for gene in [g for g, _ in sorted_gene_counts if g in self.genes_to_keep]: # print ' %5d %s %s' % (easycounts[gene], utils.color_gene(gene, width=15), self.dbg_strings[gene]) self.genes_to_remove = set(self.glfo['seqs'][self.region]) - self.genes_to_keep # print ' removing:' # for gene in [g for g, _ in sorted_gene_counts if g in self.genes_to_remove]: # print ' %5d %s %s' % (easycounts[gene], utils.color_gene(gene, width=15), self.dbg_strings[gene]) n_queries_with_removed_genes = 0 for query in swfo['queries']: line = swfo[query] if line[self.region + '_gene'] in self.genes_to_remove: n_queries_with_removed_genes += 1 # unpadded_line = copy.deepcopy(line) # unpadded_line['seqs'][0] = unpadded_line['seqs'][0][unpadded_line['padlefts'][0] : ] # if unpadded_line['padrights'][0] > 0: # unpadded_line['seqs'][0] = unpadded_line['seqs'][0][ : -unpadded_line['padrights'][0]] # utils.print_reco_event(self.glfo['seqs'], unpadded_line) print ' keeping %d %s genes' % (len(self.genes_to_keep), self.region) print ' removing %d %s genes: %d with no matches, %d with unconvincing matches (%d / %d queries had their best match removed)' % (len(self.genes_to_remove), self.region, len(set(self.glfo['seqs'][self.region]) - set(easycounts)), len(set(easycounts) - self.genes_to_keep), n_queries_with_removed_genes, len(swfo['queries'])) self.finalized = True
def new_respond(control_host, learn_host, debug=False): context = zmq.Context() eventQ = context.socket(zmq.SUB) eventQ.connect('tcp://{}:{}'.format(control_host, IO.EVENT)) eventQ.setsockopt(zmq.SUBSCRIBE, b'') projector = context.socket(zmq.PUSH) projector.connect('tcp://{}:{}'.format(control_host, IO.PROJECTOR)) sender = context.socket(zmq.PUSH) sender.connect('tcp://{}:{}'.format(control_host, IO.EXTERNAL)) brainQ = context.socket(zmq.PULL) brainQ.bind('tcp://*:{}'.format(IO.BRAIN)) counterQ = context.socket(zmq.REQ) counterQ.connect('tcp://{}:{}'.format(control_host, IO.COUNTER)) cognitionQ = context.socket(zmq.PUSH) cognitionQ.connect('tcp://{}:{}'.format(control_host, IO.COGNITION)) association = context.socket(zmq.REQ) association.connect('tcp://{}:{}'.format(learn_host, IO.ASSOCIATION)) snapshot = context.socket(zmq.REQ) snapshot.connect('tcp://{}:{}'.format(control_host, IO.SNAPSHOT)) scheduler = context.socket(zmq.PUSH) scheduler.connect('tcp://{}:{}'.format(control_host, IO.SCHEDULER)) dreamQ = context.socket(zmq.PULL) dreamQ.bind('tcp://*:{}'.format(IO.DREAM)) snapshot.send_json('Give me state!') state = snapshot.recv_json() poller = zmq.Poller() poller.register(eventQ, zmq.POLLIN) poller.register(brainQ, zmq.POLLIN) poller.register(dreamQ, zmq.POLLIN) sound_to_face = [] wordFace = {} face_to_sound = [] faceWord = {} register = {} video_producer = {} voiceType1 = 1 voiceType2 = 6 wordSpace1 = 0.3 wordSpaceDev1 = 0.3 wordSpace2 = 0.1 wordSpaceDev2 = 0.3 audio_ids = [] wavs = [] wav_audio_ids = [] NAP_hashes = {} most_significant_audio_id = [] if debug: import matplotlib.pyplot as plt plt.ion() while True: events = dict(poller.poll()) if brainQ in events: cells = brainQ.recv_pyobj() mode = cells[0] wav_file = cells[1] if wav_file not in register: register[wav_file] = [False, False, False] if mode == 'audio_learn': register[wav_file][0] = cells if mode == 'video_learn': register[wav_file][1] = cells if mode == 'face_learn': register[wav_file][2] = cells if all(register[wav_file]): _, _, audio_ids, audio_memory, most_significant_audio_id, wavs, wav_audio_ids = register[wav_file][0] _, _, tarantino = register[wav_file][1] _, _, face_id, face_recognizer = register[wav_file][2] print 'Audio - video - face recognizers related to {} arrived at responder, total processing time {} seconds'.format(wav_file, time.time() - utils.filetime(wav_file)) for audio_id in audio_ids: # If audio_ids is empty, none of this will happen video_producer[(audio_id, face_id)] = tarantino if audio_id < len(sound_to_face) and not face_id in sound_to_face[audio_id]: # sound heard before, but not said by this face sound_to_face[audio_id].append(face_id) if audio_id == len(sound_to_face): sound_to_face.append([face_id]) wordFace.setdefault(audio_id, [[face_id,0]]) found = 0 for item in wordFace[audio_id]: if item[0] == face_id: item[1] += 1 found = 1 if found == 0: wordFace[audio_id].append([face_id,1]) # We can't go from a not known face to any of the sounds, that's just the way it is. print 'face_id for audio segment learned', face_id if face_id is not -1: if face_id < len(face_to_sound) and not audio_id in face_to_sound[face_id]: #face seen before, but the sound is new face_to_sound[face_id].append(audio_id) if face_id == len(face_to_sound): face_to_sound.append([audio_id]) faceWord.setdefault(face_id, [[audio_id,0]]) found = 0 for item in faceWord[face_id]: if item[0] == audio_id: item[1] += 1 found = 1 if found == 0: faceWord[face_id].append([audio_id,1]) del register[wav_file] similar_ids = [] for audio_id in audio_ids: # I SUSPECT THIS IS WRONG, SINCE THERE IS NO SORTING OF THESE HAMMING DISTANCES IN ASSOCIATION.PY new_audio_hash = audio_memory.audio_ids[audio_id][-1].crude_hash similar_ids_for_this_audio_id = [ utils.hamming_distance(new_audio_hash, random.choice(h).crude_hash) for h in audio_memory.audio_ids.itervalues() ] similar_ids.append(similar_ids_for_this_audio_id) if len(audio_ids): association.send_pyobj(['analyze',wav_file,wav_audio_ids,audio_ids,wavs,similar_ids,wordFace,faceWord]) association.recv_pyobj() sender.send_json('last_most_significant_audio_id {}'.format(most_significant_audio_id)) cognitionQ.send_pyobj(face_recognizer) # A possiblity of recognizing a face that is not connecting to any soundfiles if eventQ in events: pushbutton = eventQ.recv_json() if 'respond_single' in pushbutton: try: filename = pushbutton['filename'] audio_segments = utils.get_segments(filename) print 'Single response to {} duration {} seconds with {} segments'.format(filename, audio_segments[-1], len(audio_segments)-1) new_sentence = utils.csv_to_array(filename + 'cochlear') norm_segments = np.rint(new_sentence.shape[0]*audio_segments/audio_segments[-1]).astype('int') segment_id = utils.get_most_significant_word(filename) NAP = utils.trim_right(new_sentence[norm_segments[segment_id]:norm_segments[segment_id+1]]) if debug: plt.imshow(NAP.T, aspect='auto') plt.draw() best_match,_,_,_,_ = audio_memory.find(NAP) soundfile = best_match.wav_file segstart, segend = best_match.segment_idxs voiceChannel = 1 speed = 1 amp = -3 # voice amplitude in dB _,dur,maxamp,_ = utils.getSoundInfo(soundfile) start = 0 voice1 = 'playfile {} {} {} {} {} {} {} {} {}'.format(1, voiceType1, start, soundfile, speed, segstart, segend, amp, maxamp) voice2 = '' print 'Recognized as sound {}'.format(best_match.audio_id) # sound_to_face, video_producer projection = _project(best_match.audio_id, sound_to_face, NAP, video_producer) scheduler.send_pyobj([[ dur, voice1, voice2, projection, FRAME_SIZE ]]) print 'Respond time from creation of wav file was {} seconds'.format(time.time() - utils.filetime(filename)) except: utils.print_exception('Single response aborted.') if 'play_sentence' in pushbutton: try: sentence = pushbutton['sentence'] sentence = eval(sentence) print '*** (play) Play sentence', sentence start = 0 nextTime1 = 0 play_events = [] for i in range(len(sentence)): word_id = sentence[i] soundfile = np.random.choice(wavs[word_id]) speed = 1 segstart, segend = wav_audio_ids[(soundfile, word_id)] NAP = _extract_NAP(segstart, segend, soundfile) amp = -3 # voice amplitude in dB _,totaldur,maxamp,_ = utils.getSoundInfo(soundfile) dur = segend-segstart if dur <= 0: dur = totaldur # play in both voices voice1 = 'playfile {} {} {} {} {} {} {} {} {}'.format(1, voiceType1, start, soundfile, speed, segstart, segend, amp, maxamp) voice2 = 'playfile {} {} {} {} {} {} {} {} {}'.format(2, voiceType1, start, soundfile, speed, segstart, segend, amp, maxamp) wordSpacing1 = wordSpace1 + np.random.random()*wordSpaceDev1 print 'PLAY RESPOND SPACING', wordSpacing1 nextTime1 += (dur/speed)+wordSpacing1 projection = _project(audio_id, sound_to_face, NAP, video_producer) play_events.append([ dur+wordSpacing1, voice1, voice2, projection, FRAME_SIZE ]) scheduler.send_pyobj(play_events) except: utils.print_exception('Sentence play aborted.') if 'respond_sentence' in pushbutton: print 'SENTENCE Respond to', pushbutton['filename'][-12:] try: filename = pushbutton['filename'] audio_segments = utils.get_segments(filename) print 'Sentence response to {} duration {} seconds with {} segments'.format(filename, audio_segments[-1], len(audio_segments)-1) new_sentence = utils.csv_to_array(filename + 'cochlear') norm_segments = np.rint(new_sentence.shape[0]*audio_segments/audio_segments[-1]).astype('int') segment_id = utils.get_most_significant_word(filename) print '**Sentence selected to respond to segment {}'.format(segment_id) NAP = utils.trim_right(new_sentence[norm_segments[segment_id]:norm_segments[segment_id+1]]) best_match,_,_,_,_ = audio_memory.find(NAP) audio_id = best_match.audio_id soundfile = best_match.wav_file numWords = len(audio_segments)-1 print numWords association.send_pyobj(['setParam', 'numWords', numWords ]) association.recv_pyobj() association.send_pyobj(['makeSentence', audio_id]) print 'respond_sentence waiting for association output...', sentence, secondaryStream = association.recv_pyobj() print '*** (respond) Play sentence', sentence, secondaryStream start = 0 nextTime1 = 0 nextTime2 = 0 enableVoice2 = 1 play_events = [] for i in range(len(sentence)): word_id = sentence[i] soundfile = np.random.choice(wavs[word_id]) voiceChannel = 1 speed = 1 # segment start and end within sound file, if zero, play whole file segstart, segend = wav_audio_ids[(soundfile, word_id)] NAP = _extract_NAP(segstart, segend, soundfile) amp = -3 # voice amplitude in dB #totaldur, maxamp = utils.getSoundParmFromFile(soundfile) _,totaldur,maxamp,_ = utils.getSoundInfo(soundfile) dur = segend-segstart if dur <= 0: dur = totaldur voice1 = 'playfile {} {} {} {} {} {} {} {} {}'.format(voiceChannel, voiceType1, start, soundfile, speed, segstart, segend, amp, maxamp) #start += dur # if we want to create a 'score section' for Csound, update start time to make segments into a contiguous sentence wordSpacing1 = wordSpace1 + np.random.random()*wordSpaceDev1 nextTime1 += (dur/speed)+wordSpacing1 #print 'voice 2 ready to play', secondaryStream[i], i if enableVoice2: word_id2 = secondaryStream[i] #print 'voice 2 playing', secondaryStream[i] soundfile2 = np.random.choice(wavs[word_id2]) voiceChannel2 = 2 start2 = 0.7 # set delay between voice 1 and 2 speed2 = 0.7 amp2 = -10 # voice amplitude in dB try: segstart2, segend2 = wav_audio_ids[(soundfile2, word_id2)] dur2 = segend2-segstart2 #totalDur2, maxamp2 = utils.getSoundParmFromFile(soundfile2) _,totalDur2,maxamp2,_ = utils.getSoundInfo(soundfile) if dur2 <= 0: dur2 = totalDur2 voice2 = 'playfile {} {} {} {} {} {} {} {} {}'.format(voiceChannel2, voiceType2, start2, soundfile2, speed2, segstart2, segend2, amp2, maxamp2) wordSpacing2 = wordSpace2 + np.random.random()*wordSpaceDev2 nextTime2 += (dur2/speed2)+wordSpacing2 except: voice2 = '' utils.print_exception('VOICE 2 tried to access an illegal soundfile/audio_id combination.') #enableVoice2 = 0 # trig another word in voice 2 only if word 2 has finished playing (and sync to start of voice 1) if nextTime1 > nextTime2: enableVoice2 = 1 projection = _project(audio_id, sound_to_face, NAP, video_producer) print 'SENTENCE RESPOND SPACING', wordSpacing1 play_events.append([ dur+wordSpacing1, voice1, voice2, projection, FRAME_SIZE ]) scheduler.send_pyobj(play_events) print 'Sentence respond time from creation of wav file was {} seconds'.format(time.time() - utils.filetime(filename)) except: utils.print_exception('Sentence response aborted.') if 'testSentence' in pushbutton: print 'testSentence', pushbutton association.send_pyobj(['makeSentence',int(pushbutton['testSentence'])]) print 'testSentence waiting for association output...' sentence, secondaryStream = association.recv_pyobj() print '*** Test sentence', sentence, secondaryStream if 'assoc_setParam' in pushbutton: try: parm, value = pushbutton['assoc_setParam'].split() association.send_pyobj(['setParam', parm, value ]) association.recv_pyobj() except: utils.print_exception('Assoc set param aborted.') if 'respond_setParam' in pushbutton: items = pushbutton['respond_setParam'].split() if items[0] == 'voiceType': chan = items[1] if chan == '1': voiceType1 = int(items[2]) if chan == '2': voiceType2 = int(items[2]) if items[0] == 'wordSpace': chan = items[1] print 'wordSpace chan', chan, items if chan == '1': wordSpace1 = float(items[2]) if chan == '2': wordSpace2 = float(items[2]) if items[0] == 'wordSpaceDev': chan = items[1] print 'wordSpaceDev1 chan', chan, items if chan == '1': wordSpaceDev1 = float(items[2]) if chan == '2': wordSpaceDev2 = float(items[2]) if 'play_id' in pushbutton: try: items = pushbutton['play_id'].split(' ') if len(items) < 3: print 'PARAMETER ERROR: play_id audio_id voiceChannel voiceType' play_audio_id = int(items[0]) voiceChannel = int(items[1]) voiceType = int(items[2]) print 'play_audio_id', play_audio_id, 'voice', voiceChannel print 'wavs[play_audio_id]', wavs[play_audio_id] #print wavs soundfile = np.random.choice(wavs[play_audio_id]) speed = 1 #print 'wav_audio_ids', wav_audio_ids segstart, segend = wav_audio_ids[(soundfile, play_audio_id)] #segstart = 0 # segment start and end within sound file #segend = 0 # if zero, play whole file amp = -3 # voice amplitude in dB #dur, maxamp = utils.getSoundParmFromFile(soundfile) _,dur,maxamp,_ = utils.getSoundInfo(soundfile) start = 0 sender.send_json('playfile {} {} {} {} {} {} {} {} {}'.format(voiceChannel, voiceType, start, soundfile, speed, segstart, segend, amp, maxamp)) except: utils.print_exception('play_id aborted.') if 'print_me' in pushbutton: # just for inspecting the contents of objects while running print 'printing '+pushbutton['print_me'] if 'brain ' in pushbutton['print_me']: print_variable = pushbutton['print_me'].split('brain ')[-1] try: print eval(print_variable) except Exception, e: print e, 'print_me in brain failed.' elif 'association ' in pushbutton['print_me']: print_variable = pushbutton['print_me'].split('association ')[-1] association.send_pyobj(['print_me',print_variable]) if 'dream' in pushbutton: play_events = [] for audio_segment in audio_memory.all_segments(): segstart, segend = audio_segment.segment_idxs dur = segend - segstart NAP = _extract_NAP(segstart, segend, audio_segment.wav_file) speed = 1 amp = -3 maxamp = 1 start = 0 voice1 = 'playfile {} {} {} {} {} {} {} {} {}'.format(1, 6, np.random.rand()/3, audio_segment.wav_file, speed, segstart, segend, amp, maxamp) projection = _project(audio_segment.audio_id, sound_to_face, NAP, video_producer) voice2 = 'playfile {} {} {} {} {} {} {} {} {}'.format(2, 6, np.random.randint(3,6), audio_segment.wav_file, speed, segstart, segend, amp, maxamp) play_events.append([ dur, voice1, voice2, projection, FRAME_SIZE ]) print 'Dream mode playing back {} memories'.format(len(play_events)) scheduler.send_pyobj(play_events) if 'save' in pushbutton: utils.save('{}.{}'.format(pushbutton['save'], mp.current_process().name), [ sound_to_face, wordFace, face_to_sound, faceWord, video_producer, wavs, wav_audio_ids, audio_classifier, maxlen, NAP_hashes, face_id, face_recognizer, audio_memory ]) if 'load' in pushbutton: sound_to_face, wordFace, face_to_sound, faceWord, video_producer, wavs, wav_audio_ids, audio_classifier, maxlen, NAP_hashes, face_id, face_recognizer, audio_memory = utils.load('{}.{}'.format(pushbutton['load'], mp.current_process().name))
#!/usr/bin/env python import utils utils.test( utils.hamming_distance(utils.str_to_bin('this is a test'), utils.str_to_bin('wokka wokka!!!')), 37) with open('files/6.txt') as f: text = f.read().decode('base64') bin_text = utils.str_to_bin(text) bin_text_arr = bin_text.split() min_distance = 0 sizes = [] keysizes = [] for keysize in range(2, 41): # first KEYSIZE worth of bytes, and the second KEYSIZE worth of bytes info = [ bin_text_arr[i:i + keysize] for i in range(0, len(bin_text_arr), keysize) ] total_normalized_distance = 0 # average the hamming distance between sets of bytes for a, b in zip(info[::2], info[1::2]): edit_distance = utils.hamming_distance(''.join(a), ''.join(b)) # Normalize this result by dividing by KEYSIZE. total_normalized_distance += float(edit_distance) / float(keysize)
raw_text = ''.join(raw_list) # convert input text from base64 to plain text = raw_text.decode('base64') hamm_scores = {} for b in range(2,41): # Split text into chunks of length b text_chunks = [text[x:x+b] for x in range(0,len(text),b)] # First chunk s1 = text_chunks[0] hamming_sum = 0 for block in range(1,sample_size): # Subsequent chunks s2 = text_chunks[block] d = hamming_distance(s1, s2) hamming_sum += d average_hamm = hamming_sum / float(sample_size) normalised_hamming_distance = average_hamm / float(b) hamm_scores[b] = normalised_hamming_distance best = min(hamm_scores.items(), key=itemgetter(1))[0] #print best rows = ['' for x in range(best)] for i, t in enumerate(text): rows[i % best] += t output_list = []
def hamming_distance_penalty(set1, set2): longest_hypo = len(max(set1 + set2, key=len)) hypos = utils.as_ndarray(set1, min_length=longest_hypo) other_hypos = utils.as_ndarray(set2, min_length=longest_hypo) return np.apply_along_axis( lambda x: utils.hamming_distance(x, other_hypos), 1, hypos)
def finalize_region(self, region, sorted_gene_counts, annotations=None, debug=False): easycounts = {gene: counts for gene, counts in sorted_gene_counts} total_counts = sum([counts for counts in easycounts.values()]) class_counts = self.separate_into_classes(region, sorted_gene_counts, easycounts) genes_to_keep = set() if debug: print ' %s groups separated by %d snps (-: same group as previous kept gene)' % ( utils.color('blue', region), self.n_max_snps[region]) print ' %-20s %5s %s removed genes (snps counts%s)%s%s' % ( 'genes to keep', 'counts', '' if self.simglfo is None else utils.color('blue', 'sim'), '' if self.simglfo is None else utils.color( 'blue', ' sim counts'), '' if self.simglfo is None else (' ' + utils.color('red', 'x:') + ' not in simulation'), '' if (annotations is None or self.reco_info is None) else (' %s sim counts/genes for the queries assigned to this kept gene %s' % (utils.color('blue', '['), utils.color('blue', ']'))), ), def count_str(cnt): if cnt < 10.: return '%.1f' % cnt else: return '%.0f' % cnt def simcountstr( gene, ws ): # counts in simulation for <gene> (note that this is _not_ the same as sim_gene_count_str(), since this takes no account of _which_ queries these counts occur in [plus it's coming from the opposite point of view]) if self.simglfo is None: rstr = '' elif gene in self.simglfo['seqs'][utils.get_region(gene)]: rstr = utils.color( 'blue', (' %' + ws + 'd') % self.simcounts[utils.get_region(gene)][gene]) else: rstr = utils.color('red', (' %' + ws + 's') % 'x') return rstr def sim_gene_count_str( kgene ): # figure out simulation genes and counts for the uids assigned to <kgene> if annotations is None or self.reco_info is None: return '' uids_this_gene = [ uid for uid, line in annotations.items() if line[region + '_gene'] == kgene ] sim_genes = { } # simulation genes for the uids that we assigned to <kgene> (note that self.simcounts doesn't have this per-uid information) for uid in uids_this_gene: sgene = self.reco_info[uid][region + '_gene'] if sgene not in sim_genes: sim_genes[sgene] = 0 sim_genes[sgene] += 1 sorted_sim_gene_counts = sorted(sim_genes.items(), key=operator.itemgetter(1), reverse=True) count_str = ' '.join([ utils.color('blue' if sg == kgene else 'red', str(c)) for sg, c in sorted_sim_gene_counts ]) sgene_str = ' '.join( [utils.color_gene(sg) for sg, _ in sorted_sim_gene_counts]) return '%s %s' % (count_str, sgene_str) for iclass in range(len(class_counts)): gclass = class_counts[iclass] kept_this_class = [] for ig in range(len(gclass)): gfo = gclass[ig] if float( gfo['counts'] ) / total_counts < self.args.min_allele_prevalence_fraction: # always skip everybody that's super uncommon pass # don't keep it elif ig == 0: # keep the first one from this class genes_to_keep.add(gfo['gene']) kept_this_class.append(gfo['gene']) elif utils.hamming_distance( gclass[0]['seq'], gclass[ig]['seq'] ) == 0: # don't keep it if it's indistinguishable from the most common one (the matches are probably mostly really the best one) pass # don't keep it elif len( kept_this_class ) < self.args.n_alleles_per_gene: # always keep the most common <self.args.n_alleles_per_gene> in each class [note: defaults to 1 if looking for new alleles, otherwise 2] genes_to_keep.add(gfo['gene']) kept_this_class.append(gfo['gene']) else: pass # don't keep it if debug and gfo['gene'] in genes_to_keep: snpstr = ' ' if ig == 0 else '(%d)' % utils.hamming_distance( gclass[0]['seq'], gfo['seq'] ) # only happens if we keep more than one from this class print '\n %s%-s %7s%s %-3s' % ( '- ' if ig > 0 else ' ', utils.color_gene(gfo['gene'], width=20), count_str(gfo['counts']), simcountstr( gfo['gene'], '4'), snpstr), if debug: if len(kept_this_class) == 0: print '\n %s%-s %7s%4s %-3s' % ( ' ', utils.color('blue', 'none', width=20, padside='right'), '-', '', ''), removedfo = [ gfo for gfo in gclass if gfo['gene'] not in genes_to_keep ] removed_str = '' if len(removedfo) > 0: number_strs = [ '(%d %3s%s)' % (gfo['hdist'], count_str( gfo['counts']), simcountstr(gfo['gene'], '1')) for gfo in removedfo ] name_strs = [ '%s' % (utils.color_gene(gfo['gene'])) for gfo in removedfo ] removed_str = '%s %s' % (' '.join(number_strs), ' '.join(name_strs)) annotation_str = '' if (annotations is not None and self.reco_info is not None) and len(kept_this_class) > 0: annotation_str = '%s %s %s' % (utils.color( 'blue', '['), sim_gene_count_str( kept_this_class[-1]), utils.color('blue', ']')) print ' %s %s %s' % ( removed_str, (70 - utils.len_excluding_colors(removed_str)) * ' ', annotation_str), if debug: print '' genes_to_remove = set(self.glfo['seqs'][region]) - genes_to_keep print ' keeping %d / %d %s gene%s' % ( len(genes_to_keep), len(self.glfo['seqs'][region]), region, utils.plural(len(genes_to_keep))) if len(genes_to_keep) == 0: print ' would\'ve kept zero genes, instead keeping all of them' genes_to_keep = copy.deepcopy(genes_to_remove) genes_to_remove.clear() if self.simglfo is not None: missing_genes = set(self.simglfo['seqs'][region]) - genes_to_keep if len(missing_genes) > 0: print ' %s %d simulation genes (counts): %s' % (utils.color( 'red', 'missing'), len(missing_genes), ' '.join( [('%s %d' % (utils.color_gene(g), self.simcounts[region][g])) for g in sorted(missing_genes)])) completely_absent_genes = missing_genes - genes_to_remove if len(completely_absent_genes) > 0: print '%s %d simulation genes completely absent: %s' % ( utils.color('red', 'warning'), len(completely_absent_genes), ' '.join( [('%s %d' % (utils.color_gene(g), self.simcounts[region][g])) for g in sorted(completely_absent_genes)])) self.genes_to_keep |= genes_to_keep # add the ones from _this_ region (rhs) to the ones from all regions (lhs) self.genes_to_remove |= genes_to_remove self.finalized = True
def print_cluster(self, iclust, clusterfo, sorted_glcounts, new_seq, true_sorted_glcounts, mean_cluster_mfreqs, has_indels): if iclust > 0: print '' print ' %-3d %4d %6.3f' % (iclust, len(clusterfo['seqfos']), mean_cluster_mfreqs['v'] / mean_cluster_mfreqs['j']), for igene in range(len(sorted_glcounts)): if igene > 0: print '%22s' % '', gene, counts = sorted_glcounts[igene] print ' %-s %4d %2d%s' % (utils.color_gene(gene, width=20), counts, utils.hamming_distance(new_seq, self.glfo['seqs'][self.region][gene], align=True), ' (%s)' % utils.color('blue', 'x') if has_indels else ' '), if igene < len(sorted_glcounts) - 1 or self.reco_info is not None: print '' if self.reco_info is not None: for igene in range(len(true_sorted_glcounts)): gene, counts = true_sorted_glcounts[igene] print '%17s %s %-s %4d %s %2d ' % ('', utils.color('green', '['), utils.color_gene(gene[:23], width=20), counts, utils.color('green', ']'), utils.hamming_distance(new_seq, self.simglfo['seqs'][self.region][gene], align=True)), if igene < len(true_sorted_glcounts) - 1: print ''
def main(): from utils import base64_to_binary, hamming_distance, single_char_key_search import numpy as np from encryption import repeating_key_xor_base64 from conversion import base64_to_text from utils import get_english_score import codecs from data_utils import generate_binary_to_hex_mapping with open('data/problem6.txt') as f: content = f.read() content = content.replace('\n', '') binary_data = base64_to_binary(content) min_info = [] for keysize in range(2, 41): num_blocks = 4 distance = 0 for i in range(num_blocks): distance += hamming_distance( binary_data[keysize * i * 8:keysize * (i + 1) * 8], binary_data[keysize * (i + 1) * 8:keysize * (i + 2) * 8]) for i in range(num_blocks): distance += hamming_distance( binary_data[keysize * i * 8:keysize * (i + 1) * 8], binary_data[keysize * (i + 2) * 8:keysize * (i + 3) * 8]) distance = distance / keysize min_info.append((keysize, distance)) best_key_sizes = sorted(min_info, key=lambda x: x[1])[:3] potential_keys = [] for keysize_data in best_key_sizes: potential_key = '' key_size = keysize_data[0] keysize_num_bits = key_size * 8 if len(binary_data) % (keysize_num_bits) != 0: binary_data_divisible = binary_data[:-(len(binary_data) % (keysize_num_bits))] else: binary_data_divisible = binary_data splatted_binary_data = np.asarray([*binary_data_divisible]) numpy_binary_data = np.reshape(splatted_binary_data, (int( len(splatted_binary_data) / keysize_num_bits), keysize_num_bits)) for block_start in range(0, keysize_num_bits - 7, 8): binary_string_builder = '' block = numpy_binary_data[:, block_start:block_start + 8] splatted_bin = list(block) for list_of_bytes in splatted_bin: binary_string_builder += ''.join(list_of_bytes) bin_to_hex = generate_binary_to_hex_mapping() hex_string = '' for i in range(0, len(binary_string_builder), 4): hex_string += bin_to_hex[binary_string_builder[i:i + 4]] potential_key += single_char_key_search(hex_string)[2] potential_keys.append(potential_key) potential_answers = [] for key in potential_keys: decrypted_output = base64_to_text( repeating_key_xor_base64(key, content)) english_score = get_english_score(bytes(decrypted_output, 'utf-8')) potential_answers.append((english_score, decrypted_output)) final_answer = sorted(potential_answers, key=lambda x: x[0], reverse=True)[0] print(final_answer[1])
def parse_bcr_phylo_output(glfo, naive_line, outdir, ievent): seqfos = utils.read_fastx( '%s/%s.fasta' % (outdir, args.extrastr)) # output mutated sequences from bcr-phylo assert len( naive_line['unique_ids'] ) == 1 # enforces that we ran naive-only, 1-leaf partis simulation above assert not indelutils.has_indels( naive_line['indelfos'][0]) # would have to handle this below if args.debug: utils.print_reco_event(naive_line) reco_info = collections.OrderedDict() for sfo in seqfos: mline = copy.deepcopy(naive_line) utils.remove_all_implicit_info(mline) del mline['tree'] mline['unique_ids'] = [sfo['name']] mline['seqs'] = [ sfo['seq'] ] # it's really important to set both the seqs (since they're both already in there from the naive line) mline['input_seqs'] = [ sfo['seq'] ] # it's really important to set both the seqs (since they're both already in there from the naive line) reco_info[sfo['name']] = mline utils.add_implicit_info(glfo, mline) final_line = utils.synthesize_multi_seq_line_from_reco_info( [sfo['name'] for sfo in seqfos], reco_info) if args.debug: utils.print_reco_event(final_line) # extract kd values from pickle file (use a separate script since it requires ete/anaconda to read) if args.stype == 'selection': cmd = 'export PATH=%s:$PATH && xvfb-run -a python ./bin/view-trees.py --pickle-tree-file %s/%s_lineage_tree.p --kdfile %s/kd-vals.csv --newick-tree-file %s/simu.nwk' % ( ete_path, outdir, args.extrastr, outdir, outdir) utils.simplerun(cmd, shell=True) kdvals = {} with open('%s/kd-vals.csv' % outdir) as kdfile: reader = csv.DictReader(kdfile) for line in reader: kdvals[line['uid']] = float(line['kd']) if len( set(kdvals) - set(final_line['unique_ids']) ) > 0: # uids in the kd file but not the <line> (i.e. not in the newick/fasta files) are probably just bcr-phylo discarding internal nodes print ' in kd file, but missing from final_line (probably just internal nodes that bcr-phylo wrote to the tree without names): %s' % ( set(kdvals) - set(final_line['unique_ids'])) if len(set(final_line['unique_ids']) - set(kdvals)) > 0: print ' in final_line, but missing from kdvals: %s' % ' '.join( set(final_line['unique_ids']) - set(kdvals)) final_line['affinities'] = [ 1. / kdvals[u] for u in final_line['unique_ids'] ] tree = treeutils.get_dendro_tree(treefname='%s/simu.nwk' % outdir) if args.debug: print utils.pad_lines(treeutils.get_ascii_tree(dendro_tree=tree), padwidth=12) final_line['tree'] = tree.as_string(schema='newick') tmp_event = RecombinationEvent( glfo) # I don't want to move the function out of event.py right now tmp_event.set_reco_id( final_line, irandom=ievent ) # not sure that setting <irandom> here actually does anything # get target sequences target_seqfos = utils.read_fastx('%s/%s_targets.fa' % (outdir, args.extrastr)) final_line['target_seqs'] = [tfo['seq'] for tfo in target_seqfos] from Bio.Seq import Seq final_line['nearest_target_indices'] = [] aa_targets = [Seq(seq).translate() for seq in final_line['target_seqs']] for mseq in final_line['input_seqs']: aa_mseq = Seq(mseq).translate() aa_hdists = [ utils.hamming_distance(aa_t, aa_mseq, amino_acid=True) for aa_t in aa_targets ] imin = aa_hdists.index( min(aa_hdists) ) # NOTE doesn't do anything differently if there's more than one min final_line['nearest_target_indices'].append(imin) return final_line
import utils with open("6.txt") as f: contents = utils.ByteArray.fromBase64(f.read()) # print contents.asBase64() normalized_edit_dists = {} min_edit_dist = 40 * 8 for keysize in range(2, 41): edit_dist = utils.hamming_distance( contents[:keysize], contents[2 * keysize:3 * keysize]) / float(keysize) edit_dist2 = utils.hamming_distance( contents[keysize:2 * keysize], contents[3 * keysize:4 * keysize]) / float(keysize) normalized_edit_dists[keysize] = (edit_dist + edit_dist2) / 2 keysizes = sorted(normalized_edit_dists.iteritems(), key=lambda v: v[1]) possible_keysizes = keysizes[:3] print "key sizes", possible_keysizes for possible_keysize, _ in possible_keysizes: transposed_blocks = [ utils.ByteArray() for _ in range(possible_keysize) ] for i, e in enumerate(contents): transposed_blocks[i % possible_keysize].append(e) possible_key = [ utils.freq_analysis(block) for block in transposed_blocks
def make_single_tree(self, partitions, annotations, uid_set, get_fasttrees=False, n_max_cons_seqs=10, debug=False): # NOTE don't call this externally -- if you want a single tree, call make_trees() with <i_only_cluster> set def getline(uidstr, uid_set=None): if uidstr in annotations: # if we have this exact annotation return annotations[uidstr] else: if uid_set is None: uid_set = set(uidstr.split(':')) # should only get called if it's a singleton # note that for internal nodes in a fasttree-derived subtree, the uids will be out of order compared the the annotation keys for line in annotations.values(): # we may actually have the annotation for every subcluster (e.g. if --calculate-alternative-annotations was set), but in case we don't, this is fine if len(uid_set & set(line['unique_ids'])) > 0: # just take the first one with any overlap. Yeah, it's not necessarily the best, but its naive sequence probably isn't that different, and for just getting the fasttree it reeeeeeaaaallly doesn't matter return line raise Exception('couldn\'t find uid %s in annotations' % uid) def getseq(uid): line = getline(uid) return line['seqs'][line['unique_ids'].index(uid)] def lget(uid_list): return ':'.join(uid_list) # check for repeated uids (was only from seed uid, which shouldn't happen any more, but the code below throws an infinite loop if we do, so may as well be careful) for partition in partitions: if sum(len(c) for c in partition) > len(set(u for c in partition for u in c)): repeated_uids = [u for u, count in collections.Counter([u for c in partition for u in c]).items() if count > 1] raise Exception('found %d uid%s in more than one cluster (%s)' % (len(repeated_uids), utils.plural(len(repeated_uids)), ', '.join(repeated_uids))) default_edge_length = 999999 # it's nice to have the edges all set to something that's numeric (so the trees print), but also obvious wrong, if we forget to set somebody assert len(partitions[-1]) == 1 root_label = lget(partitions[-1][0]) # we want the order of the uids in the label to correspond to the order in self.partitions tns = dendropy.TaxonNamespace([root_label]) root_node = dendropy.Node(taxon=tns.get_taxon(root_label)) root_node.uids = uid_set # each node keeps track of the uids of its children dtree = dendropy.Tree(taxon_namespace=tns, seed_node=root_node) if debug: print ' starting tree with %d leaves' % len(uid_set) for ipart in reversed(range(len(partitions) - 1)): # dendropy seems to only have fcns to build a tree from the root downward, so we loop starting with the last partition (- 1 is because the last partition is guaranteed to be just one cluster) for lnode in dtree.leaf_node_iter(): # look for leaf nodes that contain uids from two clusters in this partition, and add those as children tclusts = [c for c in partitions[ipart] if len(set(c) & lnode.uids) > 0] if len(tclusts) < 2: continue for tclust in tclusts: ttaxon = dendropy.Taxon(lget(tclust)) tns.add_taxon(ttaxon) child = lnode.new_child(taxon=ttaxon, edge_length=default_edge_length) child.uids = set(tclust) if debug: print ' ipart %d' % ipart print ' split node: %d --> %s %s --> %s' % (len(lnode.uids), ' '.join([str(len(tc)) for tc in tclusts]), lnode.taxon.label, ' '.join([c.taxon.label for c in lnode.child_node_iter()])) # split existing leaves, which are probably not singletons (they're probably from the initial naive sequence collapse step) into subtrees such that each leaf is a singleton for lnode in dtree.leaf_node_iter(): if len(lnode.uids) == 1: continue if get_fasttrees and len(lnode.uids) > 2: seqfos = [{'name' : uid, 'seq' : getseq(uid)} for uid in lnode.taxon.label.split(':')] # may as well add them in the right order, although I don't think it matters subtree = treeutils.get_fasttree_tree(seqfos, getline(lnode.taxon.label, uid_set=lnode.uids)['naive_seq'], suppress_internal_node_taxa=True) # note that the fasttree distances get ignored below (no idea if they'd be better than what we set down there, but they probably wouldn't be consistent, so I'd rather ignore them) for tmpnode in subtree.postorder_node_iter(): if tmpnode.is_leaf(): tmpnode.uids = set([tmpnode.taxon.label]) else: tmpnode.uids = set([uid for c in tmpnode.child_node_iter() for uid in c.uids]) ttaxon = dendropy.Taxon(lget(tmpnode.uids)) subtree.taxon_namespace.add_taxon(ttaxon) tmpnode.taxon = ttaxon # ...and use the string of leaf nodes, even though they'll be in the wrong order (I think these get ignored when I call label_nodes() below, but it's still tidier to have them right in the meantime, and anyway since I'm suppressing internal taxa I think I need to set them to something) if debug: print ' adding subtree with %d leaves from fastree at leaf node %s' % (len(seqfos), lnode.taxon.label) print utils.pad_lines(treeutils.get_ascii_tree(dendro_tree=subtree)) dtree.taxon_namespace.add_taxa(subtree.taxon_namespace) lnode.add_child(subtree.seed_node) assert len(lnode.child_edges()) == 1 # we're iterating over leaves, so this should always be true lnode.child_edges()[0].collapse() else: # just add a star subtree for uid in lnode.taxon.label.split(':'): # may as well add them in the right order, although I don't think it matters ttaxon = dendropy.Taxon(uid) tns.add_taxon(ttaxon) child = lnode.new_child(taxon=ttaxon, edge_length=default_edge_length) child.uids = set([uid]) if debug: print ' added %d singleton children for %s' % (len(lnode.uids), lnode.taxon.label) # in order to set edge lengths, we need node sequences, so first set leaf node seqs for lnode in dtree.leaf_node_iter(): assert len(lnode.uids) == 1 lnode.seq = getseq(lnode.taxon.label) lnode.n_descendent_leaves = 1 # keep track of how many leaf nodes contributed to each node's consensus sequence (these are leaves, so it's trivally 1). This is less accurate than keeping track of all the sequences, but also faster # then set internal node seqs as the consensus of their children, and set the distance as hamming distance to child seqs if debug: print ' adding edge lengths either from fasttree %s or cons seq %s' % (utils.color('blue', 'x'), utils.color('red', 'x')) min_edge_length = None # setting this is nice for better debug viewing for node in dtree.postorder_internal_node_iter(): # includes root node child_cons_seq_counts = [c.n_descendent_leaves for c in node.child_node_iter()] total_descendent_leaves = sum(child_cons_seq_counts) if total_descendent_leaves > n_max_cons_seqs: # if there's tons of descendent leaves, we don't want to pass them all to the consensus fcn since it's slow, so we choose them in proportion to their actual proportions, but scaled down to <n_max_cons_seqs> child_cons_seq_counts = [int(n_max_cons_seqs * csc / float(total_descendent_leaves)) for csc in child_cons_seq_counts] child_cons_seq_counts = [max(1, csc) for csc in child_cons_seq_counts] # don't eliminate any sequences entirely (this makes the proportions less accurate (in some cases), but is the easy way to handle the case where there's a ton of singleton children if debug: print ' %s' % utils.color('green', node.taxon.label) csc_str = ' (reduced: %s)' % ' '.join([str(csc) for csc in child_cons_seq_counts]) if total_descendent_leaves > n_max_cons_seqs else '' print ' desc leaves per child: %s%s' % (' '.join(str(c.n_descendent_leaves) for c in node.child_node_iter()), csc_str) child_seqfos = [{'name' : cn.taxon.label + '-leaf-' + str(il), 'seq' : cn.seq} for cn, count in zip(node.child_node_iter(), child_cons_seq_counts) for il in range(count)] node.seq = utils.cons_seq(0.01, aligned_seqfos=child_seqfos, tie_resolver_seq=getline(root_label)['naive_seq']) #, debug=debug) # the consensus has an N at every position where the constituent sequences gave a tie. But Ns screw up the distances (especially because once we *get* an N, we can't get rid of it and it's propagated all the way up the tree), and in almost all cases the correct choice should be the naive base, so we use that node.n_descendent_leaves = total_descendent_leaves for edge in node.child_edge_iter(): from_fasttree = False if edge.length == default_edge_length: # otherwise it was set by fasttree, and it's probably better than what we'd get from this (it'd be nice to skip the cons seq stuff for the whole fasttree subtree, but then we don't have the cons seqs we need for later) edge.length = utils.hamming_distance(edge.head_node.seq, node.seq) / float(len(node.seq)) else: from_fasttree = True if min_edge_length is not None: edge.length = max(min_edge_length, edge.length) if debug: print ' %6.3f %s %s' % (edge.length, utils.color('blue' if from_fasttree else 'red', 'x'), edge.head_node.taxon.label) if debug: print ' naive seq %s' % getline(root_label)['naive_seq'] # NOTE might be worthwhile to add an edge connecting seed node and the actual naive sequence (i.e. for cases where our approximate naive is off) print ' root cons seq %s' % utils.color_mutants(getline(root_label)['naive_seq'], dtree.seed_node.seq) for node in dtree.preorder_node_iter(): del node.uids del node.seq del node.n_descendent_leaves treeutils.label_nodes(dtree, ignore_existing_internal_node_labels=True, ignore_existing_internal_taxon_labels=True, debug=debug) dtree.update_bipartitions() # probably don't really need this if debug: print treeutils.utils.pad_lines(treeutils.get_ascii_tree(dendro_tree=dtree, width=250)) return dtree