def test_pickle(track_abundance):
import pickle
from io import BytesIO
e1 = MinHash(n=5,
ksize=6,
is_protein=False,
track_abundance=track_abundance)
seq = 'ATGGCAGTGACGATGCCG'
e1.add_sequence(seq)
e1.add_sequence(seq)
fp = BytesIO()
pickle.dump(e1, fp)
fp2 = BytesIO(fp.getvalue())
e2 = pickle.load(fp2)
assert e1.get_mins(with_abundance=track_abundance) == \
e2.get_mins(with_abundance=track_abundance)
assert e1.num == e2.num
assert e1.ksize == e2.ksize
assert e1.is_protein == e2.is_protein
assert e1.max_hash == e2.max_hash
assert e1.seed == e2.seed
def __init__(self, query_file, ksize, scaled, catlas_name, debug=True):
self.filename = query_file
self.ksize = ksize
self.kmers = set()
self.name = None
mh = MinHash(0, ksize, scaled=scaled)
self.mh = mh
self.catlas_name = catlas_name
self.debug = debug
notify('----')
notify('QUERY FILE: {}', self.filename)
# build hashes for all the query k-mers & create signature
notify('loading query kmers...', end=' ')
bf = khmer.Nodetable(ksize, 1, 1)
for record in screed.open(self.filename):
if self.name is None:
self.name = record.name
if len(record.sequence) >= int(ksize):
self.kmers.update(bf.get_kmer_hashes(record.sequence))
mh.add_sequence(record.sequence, True)
self.sig = sourmash.SourmashSignature(mh,
name=self.name,
filename=self.filename)
notify('got {} k-mers from query', len(self.kmers))
self.cdbg_match_counts = {}
self.catlas_match_counts = {}
def test_abund_similarity_zero():
E1 = MinHash(n=5, ksize=20, track_abundance=True)
E2 = MinHash(n=5, ksize=20, track_abundance=True)
for i in [1]:
E1.add_hash(i)
assert E1.similarity(E2) == 0.0
def test_common_1(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4, 6]:
E2.add_hash(i)
assert E1.count_common(E2) == 4
assert E2.count_common(E1) == 4
def test_diff_seed(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance, seed=1)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance, seed=2)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4, 6]:
E2.add_hash(i)
with pytest.raises(ValueError):
E1.count_common(E2)
def test_set_abundance_num_hypothesis(hashes, abundances, sketch_size):
a = MinHash(sketch_size, 10, track_abundance=True)
oracle = dict(zip(hashes, abundances))
a.set_abundances(oracle)
mins = a.get_mins(with_abundance=True)
size = min(sum(1 for v in oracle.values() if v > 0), sketch_size)
assert len(mins) == size
for k, v in mins.items():
assert oracle[k] == v
def test_jaccard_1(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4, 6]:
E2.add_hash(i)
# here the union is [1, 2, 3, 4, 5]
# and the intesection is [1, 2, 3, 4] => 4/5.
assert round(E1.jaccard(E2), 2) == round(4 / 5.0, 2)
assert round(E2.jaccard(E1), 2) == round(4 / 5.0, 2)
def test_abund_similarity():
E1 = MinHash(n=5, ksize=20, track_abundance=True)
E2 = MinHash(n=5, ksize=20, track_abundance=True)
for i in [1]:
E1.add_hash(i)
for i in [1, 2]:
E2.add_hash(i)
assert round(E1.similarity(E1)) == 1.0
assert round(E1.similarity(E2), 2) == 0.5
assert round(E1.similarity(E1, ignore_abundance=True)) == 1.0
assert round(E1.similarity(E2, ignore_abundance=True), 2) == 0.5
def test_jaccard_2_difflen(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4]:
E2.add_hash(i)
print(E1.jaccard(E2))
assert round(E1.jaccard(E2), 2) == 4 / 5.0
assert round(E2.jaccard(E1), 2) == 4 / 5.0
def test_bad_construct_2(track_abundance):
try:
e1 = MinHash(n=100, is_protein=False,
track_abundance=track_abundance)
assert 0, "require ksize in constructor"
except TypeError:
pass
def test_set_abundance_scaled_hypothesis(hashes, abundances, scaled):
a = MinHash(0, 10, track_abundance=True, scaled=scaled)
oracle = dict(zip(hashes, abundances))
a.set_abundances(oracle)
max_hash = get_max_hash_for_scaled(scaled)
below_max_hash = sum(1 for (k, v) in oracle.items()
if k <= max_hash and v > 0)
mins = a.get_mins(with_abundance=True)
assert len(mins) == below_max_hash
for k, v in mins.items():
assert oracle[k] == v
assert k <= max_hash
assert v > 0
def test_diff_seed(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance, seed=1)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance, seed=2)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4, 6]:
E2.add_hash(i)
with pytest.raises(ValueError):
E1.count_common(E2)
def test_common_1(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4, 6]:
E2.add_hash(i)
assert E1.count_common(E2) == 4
assert E2.count_common(E1) == 4
def test_protein_mh(track_abundance):
e1 = MinHash(n=5, ksize=6, is_protein=True,
track_abundance=track_abundance)
e2 = MinHash(n=5, ksize=6, is_protein=True,
track_abundance=track_abundance)
seq = 'ATGGCAGTGACGATGCCG'
e1.add_sequence(seq)
for i in range(len(seq) - 5):
kmer = seq[i:i + 6]
e2.add(kmer)
assert e1.get_mins() == e2.get_mins()
assert 901193879228338100 in e1.get_mins()
def _signatures(self):
"Create a _signatures member dictionary that contains {idx: sigobj}."
from sourmash import MinHash, SourmashSignature
is_protein = False
is_hp = False
is_dayhoff = False
if self.moltype == 'protein':
is_protein = True
elif self.moltype == 'hp':
is_hp = True
elif self.moltype == 'dayhoff':
is_dayhoff = True
minhash = MinHash(n=0,
ksize=self.ksize,
scaled=self.scaled,
is_protein=is_protein,
hp=is_hp,
dayhoff=is_dayhoff)
debug('creating signatures for LCA DB...')
mhd = defaultdict(minhash.copy_and_clear)
temp_vals = defaultdict(list)
# invert the hashval_to_idx dictionary
for (hashval, idlist) in self.hashval_to_idx.items():
for idx in idlist:
temp_hashes = temp_vals[idx]
temp_hashes.append(hashval)
# 50 is an arbitrary number. If you really want
# to micro-optimize, list is resized and grow in this pattern:
# 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
# (from https://github.com/python/cpython/blob/b2b4a51f7463a0392456f7772f33223e57fa4ccc/Objects/listobject.c#L57)
if len(temp_hashes) > 50:
mhd[idx].add_many(temp_hashes)
# Sigh, python 2... when it goes away,
# we can do `temp_hashes.clear()` instead.
del temp_vals[idx]
# We loop temp_vals again to add any remainder hashes
# (each list of hashes is smaller than 50 items)
for sig, vals in temp_vals.items():
mhd[sig].add_many(vals)
sigd = {}
for idx, mh in mhd.items():
ident = self.idx_to_ident[idx]
name = self.ident_to_name[ident]
sigd[idx] = SourmashSignature(mh, name=name)
debug('=> {} signatures!', len(sigd))
return sigd
def test_jaccard_1(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4, 6]:
E2.add_hash(i)
# here the union is [1, 2, 3, 4, 5]
# and the intesection is [1, 2, 3, 4] => 4/5.
assert round(E1.jaccard(E2), 2) == round(4 / 5.0, 2)
assert round(E2.jaccard(E1), 2) == round(4 / 5.0, 2)
def test_abund_similarity():
E1 = MinHash(n=5, ksize=20, track_abundance=True)
E2 = MinHash(n=5, ksize=20, track_abundance=True)
for i in [1]:
E1.add_hash(i)
for i in [1, 2]:
E2.add_hash(i)
assert round(E1.similarity(E1)) == 1.0
assert round(E1.similarity(E2), 2) == 0.5
assert round(E1.similarity(E1, ignore_abundance=True)) == 1.0
assert round(E1.similarity(E2, ignore_abundance=True), 2) == 0.5
def test_build_hashCounter():
mh1 = MinHash(0, 21, scaled=1, track_abundance=True)
mh2 = MinHash(0, 21, scaled=1, track_abundance=True)
mh1.add_many((1, 2, 3, 4))
mh2.add_many((1, 2, 5))
true_res = Counter({1: 2, 2: 2, 3: 1, 4: 1, 5: 1})
ss1 = SourmashSignature(mh1)
ss2 = SourmashSignature(mh2)
counts = Counter()
hc = build_hashCounter([ss1, ss2], counts)
print("Hash Counter: ", hc)
assert hc == true_res
def test_abund_similarity_zero():
E1 = MinHash(n=5, ksize=20, track_abundance=True)
E2 = MinHash(n=5, ksize=20, track_abundance=True)
for i in [1]:
E1.add_hash(i)
assert E1.similarity(E2) == 0.0
def test_drop_below_mincount_threshold():
mh1 = MinHash(0, 21, scaled=1, track_abundance=True)
mh2 = MinHash(0, 21, scaled=1, track_abundance=True)
mh1.add_many((1, 2, 3, 4))
mh2.add_many((1, 1, 2, 5))
ss1 = SourmashSignature(mh1)
ss2 = SourmashSignature(mh2)
counts = Counter()
hc = build_hashCounter([ss1, ss2], counts)
kept_hashes = drop_below_mincount(hc, 3)
true_kept = Counter({1: 3})
print("kept hashes: ", kept_hashes)
assert kept_hashes == true_kept
def sketch(args):
cwd = os.getcwd()
db_path = os.path.join(cwd, args.name + '.db')
# check for the existence of the database and tables
if os.path.exists(db_path):
pass
else:
print(
"Database file not found. Please make sure the name is correct or run mashpit build."
)
exit(0)
fasta_folder = os.path.join(cwd, 'fasta')
if os.path.exists(fasta_folder):
pass
else:
print("Fasta folder not found.")
exit(0)
sig_file_name = args.name + '.sig'
all_fasta_path = os.path.join(fasta_folder, "*_skeasa.fasta")
genomes_list = glob.glob(all_fasta_path)
minhashes = []
for genome in genomes_list:
mh = MinHash(n=1000, ksize=31)
for record in screed.open(genome):
mh.add_sequence(record.sequence, True)
minhashes.append(mh)
siglist = []
for i in range(len(minhashes)):
signame = genomes_list[i].strip(fasta_folder).strip('_skesa.fasta')
siglist.append(SourmashSignature(minhashes[i], name=signame))
with open(sig_file_name, 'w') as f:
save_signatures(siglist, fp=f)
def test_pickle(track_abundance):
import pickle
from io import BytesIO
e1 = MinHash(n=5, ksize=6, is_protein=False,
track_abundance=track_abundance)
seq = 'ATGGCAGTGACGATGCCG'
e1.add_sequence(seq)
e1.add_sequence(seq)
fp = BytesIO()
pickle.dump(e1, fp)
fp2 = BytesIO(fp.getvalue())
e2 = pickle.load(fp2)
assert e1.get_mins(with_abundance=track_abundance) == \
e2.get_mins(with_abundance=track_abundance)
assert e1.num == e2.num
assert e1.ksize == e2.ksize
assert e1.is_protein == e2.is_protein
assert e1.max_hash == e2.max_hash
assert e1.seed == e2.seed
def to_sourmash(self):
try:
from sourmash import MinHash
except ImportError:
print(
'Must install python sourmash to convert to sourmash.MinHash',
file=sys.stderr)
return None
sig = MinHash(self.num(),
self.ksize(),
is_protein=self.is_protein(),
dayhoff=self.dayhoff(),
hp=self.hp(),
track_abundance=self.track_abundance(),
seed=self.seed(),
mins=self.mins(),
max_hash=self.max_hash())
return sig
# remove hashes that occur only once
for hashval, ct in counts.copy().items():
print(f"{hashval}:{ct}")
if ct < min_count:
counts.pop(hashval)
# write out hashes
# let's try building a sig. we will use this sig later to intersect with sample-specific sigs
new_mins = set(counts.keys())
print(len(new_mins))
with open(outhashes, "w") as out:
for hsh in new_mins:
out.write(str(hsh) + '\n')
if len(new_mins) > 0:
minhash = MinHash(
n=0, ksize=ksize, scaled=scaled
) # scaled=1 so we keep all (though these were previously at some other scaled val)
minhash.add_many(set(counts.keys()))
# write sig to file
sigobj = sourmash.SourmashSignature(
minhash,
name=f"aggregated_hashvals_above_{min_count}",
filename=f"generated with drop_unique_hashes.py")
sigobjs += [sigobj]
## this part only handles one output file -- doesn't take care of case with many ksizes/moltypes
with open(outsig, 'wt') as sigout:
sourmash.save_signatures(sigobjs, sigout)
#notify('wrote signature to {}', args.output)
# write out hashes to a text file
def main(argv):
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('catlas_prefix', help='catlas prefix')
p.add_argument('mh_index_picklefile', help='pickled hashval index')
p.add_argument('hashval_list', help='file with list of hashvals')
p.add_argument('output')
p.add_argument('-k',
'--ksize',
default=31,
type=int,
help='k-mer size (default: 31)')
p.add_argument('--scaled',
default=1000,
type=float,
help="scaled value for contigs minhash output")
p.add_argument('-v', '--verbose', action='store_true')
args = p.parse_args(argv)
# create output directory if it doesn't exist.
outdir = args.output
notify('putting output in {}', outdir)
os.makedirs(os.path.join(outdir, "contigs"), exist_ok=True)
if not os.path.isdir(outdir):
error("output '{}' is not a directory and cannot be made", outdir)
sys.exit(-1)
# load picklefile
with open(args.mh_index_picklefile, 'rb') as fp:
hashval_to_contig_id = pickle.load(fp)
notify('loaded {} hash value -> cdbg_id mappings from {}',
len(hashval_to_contig_id), args.mh_index_picklefile)
# load list of desired hashvals
hashvals = [int(x.strip()) for x in open(args.hashval_list, 'rt')]
hashvals = set(hashvals)
notify('loaded {} search hashvalues from {}', len(hashvals),
args.hashval_list)
if not len(hashvals):
print('No hash values to search!', file=sys.stderr)
sys.exit(-1)
# load catlas DAG
catlas = CAtlas(args.catlas_prefix)
notify('loaded {} nodes from catlas {}', len(catlas), args.catlas_prefix)
notify('loaded {} layer 1 catlas nodes', len(catlas.layer1_to_cdbg))
# find the contigs filename
contigs_file = os.path.join(args.catlas_prefix, 'contigs.fa.gz')
# get a single ksize & scaled
ksize = int(args.ksize)
scaled = int(args.scaled)
# record command line
with open(os.path.join(outdir, 'command.txt'), 'wt') as fp:
fp.write(str(sys.argv))
fp.write("\n")
# output results.csv in the output directory:
csvoutfp = open(os.path.join(outdir, 'hashval_results.csv'), 'wt')
csv_writer = csv.writer(csvoutfp)
csv_writer.writerow(['hashval', 'bp', 'contigs'])
# iterate over each query, do the thing.
n_found = 0
for hashval in hashvals:
notify('----')
notify('QUERY HASHVAL: {}', hashval)
mh = MinHash(0, ksize, scaled=scaled)
result = execute_query(hashval, catlas, hashval_to_contig_id, mh=mh)
notify('done searching!')
if not result:
notify("no result for hashval {}", hashval)
continue
result.retrieve_contigs(contigs_file)
result.write(csv_writer, csvoutfp, outdir)
assert hashval in mh.get_mins()
n_found += 1
# end main loop!
notify('----')
notify("Done! Found {} hashvals of {} in {} with k={}", n_found,
len(hashvals), args.catlas_prefix, ksize)
notify("Results are in directory '{}'", outdir)
return 0
def test_protein_mh(track_abundance):
e1 = MinHash(n=5,
ksize=6,
is_protein=True,
track_abundance=track_abundance)
e2 = MinHash(n=5,
ksize=6,
is_protein=True,
track_abundance=track_abundance)
seq = 'ATGGCAGTGACGATGCCG'
e1.add_sequence(seq)
for i in range(len(seq) - 5):
kmer = seq[i:i + 6]
e2.add(kmer)
assert e1.get_mins() == e2.get_mins()
assert 901193879228338100 in e1.get_mins()
def test_dna_mh(track_abundance):
e1 = MinHash(n=5, ksize=4, track_abundance=track_abundance)
e2 = MinHash(n=5, ksize=4, track_abundance=track_abundance)
seq = 'ATGGCAGTGACGATGCCAG'
e1.add_sequence(seq)
for i in range(len(seq) - 3):
e2.add(seq[i:i + 4])
assert e1.get_mins() == e2.get_mins()
print(e1.get_mins())
assert 726311917625663847 in e1.get_mins()
assert 3697418565283905118 in e1.get_mins()
def test_jaccard_2_difflen(track_abundance):
E1 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
E2 = MinHash(n=5, ksize=20, track_abundance=track_abundance)
for i in [1, 2, 3, 4, 5]:
E1.add_hash(i)
for i in [1, 2, 3, 4]:
E2.add_hash(i)
print(E1.jaccard(E2))
assert round(E1.jaccard(E2), 2) == 4 / 5.0
assert round(E2.jaccard(E1), 2) == 4 / 5.0
def main():
p = argparse.ArgumentParser()
p.add_argument('hashfile') # file that contains hashes
p.add_argument('-o', '--output', default=None,
help='file to output signature to')
p.add_argument('-k', '--ksize', default=None, type=int)
p.add_argument('--scaled', default=None, type=int)
p.add_argument('--num', default=None, type=int)
p.add_argument('--name', default='', help='signature name')
p.add_argument('--filename', default='',
help='filename to add to signature')
args = p.parse_args()
# check arguments.
if args.scaled and args.num:
error('cannot specify both --num and --scaled! exiting.')
return -1
if not args.ksize:
error('must specify --ksize')
return -1
if not args.output:
error('must specify --output')
return -1
# first, load in all the hashes
hashes = set()
for line in open(args.hashfile, 'rt'):
hashval = int(line.strip())
hashes.add(hashval)
if not hashes:
error("ERROR, no hashes loaded from {}!", args.hashfile)
return -1
notify('loaded {} distinct hashes from {}', len(hashes), args.hashfile)
# now, create the MinHash object that we'll use.
scaled = 0
num = 0
if args.scaled:
scaled = args.scaled
elif args.num:
num = args.num
else:
notify('setting --num automatically from the number of hashes.')
num = len(hashes)
# construct empty MinHash object according to args
minhash = MinHash(n=num, ksize=args.ksize, scaled=scaled)
# add hashes into!
minhash.add_many(hashes)
if len(minhash) < len(hashes):
notify("WARNING: loaded {} hashes, but only {} made it into MinHash.",
len(hashes), len(minhash))
if scaled:
notify("This is probably because of the scaled argument.")
elif args.num:
notify("This is probably because your --num is set to {}",
args.num)
if num > len(minhash):
notify("WARNING: --num set to {}, but only {} hashes in signature.",
num, len(minhash))
sigobj = sourmash.SourmashSignature(minhash, name=args.name,
filename=args.filename)
with open(args.output, 'wt') as fp:
sourmash.save_signatures([sigobj], fp)
notify('wrote signature to {}', args.output)
def fetchneighborhood2(index, features_upstream=0, features_downstream=0):
cluster = iaa_positive_df.iloc[index, :]
acc = cluster['accession']
assembly = re.sub('.gbff', '_proteins.fa.indexprot', cluster['filename'])
#make the genome database from the .fa.index file
assembly_index_file = 'index_files/' + assembly
print(assembly_index_file)
db = pd.read_csv(assembly_index_file,
sep="!!",
header=None,
engine='python')
#db.columns = ["filename","assembly","accession","locus_tag","old_locus_tag","name","biosample","protein_name","coordinates","protein_id"]
db.columns = [
"filename", "assembly", "accession", "locus_tag", "old_locus_tag",
"name", "biosample", "protein_name", "coordinates", "protein_id",
"pseudogene", "protein_seq"
]
db['direction'] = [
-1 if re.match('complement', c) else 1 for c in db['coordinates']
]
db['start_coord'] = [
re.search('\d+?(?=\.\.(\d|\>))', str(c)).group(0)
for c in db['coordinates']
]
db['start_coord'] = [
re.sub('complement|>|<|\)|\(', "", c) for c in db['start_coord']
]
db['start_coord'] = db['start_coord'].astype(int)
db['end_coord'] = [
re.search('(?<=\.(\.|\>))\d+', str(c)).group(0)
for c in db['coordinates']
]
db['end_coord'] = [re.sub('>|<|\)|\(', "", c) for c in db['end_coord']]
db['end_coord'] = db['end_coord'].astype(int)
hit_list = cluster['hit_list']
query_list = cluster['query_list']
cluster_number = cluster['cluster_number']
hit_dict = dict(zip(hit_list, query_list))
genome = db.loc[db['accession'] == acc].copy()
start = genome[genome['locus_tag'] == hit_list[0]].index.values.astype(
int)[0] - features_upstream
stop = genome[genome['locus_tag'] == hit_list[-1]].index.values.astype(
int)[0] + features_downstream
neighborhood = genome.loc[start:stop, ].copy()
neighborhood['query_match'] = neighborhood['locus_tag'].map(hit_dict)
coord_list = list(
zip(neighborhood['start_coord'], neighborhood['end_coord'],
neighborhood['direction'], neighborhood['query_match']))
#function to find GC content of cluster vs genome
gbff_str = str(db['filename'][0][1:])
with open("gbff_files_unzipped/" + gbff_str) as file:
gbff_file = file.read()
genome_seq = "".join(re.findall("(?<=ORIGIN)[\s+\S+]+?(?=\/\/)",
gbff_file))
genome_seq = re.sub('\s|\d|\n', '', genome_seq)
Gg = genome_seq.count("g")
Gc = genome_seq.count("c")
Ga = genome_seq.count("a")
Gt = genome_seq.count("t")
genomeGC = (Gg + Gc) / (Gg + Gc + Ga + Gt)
start = min(coord_list)[0]
end = max(coord_list)[1]
regex_str = acc + "[\s+\S+]+\/\/"
all_cluster_fasta = re.findall(regex_str, gbff_file)[0]
all_cluster_fasta = re.findall("(?<=ORIGIN)[\s+\S+]+(?=\/\/)",
all_cluster_fasta)[0]
all_cluster_fasta = re.sub(" |\d|\n", "", all_cluster_fasta)
cluster_seq = all_cluster_fasta[start - 1:end - 1]
g = cluster_seq.count("g")
c = cluster_seq.count("c")
a = cluster_seq.count("a")
t = cluster_seq.count("t")
clusterGC = (g + c) / (g + c + a + t)
diffGC = abs(clusterGC - genomeGC)
#compare minhash values between cluster and genome
kmer_size = 5
n = 0
sc = 1
cluster_minhash = MinHash(n=n, ksize=kmer_size, scaled=sc)
cluster_minhash.add_sequence(cluster_seq, force=True)
cluster_minhash.add_sequence(complement(cluster_seq), force=True)
#
genome_minhash = MinHash(n=n, ksize=kmer_size, scaled=sc)
genome_minhash.add_sequence(genome_seq, force=True)
genome_minhash.add_sequence(complement(genome_seq), force=True)
minhash_sim = cluster_minhash.similarity(genome_minhash)
# genome_minus_cluster=re.sub(cluster_seq,'',genome_seq)
# #print(len(genome_seq)-len(genome_minus_cluster))
# genome_minus_cluster_minhash=MinHash(n=n, ksize=kmer_size,scaled=sc)
# genome_minus_cluster_minhash.add_sequence(genome_minus_cluster,force=True)
# genome_minus_cluster_minhash.add_sequence(complement(genome_minus_cluster),force=True)
# minhash_sim_minus_cluster=cluster_minhash.similarity(genome_minus_cluster_minhash)
#print(minhash_sim)
#compare tetranucleotide frequency between cluster and genomes
bases = ['a', 't', 'g', 'c']
four_mers = [''.join(p) for p in itertools.product(bases, repeat=4)]
four_mer_count_genome = np.add(
[genome_seq.count(i) for i in four_mers],
[complement(genome_seq).count(i) for i in four_mers])
four_mer_freq_genome = [
i / sum(four_mer_count_genome) for i in four_mer_count_genome
]
four_mer_count_cluster = np.add(
[cluster_seq.count(i) for i in four_mers],
[complement(cluster_seq).count(i) for i in four_mers])
four_mer_freq_cluster = [
i / sum(four_mer_count_cluster) for i in four_mer_count_cluster
]
four_mer_distance = scipy.spatial.distance.cityblock(
four_mer_freq_cluster, four_mer_freq_genome)
####
if sum(neighborhood[neighborhood['query_match'].notnull()]
['direction']) < 0:
neighborhood['actual_start_tmp'] = neighborhood['start_coord']
neighborhood['start_coord'] = neighborhood['end_coord'] * -1
neighborhood['end_coord'] = neighborhood['actual_start_tmp'] * -1
neighborhood['direction'] = neighborhood['direction'] * -1
neighborhood = neighborhood.sort_values(by='start_coord')
neighborhood['query_match'] = neighborhood['query_match'].replace(
np.nan, "x")
nhbrhood_hit_list = list(neighborhood['query_match'])
nhbrhood_locus_tags = list(neighborhood['locus_tag'])
nhbrhood_old_locus_tags = list(neighborhood['old_locus_tag'])
nhbrhood_prot_ids = list(neighborhood['protein_id'])
nhbrhood_prot_name = list(neighborhood['protein_name'])
nhbrhood_prot_seq = list(neighborhood['protein_seq'])
order = [("| " + gene['query_match'] + " 〉") if gene['direction'] == 1 else
("〈 " + gene['query_match'] + " |")
for index, gene in neighborhood.iterrows()]
dist = list(
np.array(neighborhood['start_coord'][1:]) -
np.array(neighborhood['end_coord'][:-1]))
dist = ["-" + str(d) + "-" for d in dist]
adj_coord_list = list(
zip(neighborhood['start_coord'], neighborhood['end_coord'],
neighborhood['direction'], neighborhood['query_match']))
if min(neighborhood['start_coord']) < 0:
tare_value = abs(min(neighborhood['start_coord']))
tared_adj_coord_list = list(
zip([v + tare_value for v in neighborhood['start_coord']],
[v + tare_value for v in neighborhood['end_coord']],
neighborhood['direction'], neighborhood['query_match']))
else:
tare_value = min(neighborhood['start_coord'])
tared_adj_coord_list = list(
zip([v - tare_value for v in neighborhood['start_coord']],
[v - tare_value for v in neighborhood['end_coord']],
neighborhood['direction'], neighborhood['query_match']))
# making an ITOL compatible string
gene_color_dict = {
'IaaP': '#ff5969',
'IaaQ': '#2db34e',
'IaaR': '#fb77e0',
'IaaA': '#00bc7e',
'IaaB': '#8d006e',
'IaaC': '#cfdd63',
'IaaD': '#0060d0',
'IaaE': '#bb7b00',
'IaaF': '#7c2c29',
'IaaG': '#f1d17a',
'IaaH': '#37589E',
'IaaI': '#ACC92A',
'IaaJ': '#752AC9',
'IaaK': '#D4B5E6',
'IaaL': '#211E45',
'IaaM': '#BFB3E6',
'x': '#d1d1d1'
}
max_len = tared_adj_coord_list[-1][1]
itol_diagram = []
for g in tared_adj_coord_list:
gene_string = []
gene_length = g[1] - g[0]
if g[2] > 0:
gene_string.append('RE')
gene_string.append(str(g[0]))
gene_string.append(str(g[1] - (0.1 * gene_length)))
#gene_string.append('#34b4eb')
gene_string.append(gene_color_dict[g[3]])
gene_string.append(str(g[3]))
gene_string.append(',')
gene_string.append('TR')
gene_string.append(str(g[1] - (0.1 * gene_length)))
gene_string.append(str(g[1]))
#gene_string.append('#34b4eb')
gene_string.append(gene_color_dict[g[3]])
gene_string.append('')
else:
gene_string.append('TL')
gene_string.append(str(g[0]))
gene_string.append(str(g[0] + (0.1 * gene_length)))
#gene_string.append('#34b4eb')
gene_string.append(gene_color_dict[g[3]])
gene_string.append('')
gene_string.append(',')
gene_string.append('RE')
gene_string.append(str(g[0] + (0.1 * gene_length)))
gene_string.append(str(g[1]))
#gene_string.append('#34b4eb')
gene_string.append(gene_color_dict[g[3]])
gene_string.append(str(g[3]))
itol_gene = '|'.join(gene_string)
itol_diagram.append(itol_gene)
itol_diagram_joined = ",".join(map(str, itol_diagram))
itol_diagram_string = str(max_len) + ',' + itol_diagram_joined
itol_diagram_string = re.sub(',\|', ',', itol_diagram_string)
#obtains "| A 〉-23-| B 〉-23-| C 〉"
synteny_dir_dist = ''.join(sum(zip(order, dist + [0]), ())[:-1])
synteny_dir_dist = re.sub("iaa", "", synteny_dir_dist)
#obtains "| A 〉| B 〉| C 〉"
synteny_dir = ''.join(order)
synteny_dir = re.sub("iaa", "", synteny_dir)
#obtains "| A:23.23 〉| B:23.23〉| C:23.23 〉"
#synteny_dir_pident =''.join(order_pident)
#synteny_dir_pident = re.sub("iaa" ,"", synteny_dir_pident)
#obtains "A-B-C"
synteny = re.sub("\n", "-",
neighborhood['query_match'].to_string(index=False))
synteny = re.sub("Iaa| ", "", synteny)
synteny_alphabet = "".join([
gene['query_match'].replace("Iaa", "").upper() if gene['direction']
== 1 else gene['query_match'].replace("Iaa", "").lower()
for index, gene in neighborhood.iterrows()
])
cluster_len = max(neighborhood['end_coord']) - min(
neighborhood['start_coord'])
assembly = re.sub("\{|\}|\'|>", "", str(set(neighborhood['assembly'])))
accession = re.sub("\{|\}|\'", "", str(set(neighborhood['accession'])))
title = re.sub("\{|\}|\'", "", str(set(neighborhood['name'])))
print(assembly_index_file + " successfully used")
return ([
accession, assembly, title,
len(neighborhood), cluster_len, synteny, synteny_alphabet,
synteny_dir_dist, synteny_dir, cluster_number, coord_list,
adj_coord_list, tared_adj_coord_list, itol_diagram_string,
nhbrhood_hit_list, nhbrhood_locus_tags, nhbrhood_old_locus_tags,
nhbrhood_prot_ids, nhbrhood_prot_name, nhbrhood_prot_seq, clusterGC,
genomeGC, diffGC, minhash_sim, four_mer_distance,
four_mer_freq_cluster, cluster_seq
])
def test_dna_mh(track_abundance):
e1 = MinHash(n=5, ksize=4, track_abundance=track_abundance)
e2 = MinHash(n=5, ksize=4, track_abundance=track_abundance)
seq = 'ATGGCAGTGACGATGCCAG'
e1.add_sequence(seq)
for i in range(len(seq) - 3):
e2.add(seq[i:i + 4])
assert e1.get_mins() == e2.get_mins()
print(e1.get_mins())
assert 726311917625663847 in e1.get_mins()
assert 3697418565283905118 in e1.get_mins()