Example #1
0
import os,re,sys
sys.path.append('/home/etseng/UW_Larry_ncRNA/code/quasiClique/')

import cluster_steps1 as c1
from networkx import *

blast_split_filename = sys.argv[1]
output_i = int(sys.argv[2])

G = Graph()
nodes_to_index = c1.get_nodes_to_index('justPaeruginosa.IGRs_M8N7Q16R2W3S35E2.WU.nodes_to_index')
c1.step1_process_blast(blast_split_filename, 35, nodes_to_index, G)
c1.export_to_db(G, nodes_to_index, output_i, 'test')

def new_cluster_pipe(rfam_fam, shuffle_ratio):
	assert type(shuffle_ratio) is int
	output_prefix = "Rfam_{fam}_shuffle{X}X".format(fam=rfam_fam, X=shuffle_ratio)
	fasta_filename = output_prefix+'.fna'
	blast_output = "{input}.M8N7Q16R2W3E2.WUblast".format(input=fasta_filename)

	report_f = open(output_prefix+'.report', 'w')
	if not os.path.exists(blast_output):
		dummy_id = 0
		nodes_to_index = {}
		with open(fasta_filename, 'w') as f:
			with get_conn_ncRNA() as cursor:
				cursor.execute("select id,seq from Rfam_fasta where rfam_fam='{fam}' order by id".format(fam=rfam_fam))
				for _id,seq in cursor.fetchall():
					id = "TP{0}_{1}".format(dummy_id, _id)
					f.write(">{id}\n{seq}\n".format(id=id, seq=seq))
					nodes_to_index[id] = dummy_id
					dummy_id += 1
					ushuffle.shuffle(seq, len(seq), 2)
					for x in xrange(shuffle_ratio):
						id = "FP{0}_{1}".format(dummy_id, _id)
						f.write(">{id}\n{seq}\n".format(id=id, seq=ushuffle.shuffle2()))
						nodes_to_index[id] = dummy_id
						dummy_id += 1
		start_t = time.time()				
		# now blast it
		os.system("xdformat -n -o {input} {input}".format(input=fasta_filename))
		os.system("blastn -d {input} -i {input} -M 8 -N -7 -Q 16 -R 2 -E 2 \
				-W 3 -mformat 2 -cpus 4 -o {output}".format(input=fasta_filename, output=blast_output))
		report_f.write("(1)  BLAST TIME: {0} sec\n".format(time.time()-start_t))

		# now parse the blast
		nodes_to_index = c1.NodesToIndex(nodes_to_index, -1)
		G = Graph()
		c1.step1_process_blast(blast_output=blast_output,\
				score_cutoff=35, nodes_to_index=nodes_to_index, G=G, program='WU')
		print >> sys.stderr, "Homology graph has {0} nodes, {1} edges....".format(\
				G.number_of_nodes(), G.number_of_edges())
		c1.export_to_db(G, nodes_to_index, 0, blast_output)
		# convert nodes_to_index into dict nodes_ind --> acc id
		nodes_to_index = dict( map(lambda (x,y):(y,x), nodes_to_index.d.items()) )
		with open(blast_output+'.nodes_to_index', 'w') as handle:
			for ind,id in nodes_to_index.iteritems():
				handle.write("{0}\t{1}\n".format(ind,id))

	# read back the .parsed and .sets_for_nodes files
	G = Graph()
	sets_for_nodes = {}
	nodes_to_index = {}
	with open(blast_output+'.parsed') as handle:
		for line in handle:
			raw = map(int, line.strip().split('\t'))
			G.add_edge(raw[0],raw[1])
	with open(blast_output+'.sets_for_nodes') as handle:
		for line in handle:
			raw = map(int, line.strip().split('\t'))
			sets_for_nodes[raw[0]] = {'nodes_ind':raw[1],'start':raw[2],'end':raw[3]}
	with open(blast_output+'.nodes_to_index') as handle:
		for line in handle:
			raw = line.strip().split()
			nodes_to_index[int(raw[0])] = raw[1]

	tmp = len(filter(lambda x: nodes_to_index[sets_for_nodes[x]['nodes_ind']].startswith('FP'), G.nodes_iter()))
	report_f.write("(2)  AFTER parsing BLAST, graph has {0} negative control nodes, {1} TP nodes\n".format(tmp, G.number_of_nodes()-tmp))

	# remove low deg (< 3) nodes
	x = filter(lambda n: G.degree(n)<NEW_MIN_CLIQUE_SIZE, G.nodes_iter())
	while len(x) > 0:
		G.delete_nodes_from(x)
		x = filter(lambda n: G.degree(n)<NEW_MIN_CLIQUE_SIZE, G.nodes_iter())
	tmp = len(filter(lambda x: nodes_to_index[sets_for_nodes[x]['nodes_ind']].startswith('FP'), G.nodes_iter()))
	report_f.write("(3)  AFTER recursively removing nodes of degree < 3, graph has {0} negative control nodes, {1} TP nodes\n".format(tmp, G.number_of_nodes()-tmp))
	report_f.write("----------------------------------------------------------------------------\n")
	report_f.write("OUT\tDIR\tCLIQUE_SIZE\tSCANNED_TP\tSCANNED_FP\tCM_time\n")
	report_f.write("----------------------------------------------------------------------------\n")

	# for now just brute force....go through node by node as seeds
	dummy_round = 0
	while G.number_of_nodes()>=NEW_MIN_CLIQUE_SIZE and G.number_of_edges()>=NEW_MIN_CLIQUE_SIZE:
		# find perfect max cliques with a random starting node
		G_nodes = G.nodes()
		S,H = p.convert_graph_connectivity_to_sparse(G, G_nodes)
		tQ = p.grasp(S, H, gamma=1.0, maxitr=20, given_starting_node=None)
		Q = map(lambda x: G_nodes[x], tQ)
		if len(Q) < NEW_MIN_CLIQUE_SIZE: # delete these nodes
			G.delete_nodes_from(Q)
			continue
		# PERFECT CLIQUE SANITY TESTING, DELETE LATER
		for x in Q:
			for y in Q:
				if x!=y:
					print >> sys.stderr, "testing....", x,y
					try:
						assert G.has_edge(x,y)
					except:
						return Q,G
		Q.sort()
		print >> sys.stderr, "clique is...", Q
		prefix = output_prefix + str(dummy_round) + '_size' + str(len(Q)) + '_'
		dummy_round += 1
		start_t = time.time()
		scan_dir,scan_result = run_cmfinder(Q, nodes_to_index, sets_for_nodes, prefix, os.path.abspath(fasta_filename))
		cm_time = time.time()-start_t
		if scan_result is not None:
			outf = open(os.path.basename(scan_dir)+'.gv','w')
			outf.write("""graph test{
			edge [ dir=none ];
			node [ style=filled, fontsize=2.0, height=0.1, width=0.1, fixedsize=true ];
			""")
			# draw this graph
			scanned = {'TP':0, 'FP':0}
			for n in G.nodes_iter():
				id = nodes_to_index[sets_for_nodes[n]['nodes_ind']] # id is something like TP1_NC_XXXX.... or FP10_NC_XXXX....
				_id = id[:id.find('_')]
				shape = 'circle' if id.startswith('TP') else 'box'
				if n in Q:
					outf.write("{0} [color=dodgerblue1, shape={1}];\n".format(n, shape))
				elif id in scan_result:
					outf.write("{0} [color=darkorange, shape={1}];\n".format(n, shape))
					scanned[_id[:2]] += 1
				else:
					outf.write("{0} [color=grey, shape={1}];\n".format(n, shape))
			for (n1,n2) in G.edges_iter(data=False):
				id1 = nodes_to_index[sets_for_nodes[n1]['nodes_ind']]
				id1 = id1[:id1.find('_')]
				id2 = nodes_to_index[sets_for_nodes[n2]['nodes_ind']]
				id2 = id2[:id2.find('_')]
				outf.write("{0} -- {1};\n".format(n1,n2))
			outf.write("}")
			outf.close()
			report_f.write("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\n".format(outf.name,scan_dir,len(Q),scanned['TP'],scanned['FP'],cm_time))
			report_f.flush()
		# delete the edges from the graph
		G.delete_edges_from(itertools.combinations(Q, 2))
		# again, remove low-degree nodes
		x = filter(lambda n: G.degree(n)<NEW_MIN_CLIQUE_SIZE, G.nodes_iter())
		while len(x) > 0:
			G.delete_nodes_from(x)
			x = filter(lambda n: G.degree(n)<NEW_MIN_CLIQUE_SIZE, G.nodes_iter())  
	report_f.close()