def subsampling_BowTie_n_inhouse(file_iter, di_method, df_output):
	"""
	Run subsampling for BowTie+inhouse (inhouse must already have been pre-processed for eligibility)
	Simply prints out per line: <sample>,<size>,<comma-separated DI>
	"""
	runner = DiversityIndexRunner()
	seqvec = np.zeros((5,520), dtype=np.int)
	print >> sys.stderr, "DFs will be written to {0}....".format(df_output)
	h = open(df_output, 'w')
	w = DF.DFWriter(h)
	for sample,file in file_iter:
		eligible_bowtie = hello.subsample_reads_BowTie_prepare(file, refmap, phred_cutoff, min_length, ecoli_lo, ecoli_hi)
		eligible_inhouse = load(open(inhouse_eligible.format(sample, phred_cutoff, min_length, max_degen, L2[ecoli_lo], L2[ecoli_hi])))
		print >> sys.stderr, "eligible reads for {0}: bowtie {1}, inhouse {2}".format(sample, \
				len(eligible_bowtie), len(eligible_inhouse))
		p = len(eligible_bowtie)*1./(len(eligible_bowtie)+len(eligible_inhouse))
		for size in SUBSAMPLE_SIZES:
			print >> sys.stderr, sample, size
			seqvec[:] = 0
			hello.subsample_reads_BowTie(file, refmap, seqvec, eligible_bowtie, int(size*p))
			hello.subsample_reads_inhouse(refmap, seqvec, eligible_inhouse, phred_cutoff, min_length, size-int(size*p))
			df = Read.ReadDF(sample, refmap)
			df.len = 520
			df.assign_vec(seqvec)
			df.add_annotation('size', size)
			w.write(df)
			di=runner.run(df, method=di_method, threshold=0, vec_pre_normalized=False, ignoreN=True)[valid_DI_pos]
			print("{0},{1},{2}".format(sample,size,",".join(map(str,di))))
	h.close()
def main(file_iter, output_df_filename, log_f):
	log_f.write("phred cutoff:{0}\n".format(phred_cutoff))
	log_f.write("min length:{0}\n".format(min_length))
	log_f.write("max degen (if used):{0}\n".format(max_degen))
	log_f.write("use ecoli range {0}-{1}\n".format(ecoli_lo, ecoli_hi))
	f = open(output_df_filename, 'w')
	dfwriter = DF.DFWriter(f)
	for sample,file in file_iter:
		print >> sys.stderr, "processing {0}.........".format(sample)
		seqvec = np.zeros((5,520), dtype=np.int)
		# --------------- for in-house aligned ------------- #
		for file in glob.iglob(inhouse.format(sample)):
#			if file.endswith('.bz2'):
#				os.system("bunzip2 " + file)
#				file = file[:-4]
			used, discarded = hello.gather_reads_inhouse(file, refmap, seqvec, phred_cutoff, min_length, max_degen, ecoli_lo, ecoli_hi)
			print >> sys.stderr, file, used, discarded
			log_f.write("FILE:{0} USED:{1} DISCARDED:{2}\n".format(file, used, discarded))
#			os.system("bzip2 " + file)
		#  ---------------- for BowTie-aligned ---------------#
#		used, discarded = hello.gather_reads_BowTie(file, refmap, seqvec, phred_cutoff, min_length, ecoli_lo, ecoli_hi)
#		print >> sys.stderr, "used:", used, "discarded:", discarded
#		log_f.write("FILE:{0} USED:{1} DISCARDED:{2}\n".format(file, used, discarded))
		df = Read.ReadDF(sample, refmap)
		df.len = 520
		df.assign_vec(seqvec)
		dfwriter.write(df)
		runner = DiversityIndexRunner()
		di=runner.run(df, method='Simpson', threshold=0, vec_pre_normalized=False, ignoreN=True)[valid_DI_pos]
		print("{0},{1}".format(sample,",".join(map(str,di))))
	f.close()
	def __init__(self, df_list, **kwargs):
		if df_list is None:
			print >> sys.stderr, "called with a None, I hope you know what you're doing!"
			print >> sys.stderr, "calling init_from_di_list later perhaps?"
			self.df_list = None
			return
		self.df_list = df_list
		self.original_names = [df.name for df in self.df_list]
		self.mask = kwargs['mask'] if 'mask' in kwargs else 1.
		self.method = kwargs['method'] if 'method' in kwargs else 'Simpson'
		self.threshold = kwargs['threshold'] if 'threshold' in kwargs else 10

		self.m = len(df_list) # number of samples (rows)
		self.n = df_list[0].len # length of the DF vectors (columns)

		self.runner = DiversityIndexRunner(self.mask)

		self.trees = [tree.Leaf(self.df_list[i].name) for i in xrange(self.m)]

		self.X = np.zeros((self.m, self.n), dtype=np.float)
		for i,df in enumerate(self.df_list):
			#print >> sys.stderr, "normalizing {0}....".format(df.name)
			df.normalized_vec(ignoreN=True)
			di = self.runner.run(df, method=self.method, threshold=self.threshold, \
					vec_pre_normalized=True, ignoreN=True)
			self.X[i, :] = di	

		# calculate the initial distance matrix
		self._dist = np.zeros((self.m, self.m), dtype=np.float)
		for i in xrange(self.m):
			self._dist[i, i] = float("inf")
			for j in xrange(i+1, self.m):
				# method 1: Euclidean distance between DIs
				d = math.sqrt(sum(x**2 for x in self.X[i,:]-self.X[j,:]))
				# method 2: sum of sum of distances squared between DFs
				#d = self.df_list[i].get_vec_diff_sqsum(self.df_list[j])
				self._dist[i, j] = d
				self._dist[j, i] = d
class Cluster:
	def __init__(self, df_list, **kwargs):
		if df_list is None:
			print >> sys.stderr, "called with a None, I hope you know what you're doing!"
			print >> sys.stderr, "calling init_from_di_list later perhaps?"
			self.df_list = None
			return
		self.df_list = df_list
		self.original_names = [df.name for df in self.df_list]
		self.mask = kwargs['mask'] if 'mask' in kwargs else 1.
		self.method = kwargs['method'] if 'method' in kwargs else 'Simpson'
		self.threshold = kwargs['threshold'] if 'threshold' in kwargs else 10

		self.m = len(df_list) # number of samples (rows)
		self.n = df_list[0].len # length of the DF vectors (columns)

		self.runner = DiversityIndexRunner(self.mask)

		self.trees = [tree.Leaf(self.df_list[i].name) for i in xrange(self.m)]

		self.X = np.zeros((self.m, self.n), dtype=np.float)
		for i,df in enumerate(self.df_list):
			#print >> sys.stderr, "normalizing {0}....".format(df.name)
			df.normalized_vec(ignoreN=True)
			di = self.runner.run(df, method=self.method, threshold=self.threshold, \
					vec_pre_normalized=True, ignoreN=True)
			self.X[i, :] = di	

		# calculate the initial distance matrix
		self._dist = np.zeros((self.m, self.m), dtype=np.float)
		for i in xrange(self.m):
			self._dist[i, i] = float("inf")
			for j in xrange(i+1, self.m):
				# method 1: Euclidean distance between DIs
				d = math.sqrt(sum(x**2 for x in self.X[i,:]-self.X[j,:]))
				# method 2: sum of sum of distances squared between DFs
				#d = self.df_list[i].get_vec_diff_sqsum(self.df_list[j])
				self._dist[i, j] = d
				self._dist[j, i] = d
	
	def init_from_di_list(self, di_dict, **kwargs):
		"""
		alternative __init__ taking a dict sample_name --> array of DI as input
		if this init is used, we're expecting to run UPGMA clustering
		"""
		self.original_names = di_dict.keys()
		self.original_names.sort()
		self.mask = kwargs['mask'] if 'mask' in kwargs else 1.
		self.method = kwargs['method'] if 'method' in kwargs else 'Simpson'
		self.threshold = kwargs['threshold'] if 'threshold' in kwargs else 10
		self.m = len(di_dict)
		self.n = len(di_dict.itervalues().next())
		self.trees = [tree.Leaf(x) for x in self.original_names]
		self.X = np.zeros((self.m, self.n), dtype=np.float) 
		# fill up X using di_dict
		for i in xrange(self.m):
			self.X[i] = di_dict[self.original_names[i]]
		self._dist = np.zeros((self.m, self.m), dtype=np.float)
		for i in xrange(self.m):
			self._dist[i, i] = float("inf")
			for j in xrange(i+1, self.m):
				# method 1: Euclidean distance between DIs
				d = math.sqrt(sum(x**2 for x in self.X[i,:]-self.X[j,:]))
				# method 2: sum of sum of distances squared between DFs
				#d = self.df_list[i].get_vec_diff_sqsum(self.df_list[j])
				self._dist[i, j] = d
				self._dist[j, i] = d
	
	def write_DI(self, output_filename, mask=None):
		with open(output_filename, 'w') as f:
			for i, name in enumerate(self.original_names):
				di = self.X[i, ] if mask is None else self.X[i, mask]
				f.write(name + ',')
				di.tofile(f, sep=",")
				f.write('\n')
						
	def run_one_cluster_step(self):
		d = self._dist.argmin()
		i, j = d / self.m, d % self.m
		_min_val = self._dist[i, j]
		if _min_val == float("inf"):
			raise StopIteration, "done!"
		#print >> sys.stderr, "combining {0} and {1}".format(self.trees[i], self.trees[j])
		
		# merge j into i
		size_i = len(self.trees[i].get_leaves())
		size_j = len(self.trees[j].get_leaves())
		t = tree.Tree()
		t.add_edge((self.trees[i], 0, _min_val/2)) # (subtree-i, bootstrap=0, branch length=dist)
		t.add_edge((self.trees[j], 0, _min_val/2)) 
		self.trees[i] = t
		self.trees[j] = None
		# NEW!!! instead of just adding df_list[j] to df_list[i], normalize the counts FIRST!!!
		if self.df_list is not None:
			self.df_list[i].normalized_vec_add(self.df_list[j], vec_pre_normalized=True, ignoreN=True)

#			print "before", self.X[i, ]
			self.X[i] = self.runner.run(self.df_list[i], method=self.method, threshold=self.threshold,\
					vec_pre_normalized=True, ignoreN=True)
#			print("merged {0} and {1}".format(i, j))
#			print "new vec is now", self.X[i, ]
		
#		self._dist[j, :] = float("inf")
#		self._dist[:, j] = float("inf")
		for k in xrange(self.m):
			if k==i or k==j or self.trees[k] is None: continue
			# method 1:
			#d = math.sqrt(sum(x**2 for x in self.X[i,:]-self.X[k,:]))
			# method 2:
			#d = self.df_list[i].get_vec_diff_sqsum(self.df_list[k])
			# method 3: UPGMA
			d = (self._dist[k, i] * size_i + self._dist[k, j] * size_j) / (size_i + size_j)
			# method 4: complete linkage
			#d = max(self._dist[k, i], self._dist[k, j])
			#print >> sys.stderr, "using Euclidean dist: {0}, using vecdiff: {1}".format(\
			#		math.sqrt(sum(x**2 for x in self.X[i,:]-self.X[k,:])), d)
			self._dist[i, k] = d
			self._dist[k, i] = d
		self._dist[j, :] = float("inf")
		self._dist[:, j] = float("inf")

	def run_till_end(self):
		while len(self.trees) > 1:
			try:
				self.run_one_cluster_step()
			except StopIteration:
				break