def db_results_printJSON(ssea_dir, ss_id):
results_json_file = os.path.join(ssea_dir,
'results.json')
with open(results_json_file, 'r') as fin:
for line in fin:
# load json document (one per line)
result = Result.from_json(line.strip())
result.ss_id = int(ss_id)
print result.to_json()
def parse_results(filename):
with open(filename, 'r') as fp:
for line in fp:
result = Result.from_json(line.strip())
yield result
def test_default_result(self):
result = Result.default()
self.assertTrue(result.t_id is None)
self.assertTrue(result.ss_rank is None)
def ssea_run(counts, size_factors, membership, rng, config):
'''
counts: numpy array of float values
size_factors: normalization factors for counts
membership: int array (0 or 1) with set membership
rng: RandomState object
config: Config object
'''
# run kernel to generate a range of observed enrichment scores
resample_rand_seeds = np.empty(config.resampling_iterations, dtype=np.int)
resample_count_ranks = np.empty((config.resampling_iterations, counts.shape[0]), dtype=np.int)
resample_es_vals = np.zeros(config.resampling_iterations, dtype=np.float)
resample_es_ranks = np.zeros(config.resampling_iterations, dtype=np.int)
for i in xrange(config.resampling_iterations):
# save random number generator seed before running kernel
resample_rand_seeds[i] = rng.seed
k = ssea_kernel(counts, size_factors, membership, rng,
resample_counts=True,
permute_samples=False,
add_noise=True,
noise_loc=config.noise_loc,
noise_scale=config.noise_scale,
method_miss=config.weight_miss,
method_hit=config.weight_hit,
method_param=config.weight_param)
k = KernelResult._make(k)
resample_count_ranks[i] = k.ranks
resample_es_vals[i] = k.es_val
resample_es_ranks[i] = k.es_rank
# find the median ES value
median_index = int(config.resampling_iterations / 2)
median_index = resample_es_vals.argsort()[median_index]
median_es_val = resample_es_vals[median_index]
# choose whether to use the positive or negative side of the
# distribution based on the median ES value
if median_es_val == 0:
return Result.default(), np.zeros((0,), dtype=np.float)
elif median_es_val < 0:
signfunc = np.less
else:
signfunc = np.greater
# subset to include only the corresponding side of the distribution
resample_sign_inds = signfunc(resample_es_vals, 0)
resample_rand_seeds = resample_rand_seeds[resample_sign_inds]
resample_count_ranks = resample_count_ranks[resample_sign_inds]
resample_es_vals = resample_es_vals[resample_sign_inds]
resample_es_ranks = resample_es_ranks[resample_sign_inds]
# determine the median value
median_index = int(resample_es_vals.shape[0] / 2)
median_index = resample_es_vals.argsort()[median_index]
# select the kernel run representing the median ES value
rand_seed = resample_rand_seeds[median_index]
es_val = resample_es_vals[median_index]
es_rank = resample_es_ranks[median_index]
ranks = resample_count_ranks[median_index]
es_sign = cmp(es_val, 0)
# permute samples and determine ES null distribution
null_es_vals = np.zeros(config.perms, dtype=np.float)
null_es_ranks = np.zeros(config.perms, dtype=np.float)
i = 0
while i < config.perms:
k = ssea_kernel(counts, size_factors, membership, rng,
resample_counts=True,
permute_samples=True,
add_noise=True,
noise_loc=config.noise_loc,
noise_scale=config.noise_scale,
method_miss=config.weight_miss,
method_hit=config.weight_hit,
method_param=config.weight_param)
k = KernelResult._make(k)
if cmp(k.es_val, 0) == es_sign:
null_es_vals[i] = k.es_val
null_es_ranks[i] = k.es_rank
i += 1
# Subset the null ES scores to only positive or negative
null_sign_inds = signfunc(null_es_vals, 0)
null_es_vals = null_es_vals[null_sign_inds]
null_es_ranks = null_es_ranks[null_sign_inds]
# Adjust for variation in gene set size. Normalize ES(S,null)
# and the observed ES(S), separately rescaling the positive and
# negative scores by dividing by the mean of the ES(S,null) to
# yield normalized scores NES(S,null)
null_es_mean = np.fabs(null_es_vals.mean())
null_nes_vals = null_es_vals / null_es_mean
# Normalize the observed ES(S) by rescaling by the mean of
# the ES(S,null) separately for positive and negative ES(S)
nes_val = es_val / null_es_mean
# estimate nominal p value for S from ES(S,null) by using the
# positive or negative portion of the distribution corresponding
# to the sign of the observed ES(S)
p_value = (np.fabs(null_es_vals) >= np.fabs(es_val)).sum().astype(np.float)
p_value /= null_es_vals.shape[0]
# Create result object for this SSEA test
res = Result()
res.rand_seed = int(rand_seed)
res.es = round(es_val, FLOAT_PRECISION)
res.es_rank = int(es_rank)
res.nominal_p_value = round(p_value, SCIENTIFIC_NOTATION_PRECISION)
res.nes = round(nes_val, FLOAT_PRECISION)
# save some of the resampled es points
res.resample_es_vals = np.around(resample_es_vals[:Result.MAX_POINTS], FLOAT_PRECISION)
res.resample_es_ranks = resample_es_ranks[:Result.MAX_POINTS]
# save null distribution points
res.null_es_mean = null_es_mean
res.null_es_vals = np.around(null_es_vals[:Result.MAX_POINTS], FLOAT_PRECISION)
res.null_es_ranks = null_es_ranks[:Result.MAX_POINTS]
# get indexes of hits in this set
m = membership[ranks]
hit_inds = (m > 0).nonzero()[0]
num_hits = hit_inds.shape[0]
num_misses = m.shape[0] - num_hits
# calculate leading edge stats
if es_val < 0:
core_hits = sum(i >= es_rank for i in hit_inds)
core_misses = (m.shape[0] - es_rank) - core_hits
else:
core_hits = sum(i <= es_rank for i in hit_inds)
core_misses = 1 + es_rank - core_hits
null_hits = num_hits - core_hits
null_misses = num_misses - core_misses
# fisher exact test (one-sided hypothesis that LE is enricheD)
fisher_p_value = fisher.pvalue(core_hits, core_misses, null_hits, null_misses).right_tail
# odds ratio
n = np.inf if null_hits == 0 else float(core_hits) / null_hits
d = np.inf if null_misses == 0 else float(core_misses) / null_misses
if np.isfinite(n) and np.isfinite(d):
if n == 0 and d == 0:
odds_ratio = np.nan
else:
odds_ratio = np.inf if d == 0 else (n / d)
elif np.isfinite(d):
odds_ratio = np.inf
else:
odds_ratio = np.nan if n == 0 else 0.0
# create dictionary result
res.core_hits = int(core_hits)
res.core_misses = int(core_misses)
res.null_hits = int(null_hits)
res.null_misses = int(null_misses)
res.fisher_p_value = np.round(fisher_p_value, SCIENTIFIC_NOTATION_PRECISION)
res.odds_ratio = odds_ratio
# return result and null distribution for subsequent fdr calculations
return res, null_nes_vals
def _cmp_json_nes(line):
'''comparison function for batch_sort'''
res = Result.from_json(line.strip())
return abs(res.nes)
def compute_qvalues(json_iterator, hists_file):
'''
computes fdr q values from json Result objects sorted
by abs(NES) (low to high)
json_iterator: iterator that yields json objects in sorted order
hists_file: contains histogram data from null distribution
'''
# load histogram data
hists = np.load(hists_file)
# compute cumulative sums for fdr interpolation
cdfs = {}
for k in ('null_nes_neg', 'null_nes_pos', 'obs_nes_neg', 'obs_nes_pos'):
h = hists[k]
cdf = np.zeros(h.shape[0]+1, dtype=np.float)
cdf[1:] = h.cumsum()
cdfs[k] = cdf
# keep track of minimum FDR and rank for positive
# and negative NES separately
NEG = 0
POS = 1
null_keys = ['null_nes_neg', 'null_nes_pos']
obs_keys = ['obs_nes_neg', 'obs_nes_pos']
tot_obs = [cdfs['obs_nes_neg'][-1], cdfs['obs_nes_pos'][-1]]
cur_ranks = [tot_obs[0], tot_obs[1]]
min_fdrs = [1.0, 1.0]
# perform merge of sorted json files
for line in json_iterator:
# load json document (one per line)
res = Result.from_json(line.strip())
es = res.es
log_nes_clip = np.log10(np.clip(abs(res.nes), NES_MIN, NES_MAX))
if es != 0:
if es < 0:
sign_ind = NEG
sign = -1.0
else:
sign_ind = POS
sign = 1.0
# For a given NES(S) = NES* >= 0, the FDR is the ratio of the
# percentage of all permutations NES(S,null) >= 0, whose
# NES(S,null) >= NES*, divided by the percentage of observed S with
# NES(S) >= 0, whose NES(S) >= NES*, and similarly for
# NES(S) = NES* <= 0.
# to compute a sample set specific FDR q value we look at the
# aggregated enrichment scores for all tests of that sample set
# compute the cumulative sums of NES histograms use interpolation
# to find fraction NES(null) >= NES* and account for the observed
# permutation in the null set interpolate NES in log space
null_nes_cumsum = cdfs[null_keys[sign_ind]]
null_n = interp(log_nes_clip, LOG_NES_BINS, null_nes_cumsum)
obs_nes_cumsum = cdfs[obs_keys[sign_ind]]
obs_n = interp(log_nes_clip, LOG_NES_BINS, obs_nes_cumsum)
n = 1.0 - (null_n / null_nes_cumsum[-1])
d = 1.0 - (obs_n / obs_nes_cumsum[-1])
#print 'SS_ID=%d ES=%f NES=%f n=%f (%f / %f) d=%f (%f / %f)' % (i, res.es, res.nes, n, null_n, null_nes_cumsum[-1], d, obs_n, obs_nes_cumsum[-1])
# update json dict
if (n <= 0.0) or (d <= 0.0):
res.ss_fdr_q_value = 0.0
else:
res.ss_fdr_q_value = n / d
#print 'SS_ID=%d ES=%f NES=%f fdr=%f minfdr=%f' % (i, res.es, res.nes, res.ss_fdr_q_value, min_fdrs[i])
# compare with minimum FDR and adjust minimum FDR if necessary
if res.ss_fdr_q_value < min_fdrs[sign_ind]:
min_fdrs[sign_ind] = res.ss_fdr_q_value
else:
res.ss_fdr_q_value = min_fdrs[sign_ind]
res.ss_rank = cur_ranks[sign_ind]
res.ss_frac = sign * (1.0 - ((res.ss_rank - 1) / float(tot_obs[sign_ind])))
cur_ranks[sign_ind] -= 1
# convert back to json
yield res.to_json()
yield os.linesep
# cleanup
hists.close()
def ssea_serial(config, sample_set, output_basename,
startrow=None, endrow=None):
'''
main SSEA loop (single processor)
matrix_dir: numpy memmap matrix containing numeric data
sample_set: SampleSet object
config: Config object
output_basename: prefix for writing result files
'''
# initialize random number generator
rng = RandomState()
# open data matrix
bm = BigCountMatrix.open(config.matrix_dir)
# determine range of matrix to process
if startrow is None:
startrow = 0
if endrow is None:
endrow = bm.shape[0]
assert startrow < endrow
# get membership array for sample set
membership = sample_set.get_array(bm.colnames)
valid_samples = (membership >= 0)
# setup histograms
hists = _init_hists()
# setup report file
unsorted_json_file = output_basename + JSON_UNSORTED_SUFFIX
outfileh = open(unsorted_json_file, 'wb')
for i in xrange(startrow, endrow):
logging.debug("\tRow: %d (%d-%d)" % (i, startrow, endrow))
# read from memmap
counts = np.array(bm.counts[i,:], dtype=np.float)
# remove 'nan' values
valid_inds = np.logical_and(valid_samples, np.isfinite(counts))
# subset counts, size_factors, and membership array
counts = counts[valid_inds]
size_factors = bm.size_factors[valid_inds]
valid_membership = membership[valid_inds]
# write dummy results for invalid rows
if (valid_inds.sum() == 0) or (np.all(counts == 0)):
res = Result.default()
else:
# run ssea
res, null_nes_vals = ssea_run(counts, size_factors,
valid_membership, rng, config)
# update histograms
null_keys = []
obs_keys = []
if res.es < 0:
null_keys.append('null_nes_neg')
obs_keys.append('obs_nes_neg')
elif res.es > 0:
null_keys.append('null_nes_pos')
obs_keys.append('obs_nes_pos')
for k in xrange(len(null_keys)):
null_nes = np.clip(np.fabs(null_nes_vals), NES_MIN, NES_MAX)
obs_nes = np.clip(np.fabs(res.nes), NES_MIN, NES_MAX)
hists[null_keys[k]] += np.histogram(null_nes, NES_BINS)[0]
hists[obs_keys[k]] += np.histogram(obs_nes, NES_BINS)[0]
# save t_id
res.t_id = i
# convert to json and write
print >>outfileh, res.to_json()
# close report file
outfileh.close()
# save histograms to a file
output_hist_file = output_basename + NPY_HISTS_SUFFIX
np.savez(output_hist_file, **hists)
# cleanup
bm.close()
# sort output json file by abs(NES)
logging.debug("Worker %s: sorting results" % (output_basename))
# make tmp dir for sorting
if os.path.exists(output_basename):
shutil.rmtree(output_basename)
os.makedirs(output_basename)
# call batch sort python function
sorted_json_file = output_basename + JSON_SORTED_SUFFIX
batch_sort(input=unsorted_json_file,
output=sorted_json_file,
key=_cmp_json_nes,
buffer_size=SORT_BUFFER_SIZE,
tempdirs=[output_basename])
# remove tmp dir
shutil.rmtree(output_basename)
# remove unsorted json file
os.remove(unsorted_json_file)
logging.debug("Worker %s: done" % (output_basename))
return 0
