def join(args, outs, chunk_defs, chunk_outs):
if args.skip:
return
np.random.seed(0)
# Load the matrix
mat = CountMatrix.load_h5_file(args.matrix_h5)
print mat.m.shape, mat.m.nnz
barcodes = mat.bcs
# Load graph-based clustering from analysis H5
clustering_key = cr_clustering.format_clustering_key(
cr_clustering.CLUSTER_TYPE_GRAPHCLUST, 0)
clustering = SingleGenomeAnalysis.load_clustering_from_h5(
args.clusters_h5, clustering_key)
labels = clustering.clusters
# Clusters that were 0 were unused in the clustering analysis (only relevant if the cluster stage was run by itself)
total_bcs = len(labels)
use_bcs = np.flatnonzero(labels > 0)
expr_mat = mat.m[:, use_bcs]
# Make cluster labels 0-based
labels = labels[use_bcs] - 1
# Convert PCA coords to dataframe
pca = SingleGenomeAnalysis.load_pca_from_h5(
args.pca_h5).transformed_pca_matrix[use_bcs, :]
pca_df = pd.DataFrame(pca)
print pca_df.shape
# 1) Run hierarchical clustering on cluster medoids in PCA-space
# 2) For each pair of clusters that are sibling leaves,
# 3) Run a differential expression analysis
# 4) Merge the clusters if not enough genes are differentially expressed
# 5) If merged, stop considering cluster-pairs and goto 1)
# Cache already-checked cluster-pairs
# set of (frozenset, frozenset)
checked_cluster_pairs = set()
while True:
print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
sys.stdout.flush()
if len(np.bincount(labels)) == 1:
# One cluster remains
break
# Compute medoids, perform hierarchical clustering
pca_df['cluster'] = labels
medoids = pca_df.groupby('cluster').apply(
lambda x: x.median(axis=0)).as_matrix()[:, :-1]
hc = linkage(medoids, 'complete')
max_label = np.max(labels)
print np.bincount(labels)
print 'max=%d' % max_label
any_merged = False
for step in xrange(hc.shape[0]):
if hc[step, 0] <= max_label and hc[step, 1] <= max_label:
leaf0, leaf1 = hc[step, 0], hc[step, 1]
group0 = np.flatnonzero(labels == leaf0)
group1 = np.flatnonzero(labels == leaf1)
# Skip this cluster pair if already checked
set0 = frozenset(group0)
set1 = frozenset(group1)
cluster_pair = tuple(sorted([set0, set1]))
if cluster_pair in checked_cluster_pairs:
continue
checked_cluster_pairs.add(cluster_pair)
print 'Comparing clusters (%d,%d)' % (1 + leaf0, 1 + leaf1)
submat = expr_mat[:, np.concatenate((group0, group1))]
print '\tComputing params on (%d,%d) matrix' % submat.shape
params = compute_sseq_params(submat)
print '\tRunning DE on %d vs %d cells' % (len(group0),
len(group1))
group0_submat = np.arange(len(group0))
group1_submat = np.arange(len(group0),
len(group0) + len(group1))
de_result = sseq_differential_expression(
submat, group0_submat, group1_submat, params)
n_de_genes = np.sum(de_result.adjusted_p_value <
MERGE_CLUSTERS_DE_ADJ_P_THRESHOLD)
if n_de_genes == 0:
print '\tFound %d DE genes. Merging clusters (%d,%d)' % (
n_de_genes, 1 + leaf0, 1 + leaf1)
# Relabel as the smaller-index cluster
labels[labels == leaf1] = leaf0
# Shift all labels above old label down
labels[labels > leaf1] = labels[labels > leaf1] - 1
any_merged = True
break
sys.stdout.flush()
if not any_merged:
break
# Convert back to one-based cluster labels
labels += 1
labels = cr_clustering.relabel_by_size(labels)
# Convert back into original bc space, with 0s for unused bcs
final_labels = np.zeros(total_bcs, dtype=int)
final_labels[use_bcs] = labels
# Save results
with analysis_io.open_h5_for_writing(outs.clusters_h5) as f:
cr_graphclust.save_graphclust_h5(f, final_labels)
clustering_key = cr_clustering.format_clustering_key(
cr_clustering.CLUSTER_TYPE_GRAPHCLUST, 0)
cr_clustering.save_clustering_csv(outs.clusters_csv, clustering_key,
final_labels, barcodes)
def join(args, outs, chunk_defs, chunk_outs):
if args.skip:
return
# Merge the neighbor matrices
with LogPerf('merge_nn'):
nn = cr_graphclust.merge_nearest_neighbors(
[chunk.chunked_neighbors for chunk in chunk_outs],
chunk_defs[0].total_rows)
print 'nn\tnn_nodes\t%0.4f' % nn.shape[0]
print 'nn\tnn_links\t%0.4f' % nn.nnz
print 'nn\tnn_density\t%0.4f' % cr_graphclust.matrix_density(nn)
sys.stdout.flush()
matrix_bin = martian.make_path('matrix.bin')
matrix_weights = martian.make_path('matrix.weights')
louvain_out = martian.make_path('louvain.out')
if args.similarity_type == 'snn':
snn = cr_graphclust.compute_snn_matrix(nn, chunk_defs[0].k_nearest)
print 'snn\tsnn_nodes\t%d' % snn.shape[0]
print 'snn\tsnn_links\t%d' % (snn.nnz / 2)
print 'snn\tsnn_density\t%0.4f' % (
(snn.nnz) / float(snn.shape[0] * (snn.shape[0] - 1)))
sys.stdout.flush()
with LogPerf('convert'):
cr_graphclust.pipe_weighted_edgelist_to_convert(
snn, matrix_bin, matrix_weights)
with LogPerf('louvain'):
cr_graphclust.run_louvain_weighted_clustering(
matrix_bin, matrix_weights, louvain_out)
else:
with LogPerf('tocoo'):
nn = nn.tocoo(copy=False)
with LogPerf('convert'):
cr_graphclust.pipe_unweighted_edgelist_to_convert(nn, matrix_bin)
with LogPerf('louvain'):
cr_graphclust.run_louvain_unweighted_clustering(
matrix_bin, louvain_out)
with LogPerf('load_bcs'):
barcodes = SingleGenomeAnalysis.load_bcs_from_matrix_h5(args.matrix_h5)
use_bcs = cr_graphclust.load_ndarray_h5(chunk_defs[0].use_bcs, 'use_bcs')
labels = cr_graphclust.load_louvain_results(len(barcodes), use_bcs,
louvain_out)
labels = cr_clustering.relabel_by_size(labels)
# Save cluster results
with analysis_io.open_h5_for_writing(outs.clusters_h5) as f:
cr_graphclust.save_graphclust_h5(f, labels)
clustering_key = cr_clustering.format_clustering_key(
cr_clustering.CLUSTER_TYPE_GRAPHCLUST, 0)
cr_clustering.save_clustering_csv(outs.clusters_csv, clustering_key,
labels, barcodes)
outs.chunked_neighbors = None
def join(args, outs, chunk_defs, chunk_outs):
if args.matrix_h5 is None:
outs.graph_clustering_summary = {}
return
outs.graph_clustering_summary = {'h5': {}, 'csv': {}}
# Merge the neighbor matrices
for method in args.factorization:
chunk_outs_def_method = [[
chunk_out, chunk_def
] for chunk_out, chunk_def in zip(chunk_outs, chunk_defs)
if chunk_def.method == method]
chunk_outs_method = [c[0] for c in chunk_outs_def_method]
chunk_defs_method = [c[1] for c in chunk_outs_def_method]
with LogPerf('merge_nn'):
nn = cr_graphclust.merge_nearest_neighbors(
[chunk.chunked_neighbors for chunk in chunk_outs_method],
chunk_defs_method[0].total_rows)
print 'nn\tnn_nodes\t%0.4f' % nn.shape[0]
print 'nn\tnn_links\t%0.4f' % nn.nnz
print 'nn\tnn_density\t%0.4f' % cr_graphclust.matrix_density(nn)
sys.stdout.flush()
matrix_bin = martian.make_path('matrix_{}.bin'.format(method))
matrix_weights = martian.make_path('matrix_{}.weights'.format(method))
louvain_out = martian.make_path('louvain_{}.out'.format(method))
if args.similarity_type == 'snn':
snn = cr_graphclust.compute_snn_matrix(
nn, chunk_defs_method[0].k_nearest)
print 'snn\tsnn_nodes\t%d' % snn.shape[0]
print 'snn\tsnn_links\t%d' % (snn.nnz / 2)
print 'snn\tsnn_density\t%0.4f' % (
(snn.nnz) / float(snn.shape[0] * (snn.shape[0] - 1)))
sys.stdout.flush()
with LogPerf('convert'):
cr_graphclust.pipe_weighted_edgelist_to_convert(
snn, matrix_bin, matrix_weights)
with LogPerf('louvain'):
cr_graphclust.run_louvain_weighted_clustering(
matrix_bin, matrix_weights, louvain_out)
else:
with LogPerf('tocoo'):
nn = nn.tocoo(copy=False)
with LogPerf('convert'):
cr_graphclust.pipe_unweighted_edgelist_to_convert(
nn, matrix_bin)
with LogPerf('louvain'):
cr_graphclust.run_louvain_unweighted_clustering(
matrix_bin, louvain_out)
with LogPerf('load_bcs'):
barcodes = None
with h5.File(args.matrix_h5, 'r') as f:
group_name = f.keys()[0]
barcodes = cr_matrix.CountMatrix.load_bcs_from_h5_group(
f[group_name])
use_bcs = cr_graphclust.load_ndarray_h5(chunk_defs_method[0].use_bcs,
'use_bcs')
labels = cr_graphclust.load_louvain_results(len(barcodes), use_bcs,
louvain_out)
labels = cr_clustering.relabel_by_size(labels)
# Save cluster results
cr_io.mkdir(outs.knn_clusters, allow_existing=True)
method_dir = os.path.join(outs.knn_clusters, method)
cr_io.mkdir(method_dir, allow_existing=True)
_h5 = os.path.join(method_dir, "clusters.h5")
_csv = os.path.join(method_dir, "clusters_csv")
with analysis_io.open_h5_for_writing(_h5) as f:
cr_graphclust.save_graphclust_h5(f, labels)
clustering_key = cr_clustering.format_clustering_key(
cr_clustering.CLUSTER_TYPE_GRAPHCLUST, 0)
cr_clustering.save_clustering_csv(_csv, clustering_key, labels,
barcodes)
outs.graph_clustering_summary['h5'][method] = _h5
outs.graph_clustering_summary['csv'][method] = _csv
outs.chunked_neighbors = None