def from_signatures(cls,
signatures,
standardize=False,
center=True,
use_median=True,
cluster_signatures=True,
signature_cluster_metric='correlation',
cluster_samples=True,
sample_cluster_metric='euclidean',
cluster_method='average'):
"""Generate a GO-PCA signature matrix from individual signatures.
The GO-PCA signature matrix contains the expression levels of all
signatures (rows) generated, across all samples (columns) in the
analysis. See the documentation of the `GOPCASignature` class for
details on how signature expression levels are calculated.
Parameters
----------
signatures: Iterable of `GOPCASignature`
The signatures generated.
"""
# TODO: finish docstring
assert isinstance(signatures, Iterable)
assert isinstance(standardize, bool)
assert isinstance(center, bool)
assert isinstance(use_median, bool)
assert isinstance(cluster_signatures, bool)
assert isinstance(cluster_samples, bool)
### generate the expression matrix
matrix = ExpMatrix(
pd.concat([
sig.get_expression(standardize=standardize,
center=center,
use_median=use_median) for sig in signatures
],
axis=1).T)
matrix.genes.name = 'Signatures'
matrix.samples.name = 'Samples'
if matrix.p == 1:
cluster_signatures = False
cluster_samples = False
### clustering
if cluster_signatures:
# cluster signatures
matrix = cluster.cluster_genes(matrix,
metric=signature_cluster_metric,
method=cluster_method)
order_samples = None
if cluster_samples:
# cluster samples
matrix = cluster.cluster_samples(matrix,
metric=sample_cluster_metric,
method=cluster_method)
return cls(matrix)
def main(args=None):
vinfo = sys.version_info
if not (vinfo >= (2, 7)):
raise SystemError('Python interpreter version >= 2.7 required, '
'found %d.%d instead.' % (vinfo.major, vinfo.minor))
if args is None:
parser = get_argument_parser()
args = parser.parse_args()
gopca_file = args.gopca_file
output_file = args.output_file
#sig_max_len = args.sig_max_len
#sig_reverse_order = args.sig_reverse_order
#sample_cluster_metric = args.sample_cluster_metric
#no_sample_clustering = args.no_sample_clustering
# configure root logger
log_file = args.log_file
quiet = args.quiet
verbose = args.verbose
logger = misc.get_logger(log_file=log_file, quiet=quiet,
verbose=verbose)
result = util.read_gopca_result(gopca_file)
sig_matrix = util.read_gopca_result(gopca_file)
sig_labels = [sig.get_label(include_id=False)
for sig in sig_matrix.signatures]
matrix = ExpMatrix(genes=sig_labels, samples=sig_matrix.samples,
X=sig_matrix.X)
matrix.index.name = 'Signatures'
#signatures = result.signatures
#sig_labels = [sig.get_label(max_name_length=sig_max_len, include_id=False)
# for sig in signatures]
#samples = list(result.samples)
# generate expression matrix
#E = ExpMatrix(genes=sig_labels, samples=samples, X=sig_matrix.X)
# clustering of signatures (rows)
#E, _ = cluster.cluster_genes(E, reverse=sig_reverse_order)
exp_logger = logging.getLogger(expression.__name__)
exp_logger.setLevel(logging.WARNING)
matrix.write_tsv(output_file)
exp_logger.setLevel(logging.NOTSET)
logger.info('Wrote %d x %d signature matrix to "%s".',
matrix.p, matrix.n, output_file)
return 0
def get_heatmap(self,
sig_matrix=None,
standardize=False,
center=True,
use_median=True,
include_id=False,
include_stats=True,
include_pval=True,
cluster_genes=True,
gene_cluster_metric='correlation',
cluster_samples=True,
sample_cluster_metric='euclidean',
cluster_method='average',
colorbar_label=None,
**kwargs):
"""Generate a heatmap of the signature gene matrix."""
# TODO: Finish docstring
assert isinstance(cluster_genes, bool)
assert isinstance(cluster_samples, bool)
assert isinstance(gene_cluster_metric, (str, _oldstr))
assert isinstance(sample_cluster_metric, (str, _oldstr))
assert isinstance(cluster_method, (str, _oldstr))
from . import GOPCASignatureMatrix
if sig_matrix is not None:
assert isinstance(sig_matrix, GOPCASignatureMatrix)
if colorbar_label is None:
colorbar_label = 'Centered expression'
matrix = self.matrix.copy()
if standardize:
matrix.standardize_genes(inplace=True)
cb_default_label = ('Standardized expression<br>'
'(based on log<sub>2</sub>-scale)')
elif center:
matrix.center_genes(use_median=use_median, inplace=True)
cb_default_label = 'Centered expression<br>(log<sub>2</sub>-scale)'
else:
cb_default_label = 'Expression<br>(log<sub>2</sub>-scale)'
if colorbar_label is None:
colorbar_label = cb_default_label
# clustering
if sig_matrix is not None:
# user has provided a GOPCASignatureMatrix instance
# make sure its samples match the signature's
logger.info('Ordering samples to match order in signature matrix.')
assert set(sig_matrix.samples) == set(self.samples.values)
# re-arrange samples according to clustering of signature matrix
matrix = matrix.loc[:, sig_matrix.samples]
elif cluster_samples:
# cluster samples (only if no signature matrix is provided)
matrix = cluster.cluster_samples(matrix,
metric=sample_cluster_metric,
method=cluster_method)
if cluster_genes:
# cluster genes
matrix = cluster.cluster_genes(matrix,
metric=gene_cluster_metric,
method=cluster_method)
# add a "Signature"-labeled row to the top,
# which represents the signature expression vector
title = self.get_label(include_id=include_id,
include_stats=include_stats,
include_pval=include_pval)
mean = np.mean(matrix.X, axis=0)
header_row = ExpMatrix(genes=['<b>Signature</b>'],
samples=matrix.samples,
X=np.atleast_2d(mean))
combined_matrix = pd.concat([header_row, matrix], axis=0)
heatmap = ExpHeatmap(combined_matrix,
title=title,
colorbar_label=colorbar_label,
**kwargs)
return heatmap
def my_matrix():
genes = ['a', 'b', 'c', 'd', 'e', 'f']
samples = ['s1', 's2', 's3']
X = np.arange(18, dtype=np.float64).reshape(6, 3)
matrix = ExpMatrix(genes=genes, samples=samples, X=X)
return matrix
def main(args=None):
vinfo = sys.version_info
if not (vinfo >= (2, 7)):
raise SystemError('Python interpreter version >= 2.7 required, '
'found %d.%d instead.' % (vinfo.major, vinfo.minor))
if args is None:
parser = get_argument_parser()
args = parser.parse_args()
expression_file = args.expression_file
entrez2gene_file = args.entrez2gene_file
gene_file = args.gene_file
output_file = args.output_file
strip_affy_suffix = args.strip_affy_suffix
log_file = args.log_file
quiet = args.quiet
verbose = args.verbose
# configure root logger
logger = misc.get_logger(log_file=log_file, quiet=quiet, verbose=verbose)
# read data
genome = ExpGeneTable.read_tsv(gene_file)
matrix = ExpMatrix.read_tsv(expression_file)
e2g = dict(misc.read_all(entrez2gene_file))
entrez = matrix.genes
if strip_affy_suffix:
# remove "_at" suffix from Entrez IDs
entrez = [e[:-3] for e in entrez]
logger.debug(str(entrez[:3]))
# check that Entrez IDs are unique
assert len(entrez) == len(set(entrez))
# convert Entrez IDs to gene names
f = 0
genes = []
X = []
# g = None
for i, e in enumerate(entrez):
# print e
try:
g = e2g[e]
except KeyError:
f += 1
else:
# check if there are multiple entrez IDs pointing to the same gene
# assert g not in genes
genes.append(g)
X.append(matrix.X[i, :])
assert len(genes) == len(set(genes))
if f > 0:
logger.warning(
'Failed to convert %d / %d entrez IDs '
'to gene symbols (%.1f%%).', f, matrix.p,
100 * (f / float(matrix.p)))
# filter for known protein-coding genes
X = np.float64(X)
p = X.shape[0]
logger.debug(str(X.shape))
sel = np.zeros(p, dtype=np.bool_)
for i in range(p):
if genes[i] in genome:
sel[i] = True
sel = np.nonzero(sel)[0]
genes = [genes[i] for i in sel]
X = X[sel, :]
f = p - sel.size
if f > 0:
logger.warning(
'Failed to find %d / %d gene symbols in list of '
'protein-coding genes (%.1f%%)', f, p, 100 * (f / float(p)))
# generate new matrix (this automatically sorts the genes alphabetically)
logger.debug('Genes: %d, Samples: %d, matrix: %s', len(genes),
len(matrix.samples), str(X.shape))
matrix_conv = ExpMatrix(genes=genes, samples=matrix.samples, X=X)
# write output file
matrix_conv.write_tsv(output_file)
return 0
def rma(cdf_file,
sample_cel_files,
pm_probes_only=True,
bg_correct=True,
quantile_normalize=True,
medianpolish=True):
"""Perform RMA on a set of samples.
Parameters
----------
cdf_file: str
The path of the Brainarray CDF file to use.
Note: Brainarray CDF files can be downloaded from
http://brainarray.mbni.med.umich.edu/Brainarray/Database/CustomCDF/genomic_curated_CDF.asp
sample_cel_files: collections.OrderedDict (st => str)
An ordered dictionary where each key/value-pair corresponds to a
sample. The *key* is the sample name, and the *value* is the (absolute)
path of the corresponding CEL file. The CEL files can be gzip'ed.
pm_probes_only: bool, optional
Whether or not to only use PM (perfect match) probes and ignore all MM
(mismatch) probes. [True]
bg_correct: bool, optional
Whether or not to apply background correction. [True]
quantile_normalize: bool, optional
Whether or not to apply quantile normalization. [True]
medianpolish: bool, optional
Whether or not to apply medianpolish. [True]
Returns
-------
genes: tuple of str
The list of gene names.
samples: tuple of str
The list of sample names.
X: np.ndarray (ndim = 2, dtype = np.float32)
The expression matrix (genes-by-samples).
Examples
--------
>>> from collections import OrderedDict
>>> import pyaffy
>>> cdf_file = '/path/to/brainarray/cdf/HGU133Plus2_Hs_ENTREZG.cdf'
>>> sample_cel_files = OrderedDict([
['Sample 1', '/path/to/sample_1.CEL.gz'],
['Sample 2', '/path/to/sample_2.CEL.gz'],
])
>>> genes, samples, X = pyaffy.rma(cdf_file, sample_cel_files)
"""
### checks
assert isinstance(cdf_file, (str, _oldstr))
assert os.path.isfile(cdf_file), \
'CDF file "%s" does not exist!' %(cdf_file)
assert isinstance(sample_cel_files, collections.OrderedDict)
for sample, cel_file in sample_cel_files.items():
assert isinstance(sample, (str, _oldstr))
assert isinstance(cel_file, (str, _oldstr))
assert os.path.isfile(cel_file), \
'CEL file "%s" does not exist!' %(cel_file)
assert isinstance(pm_probes_only, bool)
assert isinstance(bg_correct, bool)
assert isinstance(quantile_normalize, bool)
assert isinstance(medianpolish, bool)
t00 = time.time()
### read CDF data
logger.info('Parsing CDF file.')
t0 = time.time()
# parse the CDF file
probe_type = 'pm'
if not pm_probes_only:
probe_type = 'all'
name, num_rows, num_cols, pm_probesets = \
parse_cdf(cdf_file, probe_type=probe_type)
# concatenate indices of all PM probes into one long vector
pm_sel = np.concatenate(list(pm_probesets.values()))
t1 = time.time()
logger.info('CDF file parsing time: %.2f s', t1 - t0)
logger.info('CDF array design name: %s', name)
logger.info('CDF rows / columns: %d x %d', num_rows, num_cols)
### read CEL data
logger.info('Parsing CEL files...')
t0 = time.time()
p = pm_sel.size
n = len(sample_cel_files)
Y = np.empty((p, n), dtype=np.float32)
samples = []
sub_logger = logging.getLogger(celparser.__name__)
sub_logger.setLevel(logging.WARNING)
for j, (sample, cel_file) in enumerate(sample_cel_files.items()):
logger.debug('Parsing CEL file for sample "%s": %s', sample, cel_file)
samples.append(sample)
y = parse_cel(cel_file)
Y[:, j] = y[pm_sel]
sub_logger.setLevel(logging.NOTSET)
t1 = time.time()
logger.info('CEL files parsing time: %.1f s.', t1 - t0)
### background correction
if bg_correct:
logger.info('Performing background correction...')
t0 = time.time()
Y = rma_bg_correct(Y)
t1 = time.time()
logger.info('Background correction time: %.1f s.', t1 - t0)
else:
logger.info('Skipping background correction.')
matrix = ExpMatrix(genes=pm_sel, samples=samples, X=Y)
### quantile normalization
if quantile_normalize:
logger.info('Performing quantile normalization...')
t0 = time.time()
matrix = qnorm(matrix)
t1 = time.time()
logger.info('Quantile normalization time: %.1f s.', t1 - t0)
else:
logger.info('Skipping quantile normalization.')
### convert intensities to log2-scale
Y = np.log2(matrix.values)
### probeset summarization (with or without median polish)
method = 'with'
if not medianpolish:
method = 'without'
logger.info('Summarize probeset intensities (%s medianpolish)...', method)
t0 = time.time()
p = len(pm_probesets)
n = Y.shape[1]
X = np.empty((p, n), dtype=np.float32)
cur = 0
num_converged = 0
genes = []
for i, (gene_id, probes) in enumerate(pm_probesets.items()):
genes.append(gene_id)
if medianpolish:
#X_sub = np.ascontiguousarray(Y[cur:(cur + probes.size),:])
X_sub = Y[cur:(cur + probes.size), :]
_, row_eff, col_eff, global_eff, converged, num_iter = medpolish(
X_sub, copy=False)
X[i, :] = col_eff + global_eff
if converged:
num_converged += 1
else:
# simply use median across probes
X[i, :] = np.median(Y[cur:(cur + probes.size), :], axis=0)
#X[i,:] = np.ma.median(X_sub, axis = 0)
cur += probes.size
t1 = time.time()
logger.info('Probeset summarization time: %.2f s.', t1 - t0)
if medianpolish:
logger.debug('Converged: %d / %d (%.1f%%)', num_converged, p,
100 * (num_converged / float(p)))
### report total time
t11 = time.time()
logger.info('Total RMA time: %.1f s.', t11 - t00)
### sort alphabetically by gene name
a = np.lexsort([genes])
genes = [genes[i] for i in a]
X = X[a, :]
return genes, samples, X
def my_matrix(my_gene_names, my_samples, my_X):
#genes = ['a', 'b', 'c', 'd']
#samples = ['s1', 's2', 's3']
# X = np.arange(12, dtype=np.float64).reshape(4, 3)
matrix = ExpMatrix(genes=my_gene_names, samples=my_samples, X=my_X)
return matrix
