def regresswithpca(data, modelfile, regressmode):
"""Perform univariate regression,
followed by principal components analysis
to reduce dimensionality
:param data: RDD of data points as key value pairs
:param modelfile: model parameters (string with file location, array, or tuple)
:param regressmode: form of regression ("linear" or "bilinear")
:return stats: statistics of the fit
:return comps: compoents from PCA
:return scores: scores from PCA
:return latent: latent variances from PCA
"""
# create model
model = RegressionModel.load(modelfile, regressmode)
# do regression
betas, stats, resid = model.fit(data)
# do principal components analysis
scores, latent, comps = svd(betas, 2)
# compute trajectories from raw data
traj = model.fit(data, comps)
return stats, comps, latent, scores, traj
def tuning(data, tuningmodelfile, tuningmode, regressmodelfile=None, regressmode=None):
"""Estimate parameters of a tuning curve model,
optionally preceeded by regression
:param data: RDD of data points as key value pairs
:param tuningmodelfile: model parameters for tuning (string with file location, array, or tuple)
:param: tuningmode: form of tuning ("gaussian" or "circular")
:param regressmodelfile: model parameters for regression (default=None)
:param regressmode: form of regression ("linear" or "bilinear") (default=None)
:return params: tuning curve parameters
"""
# create tuning model
tuningmodel = TuningModel.load(tuningmodelfile, tuningmode)
# get tuning curves
if regressmodelfile is not None:
# use regression results
regressmodel = RegressionModel.load(regressmodelfile, regressmode)
betas, stats, resid = regressmodel.fit(data)
params = tuningmodel.fit(betas)
else:
# use data
params = tuningmodel.fit(data)
return params
def tuning(data,
tuningmodelfile,
tuningmode,
regressmodelfile=None,
regressmode=None):
"""Estimate parameters of a tuning curve model,
optionally preceeded by regression
:param data: RDD of data points as key value pairs
:param tuningmodelfile: model parameters for tuning (string with file location, array, or tuple)
:param: tuningmode: form of tuning ("gaussian" or "circular")
:param regressmodelfile: model parameters for regression (default=None)
:param regressmode: form of regression ("linear" or "bilinear") (default=None)
:return params: tuning curve parameters
"""
# create tuning model
tuningmodel = TuningModel.load(tuningmodelfile, tuningmode)
# get tuning curves
if regressmodelfile is not None:
# use regression results
regressmodel = RegressionModel.load(regressmodelfile, regressmode)
betas, stats, resid = regressmodel.fit(data)
params = tuningmodel.fit(betas)
else:
# use data
params = tuningmodel.fit(data)
return params
def regress(data, modelfile, regressmode):
"""perform mass univariate regression,
followed by principal components analysis
to reduce dimensionality
arguments:
data - RDD of data points
modelfile - model parameters (string with file location, array, or tuple)
regressmode - form of regression ("linear" or "bilinear")
returns:
stats - statistics of the fit
comps, latent, scores, traj - results of principal components analysis
"""
# create model
model = RegressionModel.load(modelfile, regressmode)
# do regression
betas, stats, resid = model.fit(data)
# do principal components analysis
scores, latent, comps = svd(betas, 2)
# compute trajectories from raw data
traj = model.fit(data, comps)
return stats, comps, latent, scores, traj
def tuning(data, tuningmodelfile, tuningmode, regressmodelfile=None, regressmode=None):
"""estimate parameters of a tuning curve model,
optionally preceeded by regression
arguments:
data - RDD of data points
tuningmodelfile - model parameters (string with file location, array, or tuple)
tuningmode - form of tuning ("gaussian" or "circular")
regressmodelfile - model parameters (default=None)
regressmode - form of regression ("linear" or "bilinear") (default=None)
returns:
params - tuning curve parameters
"""
# create tuning model
tuningmodel = TuningModel.load(tuningmodelfile, tuningmode)
# get tuning curves
if regressmodelfile is not None:
# use regression results
regressmodel = RegressionModel.load(regressmodelfile, regressmode)
betas, stats, resid = regressmodel.fit(data)
params = tuningmodel.fit(betas)
else:
# use data
params = tuningmodel.fit(data)
return params
def regresswithpca(data, modelfile, regressmode, k=2):
"""Perform univariate regression,
followed by principal components analysis
to reduce dimensionality
:param data: RDD of data points as key value pairs
:param modelfile: model parameters (string with file location, array, or tuple)
:param regressmode: form of regression ("linear" or "bilinear")
:param k: number of principal components to compute
:return stats: statistics of the fit
:return comps: compoents from PCA
:return scores: scores from PCA
:return latent: latent variances from PCA
"""
# create model
model = RegressionModel.load(modelfile, regressmode)
# do regression
betas, stats, resid = model.fit(data)
# do principal components analysis
scores, latent, comps = svd(betas, k)
# compute trajectories from raw data
traj = model.fit(data, comps)
return stats, comps, latent, scores, traj
def test_blinear_regress(self):
data = self.sc.parallelize([(1, array([1.5, 2.3, 6.2, 5.1, 3.4,
2.1]))])
x1 = array([array([1, 0, 1, 0, 1, 0]), array([0, 1, 0, 1, 0, 1])])
x2 = array([
array([1, 1, 0, 0, 0, 0]),
array([0, 0, 1, 1, 0, 0]),
array([0, 0, 0, 0, 1, 1])
])
model = RegressionModel.load((x1, x2), "bilinear")
betas, stats, resid = model.fit(data)
tol = 1E-4 # to handle rounding errors
assert (allclose(betas.map(lambda (_, v): v).collect()[0],
array([-3.1249, 5.6875, 0.4375]),
atol=tol))
def test_linear_regress(self):
data = self.sc.parallelize([(1, array([1.5, 2.3, 6.2, 5.1, 3.4, 2.1]))])
x = array([array([1, 0, 0, 0, 0, 0]), array([0, 1, 0, 0, 0, 0])])
model = RegressionModel.load(x, "linear")
betas, stats, resid = model.fit(data)
assert allclose(betas.map(lambda (_, v): v).collect()[0], array([-2.7, -1.9]))
assert allclose(stats.map(lambda (_, v): v).collect()[0], array([0.42785299]))
assert allclose(resid.map(lambda (_, v): v).collect()[0], array([0, 0, 2, 0.9, -0.8, -2.1]))
stats, betas = regress(data, x, "linear")
stats.collect()
betas.collect()
stats, comps, latent, scores, traj = regresswithpca(data, x, "linear")
stats.collect()
scores.collect()
def regress(data, modelfile, regressmode):
"""Perform mass univariate regression
:param data: RDD of data points as key value pairs
:param modelfile: model parameters (string with file location, array, or tuple)
:param regressmode: form of regression ("linear" or "bilinear")
:return stats: statistics of the fit
:return betas: regression coefficients
"""
# create model
model = RegressionModel.load(modelfile, regressmode)
# do regression
betas, stats, resid = model.fit(data)
return stats, betas
def regress(data, modelfile, regressmode):
"""Perform mass univariate regression
:param data: RDD of data points as key value pairs
:param modelfile: model parameters (string with file location, array, or tuple)
:param regressmode: form of regression ("linear" or "bilinear")
:return stats: statistics of the fit
:return betas: regression coefficients
"""
# create model
model = RegressionModel.load(modelfile, regressmode)
# do regression
betas, stats, resid = model.fit(data)
return stats, betas
def test_blinear_regress(self):
data = self.sc.parallelize([(1, array([1.5, 2.3, 6.2, 5.1, 3.4, 2.1]))])
x1 = array([array([1, 0, 1, 0, 1, 0]), array([0, 1, 0, 1, 0, 1])])
x2 = array([array([1, 1, 0, 0, 0, 0]), array([0, 0, 1, 1, 0, 0]), array([0, 0, 0, 0, 1, 1])])
model = RegressionModel.load((x1, x2), "bilinear")
betas, stats, resid = model.fit(data)
tol = 1e-4 # to handle rounding errors
assert allclose(betas.map(lambda (_, v): v).collect()[0], array([-3.1249, 5.6875, 0.4375]), atol=tol)
assert allclose(stats.map(lambda (_, v): v).collect()[0], array([0.6735]), tol)
assert allclose(resid.map(lambda (_, v): v).collect()[0], array([0, -0.8666, 0, 1.9333, 0, -1.0666]), atol=tol)
stats, betas = regress(data, (x1, x2), "bilinear")
stats.collect()
betas.collect()
stats, comps, latent, scores, traj = regresswithpca(data, (x1, x2), "bilinear")
stats.collect()
scores.collect()
def test_linear_regress(self):
data = self.sc.parallelize([(1, array([1.5, 2.3, 6.2, 5.1, 3.4,
2.1]))])
x = array([array([1, 0, 0, 0, 0, 0]), array([0, 1, 0, 0, 0, 0])])
model = RegressionModel.load(x, "linear")
betas, stats, resid = model.fit(data)
assert (allclose(
betas.map(lambda (_, v): v).collect()[0], array([-2.7, -1.9])))
assert (allclose(
stats.map(lambda (_, v): v).collect()[0], array([0.42785299])))
assert (allclose(
resid.map(lambda (_, v): v).collect()[0],
array([0, 0, 2, 0.9, -0.8, -2.1])))
stats, betas = regress(data, x, "linear")
stats.collect()
betas.collect()
stats, comps, latent, scores, traj = regresswithpca(data, x, "linear")
stats.collect()
scores.collect()
def runtest(self):
model = RegressionModel.load(os.path.join(self.modelfile, "linear"), "linear")
betas, stats, resid = model.fit(self.rdd)
result = stats.map(lambda (_, v): float16(v)).collect()
savemat(self.savefile + "tmp.mat", mdict={"tmp": result}, oned_as='column')
def runtest(self):
model = RegressionModel.load(os.path.join(self.modelfile, "linear"), "linear")
betas, stats, resid = model.fit(self.rdd)
stats.count()