def test_crosscorr(self):
data_local = array([
array([1.0, 2.0, -4.0, 5.0, 8.0, 3.0, 4.1, 0.9, 2.3]),
array([2.0, 2.0, -4.0, 5.0, 3.1, 4.5, 8.2, 8.1, 9.1]),
])
sig = array([1.5, 2.1, -4.2, 5.6, 8.1, 3.9, 4.2, 0.3, 2.1])
data = self.sc.parallelize(zip(range(1, 3), data_local))
method = CrossCorr(sigfile=sig, lag=0)
betas = method.calc(data).map(lambda (_, v): v)
assert(allclose(betas.collect()[0], corrcoef(data_local[0, :], sig)[0, 1]))
assert(allclose(betas.collect()[1], corrcoef(data_local[1, :], sig)[0, 1]))
method = CrossCorr(sigfile=sig, lag=2)
betas = method.calc(data).map(lambda (_, v): v)
tol = 1E-5 # to handle rounding errors
assert(allclose(betas.collect()[0], array([-0.18511, 0.03817, 0.99221, 0.06567, -0.25750]), atol=tol))
assert(allclose(betas.collect()[1], array([-0.35119, -0.14190, 0.44777, -0.00408, 0.45435]), atol=tol))
def test_crosscorr(self):
data_local = array([
array([1.0, 2.0, -4.0, 5.0, 8.0, 3.0, 4.1, 0.9, 2.3]),
array([2.0, 2.0, -4.0, 5.0, 3.1, 4.5, 8.2, 8.1, 9.1]),
])
sig = array([1.5, 2.1, -4.2, 5.6, 8.1, 3.9, 4.2, 0.3, 2.1])
data = self.sc.parallelize(zip(range(1, 3), data_local))
method = CrossCorr(sigfile=sig, lag=0)
betas = method.calc(data).map(lambda (_, v): v)
assert (allclose(betas.collect()[0],
corrcoef(data_local[0, :], sig)[0, 1]))
assert (allclose(betas.collect()[1],
corrcoef(data_local[1, :], sig)[0, 1]))
method = CrossCorr(sigfile=sig, lag=2)
betas = method.calc(data).map(lambda (_, v): v)
tol = 1E-5 # to handle rounding errors
assert (allclose(betas.collect()[0],
array([-0.18511, 0.03817, 0.99221, 0.06567,
-0.25750]),
atol=tol))
assert (allclose(betas.collect()[1],
array(
[-0.35119, -0.14190, 0.44777, -0.00408, 0.45435]),
atol=tol))
def runtest(self):
method = CrossCorr(sigfile=os.path.join(self.modelfile, "crosscorr"), lag=0)
betas = method.calc(self.rdd)
betas.count()
def runtest(self):
method = CrossCorr(sigfile=os.path.join(self.modelfile, "crosscorr"),
lag=0)
betas = method.calc(self.rdd)
betas.count()
parser.add_argument("datafile", type=str)
parser.add_argument("sigfile", type=str)
parser.add_argument("outputdir", type=str)
parser.add_argument("lag", type=int)
parser.add_argument("--preprocess",
choices=("raw", "dff", "dff-highpass", "sub"),
default="raw",
required=False)
args = parser.parse_args()
sc = SparkContext(args.master, "crosscorr")
if args.master != "local":
egg = glob.glob(os.path.join(os.environ['THUNDER_EGG'], "*.egg"))
sc.addPyFile(egg[0])
data = load(sc, args.datafile, args.preprocess).cache()
outputdir = args.outputdir + "-crosscorr"
# post-process data with pca if lag greater than 0
vals = CrossCorr(args.sigfile, args.lag).calc(data)
if args.lag is not 0:
out = PCA(2).fit(vals)
save(out.comps, outputdir, "comps", "matlab")
save(out.latent, outputdir, "latent", "matlab")
save(out.scores, outputdir, "scores", "matlab")
else:
save(vals, outputdir, "betas", "matlab")