import gc
gc.collect()
for ii in range(1000):
temp_object = [1,2,3]
# don't include the peaks
df = df.head(184)
# try constant lld
#df['lld'] = 175
# test with emcee deconvolution
deconv_func_kwargs = dict(model_version='fft', iterations=500, thin=5)
dfret2 = util_emcee.deconv_in_chunks(df, 'lld', one_sided_prf, 6, max_chunklen=184, n_jobs=4, deconv_func_kwargs=deconv_func_kwargs)
dfret2.to_pickle('data-processed/lab_test_emcee_deconv.pkl')
# simulated data test
dfsim = util.get_simulated_data()
dfsim = util_emcee.deconv_in_chunks(dfsim, 'lld', one_sided_prf, 6, max_chunklen=184, n_jobs=4, deconv_func_kwargs=deconv_func_kwargs)
dfsim.to_pickle('data-processed/simulated_lab_test_emcee_deconv.pkl')
if False:
# test with stan deconvolution
deconv_func_kwargs = dict(model_version='log_difference')
dfret = util_emcee.deconv_in_chunks(df, 'lld', one_sided_prf, 6, max_chunklen=184, deconv_func=util_stan.stan_deconvolve, n_jobs=1, deconv_func_kwargs=deconv_func_kwargs)
dfret.to_pickle('data-processed/lab_test_stan_deconv.pkl')
observed_data = df.lld.values
N = len(observed_data)
# model versions: lognormal_difference, lognormal_from_data, lognormal, uniform
if True:
if job_opt in ['1', 'all']:
# stan methods
for model_version in "standard log_difference".split():
deconv_func_kwargs = dict(model_version = model_version,
column_name = 'lld_'+model_version+'_deconv',
chains=8, n_jobs=8)
df = util_emcee.deconv_in_chunks(df, 'lld', one_sided_prf, 20,
max_chunklen=205,
deconv_func=util_stan.stan_deconvolve,
n_jobs=1,
deconv_func_kwargs=deconv_func_kwargs,
parallel_backend='threading')
df.to_csv('data-processed/lab_test_deconv.csv')
df.to_pickle('data-processed/lab_test_deconv.pkl')
if job_opt in ['2', 'all']:
#emcee methods
for model_version in "lognormal_difference lognormal_from_data lognormal uniform multiscale".split():
deconv_func_kwargs = dict(model_version = model_version,
column_name = 'lld_'+model_version+'_deconv')
df = util_emcee.deconv_in_chunks(df, 'lld', one_sided_prf, 10, max_chunklen=205,
n_jobs=8,
deconv_func_kwargs=deconv_func_kwargs)
df.to_csv('data-processed/lab_test_deconv.csv')
df.to_pickle('data-processed/lab_test_deconv.pkl')
