def find_git_repo():
from sos.utils import get_output
try:
res = get_output('git rev-parse --is-inside-work-tree').strip()
return get_output(
'git rev-parse --show-toplevel').strip() if res == 'true' else None
except Exception:
return None
def __R(code):
try:
output = get_output(
f"R --slave -e \"cat(dscrutils::dscreval({repr(code[1:-1])}))\""
).strip()
except Exception:
from .utils import install_package_interactive
from .version import __version__
install_package_interactive(
f'dscrutils@stephenslab/dsc/dscrutils>={__version__}',
'R_library')
try:
output = get_output(
f"R --slave -e \"cat(dscrutils::dscreval({repr(code[1:-1])}))\""
).strip()
except Exception:
raise ValueError(
f"Failed to evaluate R expression ``{code[1:-1]}``")
return output
def get_rlib_versions(rlibs):
from sos.utils import get_output
rlibs = uniq_list(
sorted([
x.split()[0].split('@')[0] for x in rlibs
if not x.startswith('dscrutils')
]))
versions = []
for l in rlibs:
try:
versions.append(
get_output(
f'Rscript -e "cat(toString(packageVersion(\'{l}\')))"'))
except Exception:
versions.append('No version info available on local machine')
return rlibs, versions
def testSyntaxPass(self):
self.touch(['1.csv'])
# use built-in group names
res = Query_Processor(
ash_db, 'simulate.nsamp shrink.mixcompdist score.mse'.split(),
['simulate.nsamp > 20', 'shrink.mixcompdist = "normal"'], [])
q1 = test_outcome(res, '1.csv')
q2 = [
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".nsamp AS simulate_DSC_FIELD_nsamp, "shrink".mixcompdist AS shrink_DSC_FIELD_mixcompdist, "score_beta".__output__ AS score_beta_DSC_VAR_mse FROM "score_beta" INNER JOIN "shrink" ON "score_beta".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))',
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".nsamp AS simulate_DSC_FIELD_nsamp, "shrink".mixcompdist AS shrink_DSC_FIELD_mixcompdist, "score_pi0".__output__ AS score_pi0_DSC_VAR_mse FROM "score_pi0" INNER JOIN "shrink" ON "score_pi0".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))'
]
self.assertEqual(q1, q2)
# Handle empty field input: return the file path
res = Query_Processor(
ash_db, 'simulate shrink score'.split(),
['simulate.nsamp > 20', 'shrink.mixcompdist = "normal"'], [])
q1 = test_outcome(res, '1.csv')
q2 = [
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".__output__ AS simulate_DSC_OUTPUT_, "shrink".__output__ AS shrink_DSC_OUTPUT_, "score_beta".__output__ AS score_beta_DSC_OUTPUT_ FROM "score_beta" INNER JOIN "shrink" ON "score_beta".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))',
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".__output__ AS simulate_DSC_OUTPUT_, "shrink".__output__ AS shrink_DSC_OUTPUT_, "score_pi0".__output__ AS score_pi0_DSC_OUTPUT_ FROM "score_pi0" INNER JOIN "shrink" ON "score_pi0".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))'
]
self.assertEqual(q1, q2)
# handle group merger
res = Query_Processor(
reg_db, 'simulate.scenario analyze score score.error'.split(), [],
[])
q = test_outcome(res, '1.csv')
observed = sorted(
get_output('(head -n 2 1.csv && tail -n +3 1.csv | sort) | head').
strip().split('\n'))
expected = sorted('''
DSC,simulate,simulate.scenario,analyze,analyze.output.file,score,score.output.file,score.error:output
1,en_sim,eg1,lasso,lasso/en_sim_1_lasso_1,sq_err,sq_err/en_sim_1_lasso_1_sq_err_1,sq_err/en_sim_1_lasso_1_sq_err_1
1,dense,NA,en,en/dense_1_en_1,sq_err,sq_err/dense_1_en_1_sq_err_1,sq_err/dense_1_en_1_sq_err_1
1,dense,NA,lasso,lasso/dense_1_lasso_1,sq_err,sq_err/dense_1_lasso_1_sq_err_1,sq_err/dense_1_lasso_1_sq_err_1
1,dense,NA,ridge,ridge/dense_1_ridge_1,sq_err,sq_err/dense_1_ridge_1_sq_err_1,sq_err/dense_1_ridge_1_sq_err_1
1,en_sim,eg1,en,en/en_sim_1_en_1,sq_err,sq_err/en_sim_1_en_1_sq_err_1,sq_err/en_sim_1_en_1_sq_err_1
1,en_sim,eg1,ridge,ridge/en_sim_1_ridge_1,sq_err,sq_err/en_sim_1_ridge_1_sq_err_1,sq_err/en_sim_1_ridge_1_sq_err_1
1,en_sim,eg2,en,en/en_sim_2_en_1,sq_err,sq_err/en_sim_2_en_1_sq_err_1,sq_err/en_sim_2_en_1_sq_err_1
1,en_sim,eg2,lasso,lasso/en_sim_2_lasso_1,sq_err,sq_err/en_sim_2_lasso_1_sq_err_1,sq_err/en_sim_2_lasso_1_sq_err_1
1,en_sim,eg2,ridge,ridge/en_sim_2_ridge_1,sq_err,sq_err/en_sim_2_ridge_1_sq_err_1,sq_err/en_sim_2_ridge_1_sq_err_1
'''.strip().split('\n'))
self.assertEqual(observed, expected)
# another group merger test
# FIXME: this test fails on CircleCI:
# ======================================================================
#FAIL: testSyntaxPass (__main__.TestQuery)
#----------------------------------------------------------------------
#Traceback (most recent call last):
# File "test_query.py", line 118, in testSyntaxPass
# self.assertEqual(observed, expected)
#AssertionError: Lists differ: ['1,0[256 chars],0.0,NA,NA,NA,NA,q_prob_large,q_prob_large/sim[852 chars]put'] != ['1,0[256 chars],0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_a[1224 chars]put']
#
#First differing element 3:
#'1,0.0,NA,NA,NA,NA,q_prob_large,q_prob_large/sim[37 chars]ge_1'
#'1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_a[119 chars]A,NA'
#
# ['1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_1_gamma_lfsr_1',
# '1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_2_gamma_lfsr_1',
# '1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_3_gamma_lfsr_1',
#- '1,0.0,NA,NA,NA,NA,q_prob_large,q_prob_large/simulate_1_cause_grid_adapt_1_q_prob_large_1',
#- '1,0.0,NA,NA,NA,NA,q_prob_large,q_prob_large/simulate_1_cause_grid_adapt_2_q_prob_large_1',
#- '1,0.0,NA,NA,NA,NA,q_prob_large,q_prob_large/simulate_1_cause_grid_adapt_3_q_prob_large_1',
# '1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,NA,NA',
# '1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,NA,NA',
# '1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,NA,NA',
#+ '1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,NA,NA',
#+ '1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,NA,NA',
#+ '1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,NA,NA',
# 'DSC,simulate.q,cis,cis.output.file,cis.ci_upr:output,cis.ci_lwr:output,summ_probs,summ_pr
res = Query_Processor(
cause_db,
'simulate.q cis.ci_lwr cis.ci_upr summ_probs.prob cis'.split())
q = test_outcome(res, '1.csv')
observed = sorted(
get_output('(head -n 2 1.csv && tail -n +3 1.csv | sort) | head').
strip().split('\n'))
expected = sorted('''
DSC,simulate.q,cis,cis.output.file,cis.ci_upr:output,cis.ci_lwr:output,summ_probs,summ_probs.prob:output
1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,NA,NA
1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,NA,NA
1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,NA,NA
1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,NA,NA
1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,NA,NA
1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,NA,NA
1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_1_gamma_lfsr_1
1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_2_gamma_lfsr_1
1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_3_gamma_lfsr_1
'''.strip().split('\n'))
def __Shell(code):
# FIXME: is this behavior any good?
out = get_output(code[1:-1])
return ','.join(
flatten_list([x.split() for x in out.strip().split("\n")]))
def testSyntaxPass(self):
self.touch(['1.csv'])
# use built-in group names
res = Query_Processor(
ash_db, 'simulate.nsamp shrink.mixcompdist score.mse'.split(),
['simulate.nsamp > 20', 'shrink.mixcompdist = "normal"'], [])
q1 = test_outcome(res, '1.csv')
q2 = [
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".nsamp AS simulate_DSC_FIELD_nsamp, "shrink".mixcompdist AS shrink_DSC_FIELD_mixcompdist, "score_beta".__output__ AS score_beta_DSC_VAR_mse FROM "score_beta" INNER JOIN "shrink" ON "score_beta".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))',
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".nsamp AS simulate_DSC_FIELD_nsamp, "shrink".mixcompdist AS shrink_DSC_FIELD_mixcompdist, "score_pi0".__output__ AS score_pi0_DSC_VAR_mse FROM "score_pi0" INNER JOIN "shrink" ON "score_pi0".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))'
]
self.assertEqual(q1, q2)
# Handle empty field input: return the file path
res = Query_Processor(
ash_db, 'simulate shrink score'.split(),
['simulate.nsamp > 20', 'shrink.mixcompdist = "normal"'], [])
q1 = test_outcome(res, '1.csv')
q2 = [
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".__output__ AS simulate_DSC_OUTPUT_, "shrink".__output__ AS shrink_DSC_OUTPUT_, "score_beta".__output__ AS score_beta_DSC_OUTPUT_ FROM "score_beta" INNER JOIN "shrink" ON "score_beta".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))',
'SELECT "simulate".DSC_REPLICATE AS simulate_DSC_FIELD_DSC_REPLICATE, "simulate".__output__ AS simulate_DSC_OUTPUT_, "shrink".__output__ AS shrink_DSC_OUTPUT_, "score_pi0".__output__ AS score_pi0_DSC_OUTPUT_ FROM "score_pi0" INNER JOIN "shrink" ON "score_pi0".__parent__ = "shrink".__id__ INNER JOIN "simulate" ON "shrink".__parent__ = "simulate".__id__ WHERE (("simulate".nsamp > 20) AND ("shrink".mixcompdist == "normal"))'
]
self.assertEqual(q1, q2)
# handle group merger
res = Query_Processor(
reg_db, 'simulate.scenario analyze score score.error'.split(), [],
[])
q = test_outcome(res, '1.csv')
observed = get_output(
'(head -n 2 1.csv && tail -n +3 1.csv | sort) | head').strip(
).split('\n')
expected = '''
DSC,simulate,simulate.scenario,analyze,analyze.output.file,score,score.output.file,score.error:output
1,en_sim,eg1,lasso,lasso/en_sim_1_lasso_1,sq_err,sq_err/en_sim_1_lasso_1_sq_err_1,sq_err/en_sim_1_lasso_1_sq_err_1
1,dense,NA,en,en/dense_1_en_1,sq_err,sq_err/dense_1_en_1_sq_err_1,sq_err/dense_1_en_1_sq_err_1
1,dense,NA,lasso,lasso/dense_1_lasso_1,sq_err,sq_err/dense_1_lasso_1_sq_err_1,sq_err/dense_1_lasso_1_sq_err_1
1,dense,NA,ridge,ridge/dense_1_ridge_1,sq_err,sq_err/dense_1_ridge_1_sq_err_1,sq_err/dense_1_ridge_1_sq_err_1
1,en_sim,eg1,en,en/en_sim_1_en_1,sq_err,sq_err/en_sim_1_en_1_sq_err_1,sq_err/en_sim_1_en_1_sq_err_1
1,en_sim,eg1,ridge,ridge/en_sim_1_ridge_1,sq_err,sq_err/en_sim_1_ridge_1_sq_err_1,sq_err/en_sim_1_ridge_1_sq_err_1
1,en_sim,eg2,en,en/en_sim_2_en_1,sq_err,sq_err/en_sim_2_en_1_sq_err_1,sq_err/en_sim_2_en_1_sq_err_1
1,en_sim,eg2,lasso,lasso/en_sim_2_lasso_1,sq_err,sq_err/en_sim_2_lasso_1_sq_err_1,sq_err/en_sim_2_lasso_1_sq_err_1
1,en_sim,eg2,ridge,ridge/en_sim_2_ridge_1,sq_err,sq_err/en_sim_2_ridge_1_sq_err_1,sq_err/en_sim_2_ridge_1_sq_err_1
'''.strip().split('\n')
self.assertEqual(observed, expected)
# another group merger test
res = Query_Processor(
cause_db,
'simulate.q cis.ci_lwr cis.ci_upr summ_probs.prob cis'.split())
q = test_outcome(res, '1.csv')
observed = get_output(
'(head -n 2 1.csv && tail -n +3 1.csv | sort) | head').strip(
).split('\n')
expected = '''
DSC,simulate.q,cis,cis.output.file,cis.ci_upr:output,cis.ci_lwr:output,summ_probs,summ_probs.prob:output
1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_1_gamma_ci_1,NA,NA
1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_2_gamma_ci_1,NA,NA
1,0.0,gamma_ci,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,gamma_ci/simulate_1_cause_grid_adapt_3_gamma_ci_1,NA,NA
1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_1_gamma_prime_ci_1,NA,NA
1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_2_gamma_prime_ci_1,NA,NA
1,0.0,gamma_prime_ci,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,gamma_prime_ci/simulate_1_cause_grid_adapt_3_gamma_prime_ci_1,NA,NA
1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_1_gamma_lfsr_1
1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_2_gamma_lfsr_1
1,0.0,NA,NA,NA,NA,gamma_lfsr,gamma_lfsr/simulate_1_cause_grid_adapt_3_gamma_lfsr_1
'''.strip().split('\n')
self.assertEqual(observed, expected)
def __Shell(code):
# FIXME: is this behavior any good?
import shutil
out = get_output(code[1:-1], executable=shutil.which('bash'), shell=True)
return ','.join(
flatten_list([x.split() for x in out.strip().split("\n")]))