type=int,
default=2,
help='Number of tails (1 or 2)')
args, unknown = argparser.parse_known_args()
assert args.metric in ['loss', 'loglik'
], 'Metric must be one of ["loss", "loglik"].'
if args.pool:
args.ablation = True
ablations = None
basenames_to_pool = None
exps_outdirs = []
for path in args.config_paths:
p = Config(path)
models = filter_models(p.model_list, args.models)
cdr_models = [
x for x in models if (x.startswith('CDR') or x.startswith('DTSR'))
]
partitions = get_partition_list(args.partition)
partition_str = '-'.join(partitions)
if args.metric == 'loss':
file_name = 'losses_mse_%s.txt' % partition_str
else:
file_name = 'loglik_%s.txt' % partition_str
if args.twostep:
file_name = 'LM_2STEP_' + file_name
'Type of significance test to use (one of ["pt", "2step"] for permutation testing or 2-step LME/LRT, respectively).'
)
argparser.add_argument(
'-p',
'--partition',
type=str,
default='dev',
help='Name of partition to use (one of "train", "dev", "test")')
argparser.add_argument(
'-H',
'--human_readable',
action='store_true',
help='Return table in human readable format (otherwise return as CSV)')
args, unknown = argparser.parse_known_args()
p = Config(args.config_path)
partitions = get_partition_list(args.partition)
partition_str = '-'.join(partitions)
suffix = '_2stepLRT_%s.txt' % partition_str if args.mode == '2step' else '_PT_%s.txt' % partition_str
paths = [
p.outdir + '/' + x for x in os.listdir(p.outdir) if x.endswith(suffix)
]
models, comparisons, comparisons_converged = extract_comparisons(paths)
comparison_keys = sorted(list(comparisons.keys()), key=lambda x: len(x[0]))
models = sorted(models)
cols = ['model']
for c in comparison_keys:
'-e',
'--force_training_evaluation',
action='store_true',
help=
'Recompute training evaluation even for models that are already finished.'
)
argparser.add_argument(
'-s',
'--save_and_exit',
action='store_true',
help=
'Initialize, save, and exit (CDR only). Useful for bringing non-backward compatible trained models up to spec for plotting and evaluation.'
)
args, unknown = argparser.parse_known_args()
p = Config(args.config_path)
if not p.use_gpu_if_available:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
models = filter_models(p.model_list, args.models)
run_baseline = False
run_cdr = False
for m in models:
if not run_baseline and m.startswith('LM') or m.startswith('GAM'):
run_baseline = True
elif not run_cdr and (m.startswith('CDR') or m.startswith('DTSR')):
run_cdr = True
if not (run_baseline or run_cdr):
argparser = argparse.ArgumentParser('''
Adds convolved columns to dataframe using pre-trained CDR model
''')
argparser.add_argument('config_paths', nargs='+', help='Path(s) to configuration (*.ini) file')
argparser.add_argument('-m', '--models', nargs='*', default=[], help='Path to configuration (*.ini) file')
argparser.add_argument('-d', '--data', nargs='+', default=[], help='Pairs of paths or buffers <predictors, responses>, allowing convolution of arbitrary data. Predictors may consist of ``;``-delimited paths designating files containing predictors with different timestamps. Within each file, timestamps are treated as identical across predictors.')
argparser.add_argument('-p', '--partition', nargs='+', default=['dev'], help='Name of partition to use ("train", "dev", "test", or hyphen-delimited subset of these). Ignored if **data** is provided.')
argparser.add_argument('-z', '--standardize_response', action='store_true', help='Standardize (Z-transform) response in plots. Ignored unless model was fitted using setting ``standardize_respose=True``.')
argparser.add_argument('-n', '--nsamples', type=int, default=None, help='Number of posterior samples to average (only used for CDRBayes)')
argparser.add_argument('-u', '--unscaled', action='store_true', help='Do not multiply outputs by CDR-fitted coefficients')
argparser.add_argument('-a', '--algorithm', type=str, default='MAP', help='Algorithm ("sampling" or "MAP") to use for extracting predictions.')
argparser.add_argument('-A', '--ablated_models', action='store_true', help='Perform convolution using ablated models. Otherwise only convolves using the full model in each ablation set.')
args, unknown = argparser.parse_known_args()
for path in args.config_paths:
p = Config(path)
if not p.use_gpu_if_available:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
models = filter_models(p.model_list, args.models)
cdr_formula_list = [Formula(p.models[m]['formula']) for m in models if (m.startswith('CDR') or m.startswith('DTSR'))]
cdr_models = [m for m in models if (m.startswith('CDR') or m.startswith('DTSR'))]
if not args.ablated_models:
cdr_models_new = []
for model_name in cdr_models:
if len(model_name.split('!')) == 1: #No ablated variables, which are flagged with "!"
cdr_models_new.append(model_name)
cdr_models = cdr_models_new
nargs='+',
default=[],
help=
'List of model names from which to extract betas. Regex permitted. If unspecified, extracts betas from all CDR models.'
)
argparser.add_argument(
'-n',
'--names',
nargs='*',
default=[],
help=
'Model names to print in table (must be omitted or same length as --models).'
)
args = argparser.parse_args()
p = Config(args.config)
if not p.use_gpu_if_available:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
models = filter_models(p.model_list, args.models, cdr_only=True)
if len(args.names) == 0:
names = p.model_list[:]
else:
assert len(args.names) == len(
models
), 'Length mismatch between number of models and number of model names'
names = args.names
beta_summaries = {}