type=int, default=2, help='Number of tails (1 or 2)') args, unknown = argparser.parse_known_args() assert args.metric in ['loss', 'loglik' ], 'Metric must be one of ["loss", "loglik"].' if args.pool: args.ablation = True ablations = None basenames_to_pool = None exps_outdirs = [] for path in args.config_paths: p = Config(path) models = filter_models(p.model_list, args.models) cdr_models = [ x for x in models if (x.startswith('CDR') or x.startswith('DTSR')) ] partitions = get_partition_list(args.partition) partition_str = '-'.join(partitions) if args.metric == 'loss': file_name = 'losses_mse_%s.txt' % partition_str else: file_name = 'loglik_%s.txt' % partition_str if args.twostep: file_name = 'LM_2STEP_' + file_name
'Type of significance test to use (one of ["pt", "2step"] for permutation testing or 2-step LME/LRT, respectively).' ) argparser.add_argument( '-p', '--partition', type=str, default='dev', help='Name of partition to use (one of "train", "dev", "test")') argparser.add_argument( '-H', '--human_readable', action='store_true', help='Return table in human readable format (otherwise return as CSV)') args, unknown = argparser.parse_known_args() p = Config(args.config_path) partitions = get_partition_list(args.partition) partition_str = '-'.join(partitions) suffix = '_2stepLRT_%s.txt' % partition_str if args.mode == '2step' else '_PT_%s.txt' % partition_str paths = [ p.outdir + '/' + x for x in os.listdir(p.outdir) if x.endswith(suffix) ] models, comparisons, comparisons_converged = extract_comparisons(paths) comparison_keys = sorted(list(comparisons.keys()), key=lambda x: len(x[0])) models = sorted(models) cols = ['model'] for c in comparison_keys:
'-e', '--force_training_evaluation', action='store_true', help= 'Recompute training evaluation even for models that are already finished.' ) argparser.add_argument( '-s', '--save_and_exit', action='store_true', help= 'Initialize, save, and exit (CDR only). Useful for bringing non-backward compatible trained models up to spec for plotting and evaluation.' ) args, unknown = argparser.parse_known_args() p = Config(args.config_path) if not p.use_gpu_if_available: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' models = filter_models(p.model_list, args.models) run_baseline = False run_cdr = False for m in models: if not run_baseline and m.startswith('LM') or m.startswith('GAM'): run_baseline = True elif not run_cdr and (m.startswith('CDR') or m.startswith('DTSR')): run_cdr = True if not (run_baseline or run_cdr):
argparser = argparse.ArgumentParser(''' Adds convolved columns to dataframe using pre-trained CDR model ''') argparser.add_argument('config_paths', nargs='+', help='Path(s) to configuration (*.ini) file') argparser.add_argument('-m', '--models', nargs='*', default=[], help='Path to configuration (*.ini) file') argparser.add_argument('-d', '--data', nargs='+', default=[], help='Pairs of paths or buffers <predictors, responses>, allowing convolution of arbitrary data. Predictors may consist of ``;``-delimited paths designating files containing predictors with different timestamps. Within each file, timestamps are treated as identical across predictors.') argparser.add_argument('-p', '--partition', nargs='+', default=['dev'], help='Name of partition to use ("train", "dev", "test", or hyphen-delimited subset of these). Ignored if **data** is provided.') argparser.add_argument('-z', '--standardize_response', action='store_true', help='Standardize (Z-transform) response in plots. Ignored unless model was fitted using setting ``standardize_respose=True``.') argparser.add_argument('-n', '--nsamples', type=int, default=None, help='Number of posterior samples to average (only used for CDRBayes)') argparser.add_argument('-u', '--unscaled', action='store_true', help='Do not multiply outputs by CDR-fitted coefficients') argparser.add_argument('-a', '--algorithm', type=str, default='MAP', help='Algorithm ("sampling" or "MAP") to use for extracting predictions.') argparser.add_argument('-A', '--ablated_models', action='store_true', help='Perform convolution using ablated models. Otherwise only convolves using the full model in each ablation set.') args, unknown = argparser.parse_known_args() for path in args.config_paths: p = Config(path) if not p.use_gpu_if_available: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' models = filter_models(p.model_list, args.models) cdr_formula_list = [Formula(p.models[m]['formula']) for m in models if (m.startswith('CDR') or m.startswith('DTSR'))] cdr_models = [m for m in models if (m.startswith('CDR') or m.startswith('DTSR'))] if not args.ablated_models: cdr_models_new = [] for model_name in cdr_models: if len(model_name.split('!')) == 1: #No ablated variables, which are flagged with "!" cdr_models_new.append(model_name) cdr_models = cdr_models_new
nargs='+', default=[], help= 'List of model names from which to extract betas. Regex permitted. If unspecified, extracts betas from all CDR models.' ) argparser.add_argument( '-n', '--names', nargs='*', default=[], help= 'Model names to print in table (must be omitted or same length as --models).' ) args = argparser.parse_args() p = Config(args.config) if not p.use_gpu_if_available: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' models = filter_models(p.model_list, args.models, cdr_only=True) if len(args.names) == 0: names = p.model_list[:] else: assert len(args.names) == len( models ), 'Length mismatch between number of models and number of model names' names = args.names beta_summaries = {}