def parse_group_option(options=None, groups=[]): "Given a list of Group objects, filter them according to options.group and options.exclude_group" def is_regex(expr=''): regex_common_operators = '.^$*+?' return any(o in expr for o in regex_common_operators) def is_list(expr=''): return ',' in expr if options.group: group_opt = options.group if is_regex(group_opt): groups = utils.filterWithRegexp(groups, group_opt, lambda g: g.name) elif is_list(group_opt): group_list = group_opt.strip('[]').split(',') groups = [g for g in groups if g.name in group_list] else: # assume this is a single group groups = [g for g in groups if g.name==group_opt] if options.exclude_group: group_opt = options.exclude_group if is_regex(group_opt): print 'excludeWithRegexp ',group_opt groups = utils.excludeWithRegexp(groups, group_opt, lambda g: g.name) elif is_list(group_opt): group_list = group_opt.strip('[]').split(',') groups = [g for g in groups if g.name not in group_list] else: # assume this is a single group groups = [g for g in groups if g.name!=group_opt] if options.verbose: print "parse_group_option: selected ",list(g.name for g in groups) return groups
def parse_group_option(options=None, groups=[]): "Given a list of Group objects, filter them according to options.group and options.exclude_group" def is_regex(expr=''): regex_common_operators = '.^$*+?' return any(o in expr for o in regex_common_operators) def is_list(expr=''): return ',' in expr if options.group: group_opt = options.group if is_regex(group_opt): groups = utils.filterWithRegexp(groups, group_opt, lambda g: g.name) elif is_list(group_opt): group_list = group_opt.strip('[]').split(',') groups = [g for g in groups if g.name in group_list] else: # assume this is a single group groups = [g for g in groups if g.name == group_opt] if options.exclude_group: group_opt = options.exclude_group if is_regex(group_opt): print 'excludeWithRegexp ', group_opt groups = utils.excludeWithRegexp(groups, group_opt, lambda g: g.name) elif is_list(group_opt): group_list = group_opt.strip('[]').split(',') groups = [g for g in groups if g.name not in group_list] else: # assume this is a single group groups = [g for g in groups if g.name != group_opt] if options.verbose: print "parse_group_option: selected ", list(g.name for g in groups) return groups
def main(): options = parse_options() exe = options.executable inputdf = options.input include = options.include_regexp exclude = options.exclude_regexp tag = options.tag verbose = options.verbose submit = options.submit datasets = dataset.build_all_datasets_from_dir_or_file(inputdf) datasets = utils.filterWithRegexp( datasets, include, lambda _: _.name) if include else datasets datasets = utils.excludeWithRegexp( datasets, exclude, lambda _: _.name) if exclude else datasets for dset in datasets: if not get_filelist(dset.name): print "# skipping (missing filelist) {0}".format(dset.name) continue script = get_batch_script(dset, options) if not script: if verbose: print "skipping (do-not-overwrite) {0}".format(dset.name) continue cmd = "sbatch %s" % script if verbose: print cmd if submit: out = utils.getCommandOutput(cmd) if verbose: print out['stdout'] if not submit: print "This was a dry run; use '--submit' to actually submit the jobs"
def regions_to_plot(include='.*', exclude=None, regions=None): selected_regions = selection_formulas().keys() if regions: selected_regions = [r for r in selected_regions if r in regions.split(',')] selected_regions = utils.filterWithRegexp(selected_regions, include) selected_regions = utils.excludeWithRegexp(selected_regions, exclude) if exclude else selected_regions return selected_regions
def main(): options = parse_options() exe = options.executable inputdf = options.input include = options.include_regexp exclude = options.exclude_regexp tag = options.tag verbose = options.verbose submit = options.submit datasets = dataset.build_all_datasets_from_dir_or_file(inputdf) datasets = utils.filterWithRegexp (datasets, include, lambda _: _.name) if include else datasets datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets for dset in datasets : if not get_filelist(dset.name): print "# skipping (missing filelist) {0}".format(dset.name) continue script = get_batch_script(dset, options) if not script: if verbose : print "skipping (do-not-overwrite) {0}".format(dset.name) continue cmd = "sbatch %s" % script if verbose : print cmd if submit : out = utils.getCommandOutput(cmd) if verbose : print out['stdout'] if not submit : print "This was a dry run; use '--submit' to actually submit the jobs"
def main(): options = parse_options() inputdf = options.input outdir = options.output_dir regexp = options.sample_regexp exclude = options.exclude_regexp tag = options.tag verbose = options.verbose debug = options.debug utils.mkdirIfNeeded(outdir) if debug: dataset.Dataset.verbose_parsing = True datasets = dataset.build_all_datasets_from_dir_or_file(inputdf) datasets = utils.filterWithRegexp(datasets, regexp, lambda _: _.name) if regexp else datasets datasets = utils.excludeWithRegexp( datasets, exclude, lambda _: _.name) if exclude else datasets counter = {'fail': 0, 'pass': 0} for d in datasets: outcome = 'pass' if d.build_filelist(gpatlas_dir(d, tag), outdir, verbose) else 'fail' counter[outcome] += 1 if verbose: print "created %d filelists (%d failures)" % (counter['pass'], counter['fail'])
def regions_to_plot(include='.*', exclude=None, regions=None): selected_regions = selection_formulas().keys() if regions: selected_regions = [ r for r in selected_regions if r in regions.split(',') ] selected_regions = utils.filterWithRegexp(selected_regions, include) selected_regions = utils.excludeWithRegexp( selected_regions, exclude) if exclude else selected_regions return selected_regions
def regions_to_plot(include='.*', exclude=None, regions=''): "include and exclude are regexp; regions is a string with either one region or a comma-separated list of regions" # return ['vr_emu_mue_ss'] # test to debug fake # return ['vr_emu_ss_razor'] # return [k for k in selection_formulas().keys() if ('vr' in k and 'ss' in k)] # test to debug fake # return [k for k in selection_formulas().keys() if 'vr' not in k] # tmp until I have vrs # return [k for k in selection_formulas().keys() if 'sr' in k] # tmp dbg # used to be the default: # ['sr_emu_os', 'sr_mue_os', 'vr_emu_os', 'vr_mue_os', # 'sr_emu_ss', 'sr_mue_ss', 'vr_emu_ss', 'vr_mue_ss', # 'ext_mumu_ss', 'ext_emu_mue_ss', 'ext_emu_pt0_40_ss', 'ext_mue_pt0_40_ss', 'ext_mumu_pt0_40_ss', # 'sr_mue_os_low_pt1_15' # ] selected_regions = selection_formulas().keys() if regions: regions = regions if type(regions)==list else regions.split(',') if ',' in regions else [regions] # if it's a comma-sep string, convert it to list selected_regions = [r for r in selected_regions if r in regions] selected_regions = utils.filterWithRegexp(selected_regions, include) selected_regions = utils.excludeWithRegexp(selected_regions, exclude) if exclude else selected_regions return selected_regions
def main(): options = parse_options() inputdf = options.input outdir = options.output_dir regexp = options.sample_regexp exclude = options.exclude_regexp tag = options.tag verbose = options.verbose debug = options.debug utils.mkdirIfNeeded(outdir) if debug : dataset.Dataset.verbose_parsing = True datasets = dataset.build_all_datasets_from_dir_or_file(inputdf) datasets = utils.filterWithRegexp (datasets, regexp, lambda _: _.name) if regexp else datasets datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets counter = {'fail':0, 'pass':0} for d in datasets: outcome = 'pass' if d.build_filelist(gpatlas_dir(d, tag), outdir, verbose) else 'fail' counter[outcome] += 1 if verbose: print "created %d filelists (%d failures)" % (counter['pass'], counter['fail'])
def regions_to_plot(include='.*', exclude=None, regions=''): "include and exclude are regexp; regions is a string with either one region or a comma-separated list of regions" # return ['vr_emu_mue_ss'] # test to debug fake # return ['vr_emu_ss_razor'] # return [k for k in selection_formulas().keys() if ('vr' in k and 'ss' in k)] # test to debug fake # return [k for k in selection_formulas().keys() if 'vr' not in k] # tmp until I have vrs # return [k for k in selection_formulas().keys() if 'sr' in k] # tmp dbg # used to be the default: # ['sr_emu_os', 'sr_mue_os', 'vr_emu_os', 'vr_mue_os', # 'sr_emu_ss', 'sr_mue_ss', 'vr_emu_ss', 'vr_mue_ss', # 'ext_mumu_ss', 'ext_emu_mue_ss', 'ext_emu_pt0_40_ss', 'ext_mue_pt0_40_ss', 'ext_mumu_pt0_40_ss', # 'sr_mue_os_low_pt1_15' # ] selected_regions = selection_formulas().keys() if regions: regions = regions if type( regions) == list else regions.split(',') if ',' in regions else [ regions ] # if it's a comma-sep string, convert it to list selected_regions = [r for r in selected_regions if r in regions] selected_regions = utils.filterWithRegexp(selected_regions, include) selected_regions = utils.excludeWithRegexp( selected_regions, exclude) if exclude else selected_regions return selected_regions