def parse_group_option(options=None, groups=[]):
"Given a list of Group objects, filter them according to options.group and options.exclude_group"
def is_regex(expr=''):
regex_common_operators = '.^$*+?'
return any(o in expr for o in regex_common_operators)
def is_list(expr=''):
return ',' in expr
if options.group:
group_opt = options.group
if is_regex(group_opt):
groups = utils.filterWithRegexp(groups, group_opt, lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name==group_opt]
if options.exclude_group:
group_opt = options.exclude_group
if is_regex(group_opt):
print 'excludeWithRegexp ',group_opt
groups = utils.excludeWithRegexp(groups, group_opt, lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name not in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name!=group_opt]
if options.verbose:
print "parse_group_option: selected ",list(g.name for g in groups)
return groups
def parse_group_option(options=None, groups=[]):
"Given a list of Group objects, filter them according to options.group and options.exclude_group"
def is_regex(expr=''):
regex_common_operators = '.^$*+?'
return any(o in expr for o in regex_common_operators)
def is_list(expr=''):
return ',' in expr
if options.group:
group_opt = options.group
if is_regex(group_opt):
groups = utils.filterWithRegexp(groups, group_opt,
lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name == group_opt]
if options.exclude_group:
group_opt = options.exclude_group
if is_regex(group_opt):
print 'excludeWithRegexp ', group_opt
groups = utils.excludeWithRegexp(groups, group_opt,
lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name not in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name != group_opt]
if options.verbose:
print "parse_group_option: selected ", list(g.name for g in groups)
return groups
def main():
options = parse_options()
exe = options.executable
inputdf = options.input
include = options.include_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
submit = options.submit
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp(
datasets, include, lambda _: _.name) if include else datasets
datasets = utils.excludeWithRegexp(
datasets, exclude, lambda _: _.name) if exclude else datasets
for dset in datasets:
if not get_filelist(dset.name):
print "# skipping (missing filelist) {0}".format(dset.name)
continue
script = get_batch_script(dset, options)
if not script:
if verbose:
print "skipping (do-not-overwrite) {0}".format(dset.name)
continue
cmd = "sbatch %s" % script
if verbose: print cmd
if submit:
out = utils.getCommandOutput(cmd)
if verbose: print out['stdout']
if not submit:
print "This was a dry run; use '--submit' to actually submit the jobs"
def regions_to_plot(include='.*', exclude=None, regions=None):
selected_regions = selection_formulas().keys()
if regions:
selected_regions = [r for r in selected_regions if r in regions.split(',')]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(selected_regions, exclude) if exclude else selected_regions
return selected_regions
def main():
options = parse_options()
exe = options.executable
inputdf = options.input
include = options.include_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
submit = options.submit
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp (datasets, include, lambda _: _.name) if include else datasets
datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets
for dset in datasets :
if not get_filelist(dset.name):
print "# skipping (missing filelist) {0}".format(dset.name)
continue
script = get_batch_script(dset, options)
if not script:
if verbose : print "skipping (do-not-overwrite) {0}".format(dset.name)
continue
cmd = "sbatch %s" % script
if verbose : print cmd
if submit :
out = utils.getCommandOutput(cmd)
if verbose : print out['stdout']
if not submit : print "This was a dry run; use '--submit' to actually submit the jobs"
def main():
options = parse_options()
inputdf = options.input
outdir = options.output_dir
regexp = options.sample_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
debug = options.debug
utils.mkdirIfNeeded(outdir)
if debug: dataset.Dataset.verbose_parsing = True
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp(datasets, regexp,
lambda _: _.name) if regexp else datasets
datasets = utils.excludeWithRegexp(
datasets, exclude, lambda _: _.name) if exclude else datasets
counter = {'fail': 0, 'pass': 0}
for d in datasets:
outcome = 'pass' if d.build_filelist(gpatlas_dir(d, tag), outdir,
verbose) else 'fail'
counter[outcome] += 1
if verbose:
print "created %d filelists (%d failures)" % (counter['pass'],
counter['fail'])
def regions_to_plot(include='.*', exclude=None, regions=None):
selected_regions = selection_formulas().keys()
if regions:
selected_regions = [
r for r in selected_regions if r in regions.split(',')
]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(
selected_regions, exclude) if exclude else selected_regions
return selected_regions
def regions_to_plot(include='.*', exclude=None, regions=''):
"include and exclude are regexp; regions is a string with either one region or a comma-separated list of regions"
# return ['vr_emu_mue_ss'] # test to debug fake
# return ['vr_emu_ss_razor']
# return [k for k in selection_formulas().keys() if ('vr' in k and 'ss' in k)] # test to debug fake
# return [k for k in selection_formulas().keys() if 'vr' not in k] # tmp until I have vrs
# return [k for k in selection_formulas().keys() if 'sr' in k] # tmp dbg
# used to be the default:
# ['sr_emu_os', 'sr_mue_os', 'vr_emu_os', 'vr_mue_os',
# 'sr_emu_ss', 'sr_mue_ss', 'vr_emu_ss', 'vr_mue_ss',
# 'ext_mumu_ss', 'ext_emu_mue_ss', 'ext_emu_pt0_40_ss', 'ext_mue_pt0_40_ss', 'ext_mumu_pt0_40_ss',
# 'sr_mue_os_low_pt1_15'
# ]
selected_regions = selection_formulas().keys()
if regions:
regions = regions if type(regions)==list else regions.split(',') if ',' in regions else [regions] # if it's a comma-sep string, convert it to list
selected_regions = [r for r in selected_regions if r in regions]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(selected_regions, exclude) if exclude else selected_regions
return selected_regions
def main():
options = parse_options()
inputdf = options.input
outdir = options.output_dir
regexp = options.sample_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
debug = options.debug
utils.mkdirIfNeeded(outdir)
if debug : dataset.Dataset.verbose_parsing = True
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp (datasets, regexp, lambda _: _.name) if regexp else datasets
datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets
counter = {'fail':0, 'pass':0}
for d in datasets:
outcome = 'pass' if d.build_filelist(gpatlas_dir(d, tag), outdir, verbose) else 'fail'
counter[outcome] += 1
if verbose:
print "created %d filelists (%d failures)" % (counter['pass'], counter['fail'])
def regions_to_plot(include='.*', exclude=None, regions=''):
"include and exclude are regexp; regions is a string with either one region or a comma-separated list of regions"
# return ['vr_emu_mue_ss'] # test to debug fake
# return ['vr_emu_ss_razor']
# return [k for k in selection_formulas().keys() if ('vr' in k and 'ss' in k)] # test to debug fake
# return [k for k in selection_formulas().keys() if 'vr' not in k] # tmp until I have vrs
# return [k for k in selection_formulas().keys() if 'sr' in k] # tmp dbg
# used to be the default:
# ['sr_emu_os', 'sr_mue_os', 'vr_emu_os', 'vr_mue_os',
# 'sr_emu_ss', 'sr_mue_ss', 'vr_emu_ss', 'vr_mue_ss',
# 'ext_mumu_ss', 'ext_emu_mue_ss', 'ext_emu_pt0_40_ss', 'ext_mue_pt0_40_ss', 'ext_mumu_pt0_40_ss',
# 'sr_mue_os_low_pt1_15'
# ]
selected_regions = selection_formulas().keys()
if regions:
regions = regions if type(
regions) == list else regions.split(',') if ',' in regions else [
regions
] # if it's a comma-sep string, convert it to list
selected_regions = [r for r in selected_regions if r in regions]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(
selected_regions, exclude) if exclude else selected_regions
return selected_regions
