def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
NAME_FACTORY.update_base_dict(args['data'])
inputfiles = create_inputlist(args['ft1file'])
outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')
nfiles = len(inputfiles)
for idx, infile in enumerate(inputfiles):
key = "%06i" % idx
output_dir = os.path.join(outdir_base, key)
try:
os.mkdir(output_dir)
except OSError:
pass
logfile = make_nfs_path(os.path.join(output_dir, 'scatter_%s.log' % key))
job_configs[key] = dict(ft1file=infile,
comp=args['comp'],
hpx_order_max=args['hpx_order_max'],
outdir=outdir_base,
outkey=key,
logfile=logfile,
pfiles=output_dir)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
models = load_yaml(args['models'])
for modelkey, modelpath in models.items():
manifest = os.path.join('analysis', 'model_%s' % modelkey,
'srcmap_manifest_%s.yaml' % modelkey)
for comp in components:
key = comp.make_key('{ebin_name}_{evtype_name}')
fullkey = "%s_%s" % (modelkey, key)
outfile = NAME_FACTORY.merged_srcmaps(
modelkey=modelkey,
component=key,
coordsys=comp.coordsys,
mktime='none',
irf_ver=NAME_FACTORY.irf_ver())
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
job_configs[fullkey] = dict(input=manifest,
compname=key,
logfile=logfile)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
datafile = args['data']
if datafile is None or datafile == 'None':
return job_configs
NAME_FACTORY.update_base_dict(args['data'])
outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'],
'counts_cubes')
inputfiles = create_inputlist(args['ft1file'])
num_files = len(inputfiles)
for comp in components:
zcut = "zmax%i" % comp.zmax
mktimelist = copy.copy(comp.mktimefilters)
if len(mktimelist) == 0:
mktimelist.append('none')
evtclasslist_keys = copy.copy(comp.evtclasses)
if len(evtclasslist_keys) == 0:
evtclasslist_keys.append('default')
evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]
else:
evtclasslist_vals = copy.copy(evtclasslist_keys)
for mktimekey in mktimelist:
for evtclasskey, evtclassval in zip(evtclasslist_keys,
evtclasslist_vals):
fullkey = comp.make_key(
'%s_%s_{ebin_name}_%s_{evtype_name}' %
(evtclassval, zcut, mktimekey))
name_keys = dict(zcut=zcut,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
irf_ver=NAME_FACTORY.irf_ver(),
mktime=mktimekey,
evclass=evtclassval,
fullpath=True)
ccube_name = os.path.basename(
NAME_FACTORY.ccube(**name_keys))
outfile = os.path.join(outdir_base, ccube_name)
infiles = make_input_file_list(outfile, num_files)
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
job_configs[fullkey] = dict(args=infiles,
output=outfile,
logfile=logfile)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
datafile = args['data']
if datafile is None or datafile == 'None':
return job_configs
NAME_FACTORY.update_base_dict(args['data'])
for comp in components:
zcut = "zmax%i" % comp.zmax
mktimelist = copy.copy(comp.mktimefilters)
if len(mktimelist) == 0:
mktimelist.append('none')
evtclasslist_keys = copy.copy(comp.evtclasses)
if len(evtclasslist_keys) == 0:
evtclasslist_keys.append('default')
evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]
else:
evtclasslist_vals = copy.copy(evtclasslist_keys)
for mktimekey in mktimelist:
for evtclasskey, evtclassval in zip(evtclasslist_keys,
evtclasslist_vals):
fullkey = comp.make_key(
'%s_%s_{ebin_name}_%s_{evtype_name}' %
(evtclassval, zcut, mktimekey))
name_keys = dict(zcut=zcut,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
irf_ver=NAME_FACTORY.irf_ver(),
mktime=mktimekey,
evclass=evtclassval,
fullpath=True)
outfile = NAME_FACTORY.bexpcube(**name_keys)
cmap = NAME_FACTORY.ccube(**name_keys)
infile = NAME_FACTORY.ltcube(**name_keys)
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
job_configs[fullkey] = dict(
cmap=cmap,
infile=infile,
outfile=outfile,
irfs=NAME_FACTORY.irfs(**name_keys),
hpx_order=min(comp.hpx_order, args['hpx_order_max']),
evtype=comp.evtype,
logfile=logfile)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
ret_dict = make_catalog_comp_dict(sources=args['library'],
basedir=NAME_FACTORY.base_dict['basedir'])
catalog_info_dict = ret_dict['catalog_info_dict']
comp_info_dict = ret_dict['comp_info_dict']
n_src_per_job = args['nsrc']
if args['make_xml']:
ConfigMaker_SrcmapsCatalog._make_xml_files(catalog_info_dict, comp_info_dict)
for catalog_name, catalog_info in catalog_info_dict.items():
n_cat_src = len(catalog_info.catalog.table)
n_job = int(math.ceil(float(n_cat_src)/n_src_per_job))
for comp in components:
zcut = "zmax%i" % comp.zmax
key = comp.make_key('{ebin_name}_{evtype_name}')
name_keys = dict(zcut=zcut,
sourcekey=catalog_name,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
irf_ver=NAME_FACTORY.irf_ver(),
mktime='none',
fullpath=True)
for i_job in range(n_job):
full_key = "%s_%02i"%(key, i_job)
srcmin = i_job*n_src_per_job
srcmax = min(srcmin+n_src_per_job, n_cat_src)
outfile = NAME_FACTORY.srcmaps(**name_keys).replace('.fits', "_%02i.fits"%(i_job))
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
job_configs[full_key] = dict(cmap=NAME_FACTORY.ccube(**name_keys),
expcube=NAME_FACTORY.ltcube(**name_keys),
irfs=NAME_FACTORY.irfs(**name_keys),
bexpmap=NAME_FACTORY.bexpcube(**name_keys),
outfile=outfile,
logfile=logfile,
srcmdl=catalog_info.srcmdl_name,
evtype=comp.evtype,
srcmin=srcmin,
srcmax=srcmax)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
ret_dict = make_catalog_comp_dict(sources=args['library'], basedir='.')
comp_info_dict = ret_dict['comp_info_dict']
for split_ver, split_dict in comp_info_dict.items():
for source_key, source_dict in split_dict.items():
full_key = "%s_%s" % (split_ver, source_key)
merged_name = "%s_%s" % (source_dict.catalog_info.catalog_name,
source_key)
if source_dict.model_type != 'CompositeSource':
continue
for comp in components:
zcut = "zmax%i" % comp.zmax
key = "%s_%s" % (
full_key, comp.make_key('{ebin_name}_{evtype_name}'))
name_keys = dict(zcut=zcut,
sourcekey=full_key,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
mktime='none',
irf_ver=NAME_FACTORY.irf_ver())
nested_name_keys = dict(
zcut=zcut,
sourcekey=source_dict.catalog_info.catalog_name,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
mktime='none',
irf_ver=NAME_FACTORY.irf_ver())
outfile = NAME_FACTORY.srcmaps(**name_keys)
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
print(key, merged_name, logfile)
job_configs[key] = dict(
srcmaps=NAME_FACTORY.srcmaps(**nested_name_keys),
expcube=NAME_FACTORY.ltcube(**name_keys),
irfs=NAME_FACTORY.irfs(**name_keys),
bexpmap=NAME_FACTORY.bexpcube(**name_keys),
srcmdl=NAME_FACTORY.srcmdl_xml(**name_keys),
merged=merged_name,
outfile=outfile,
outxml=NAME_FACTORY.nested_srcmdl_xml(**name_keys),
logfile=logfile)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
ret_dict = make_diffuse_comp_info_dict(
components=components,
library=args['library'],
basedir=NAME_FACTORY.base_dict['basedir'])
diffuse_comp_info_dict = ret_dict['comp_info_dict']
for diffuse_comp_info_key in sorted(diffuse_comp_info_dict.keys()):
diffuse_comp_info_value = diffuse_comp_info_dict[
diffuse_comp_info_key]
for comp in components:
zcut = "zmax%i" % comp.zmax
key = comp.make_key('{ebin_name}_{evtype_name}')
if diffuse_comp_info_value.components is None:
sub_comp_info = diffuse_comp_info_value
else:
sub_comp_info = diffuse_comp_info_value.get_component_info(
comp)
full_key = "%s_%s" % (sub_comp_info.sourcekey, key)
name_keys = dict(zcut=zcut,
sourcekey=sub_comp_info.sourcekey,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
irf_ver=NAME_FACTORY.irf_ver(),
mktime='none',
fullpath=True)
infile = NAME_FACTORY.srcmaps(**name_keys)
outfile = infile.replace('.fits', '.png')
logfile = make_nfs_path(outfile.replace('.png', '_png.log'))
job_configs[full_key] = dict(
input=infile,
output=outfile,
extension=sub_comp_info.source_name,
zscale=args.get('zscale', 'log'),
logfile=logfile)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
gmm = make_ring_dicts(library=args['library'], basedir='.')
for galkey in gmm.galkeys():
ring_dict = gmm.ring_dict(galkey)
for ring_key, ring_info in ring_dict.items():
output_file = ring_info.merged_gasmap
file_string = ""
for fname in ring_info.files:
file_string += " %s" % fname
logfile = make_nfs_path(output_file.replace('.fits', '.log'))
job_configs[ring_key] = dict(output=output_file,
args=file_string,
logfile=logfile)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
ret_dict = make_catalog_comp_dict(
library=args['library'], basedir=NAME_FACTORY.base_dict['basedir'])
catalog_info_dict = ret_dict['catalog_info_dict']
comp_info_dict = ret_dict['comp_info_dict']
for catalog_name, catalog_info in catalog_info_dict.items():
for comp in components:
zcut = "zmax%i" % comp.zmax
key = comp.make_key('{ebin_name}_{evtype_name}')
name_keys = dict(zcut=zcut,
sourcekey=catalog_name,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
coordsys=comp.coordsys,
irf_ver=NAME_FACTORY.irf_ver(),
mktime='none',
fullpath=True)
outfile = NAME_FACTORY.srcmaps(**name_keys)
outfile_tokens = os.path.splitext(outfile)
infile_regexp = "%s_*.fits" % outfile_tokens[0]
logfile = make_nfs_path(outfile.replace('.fits', '.log'))
job_configs[key] = dict(output=outfile,
args=infile_regexp,
logfile=logfile)
return job_configs
def build_job_configs(self, args):
"""Hook to build job configurations
"""
job_configs = {}
components = Component.build_from_yamlfile(args['comp'])
NAME_FACTORY.update_base_dict(args['data'])
ret_dict = make_diffuse_comp_info_dict(components=components,
library=args['library'],
basedir='.')
diffuse_comp_info_dict = ret_dict['comp_info_dict']
if args['make_xml']:
ConfigMaker_SrcmapPartial._make_xml_files(diffuse_comp_info_dict)
for diffuse_comp_info_key in sorted(diffuse_comp_info_dict.keys()):
diffuse_comp_info_value = diffuse_comp_info_dict[diffuse_comp_info_key]
no_psf = diffuse_comp_info_value.no_psf
for comp in components:
zcut = "zmax%i" % comp.zmax
key = comp.make_key('{ebin_name}_{evtype_name}')
if diffuse_comp_info_value.components is None:
sub_comp_info = diffuse_comp_info_value
else:
sub_comp_info = diffuse_comp_info_value.get_component_info(comp)
name_keys = dict(zcut=zcut,
sourcekey=sub_comp_info.sourcekey,
ebin=comp.ebin_name,
psftype=comp.evtype_name,
mktime='none',
coordsys=comp.coordsys,
irf_ver=NAME_FACTORY.irf_ver(),
fullpath=True)
kmin = 0
kmax = comp.enumbins + 1
outfile_base = NAME_FACTORY.srcmaps(**name_keys)
kstep = HPX_ORDER_TO_KSTEP[comp.hpx_order]
base_dict = dict(cmap=NAME_FACTORY.ccube(**name_keys),
expcube=NAME_FACTORY.ltcube(**name_keys),
irfs=NAME_FACTORY.irfs(**name_keys),
bexpmap=NAME_FACTORY.bexpcube(**name_keys),
srcmdl=sub_comp_info.srcmdl_name,
source=sub_comp_info.source_name,
no_psf=no_psf,
evtype=comp.evtype)
if kstep < 0:
kstep = kmax
else:
pass
for k in range(kmin, kmax, kstep):
full_key = "%s_%s_%02i" % (diffuse_comp_info_key, key, k)
khi = min(kmax, k + kstep)
full_dict = base_dict.copy()
outfile = outfile_base.replace('.fits', '_%02i.fits' % k)
logfile = make_nfs_path(outfile_base.replace('.fits', '_%02i.log' % k))
full_dict.update(dict(outfile=outfile,
kmin=k, kmax=khi,
logfile=logfile))
job_configs[full_key] = full_dict
return job_configs