def create_metadata(kwargs=None):
"""fits_table with configuration-like params for the simulated sources
TODO: Should metacat table have a rowstart column?
TODO: One metacat table per brick, instead of one per `rs*` directory?
Args:
kwargs: configuration-like params for the simulated sources
{'brickname': which chunk of sky
'objtype': star,elg,lrg,qso
'nobj': number of simulated sources for this run
}
Returns:
Nothing
writes the 'metacat' fits_table to disk and stores it
in the kwargs input arg
"""
assert (kwargs is not None)
log = logging.getLogger('decals_sim')
# Pack the input parameters into a meta-data table and write out.
#metacols = [
# ('BRICKNAME', 'S10'),
# ('OBJTYPE', 'S10'),
# ('NOBJ', 'i4'),
# ('CHUNKSIZE', 'i2'),
# ('NCHUNK', 'i2'),
# ('ZOOM', 'i4', (4,)),
# ('SEED', 'S20'),
# ('RMAG_RANGE', 'f4', (2,))]
#metacat = Table(np.zeros(1, dtype=metacols))
metacat = fits_table()
for key in ['brickname', 'objtype']: #,'nchunk']:
metacat.set(key, np.array([kwargs[key]]))
metacat.set('nobj', np.array([kwargs['args'].nobj]))
metacat.set('zoom', np.array([kwargs['args'].zoom]))
#metacat['RMAG_RANGE'] = kwargs['args'].rmag_range
#if not kwargs['args'].seed:
# log.info('Random seed = {}'.format(kwargs['args'].seed))
# metacat['SEED'] = kwargs['args'].seed
#metacat_dir = os.path.join(kwargs['decals_sim_dir'], kwargs['objtype'],kwargs['brickname'][:3],kwargs['brickname'])
metacat_dir = get_outdir_runbrick(kwargs['decals_sim_dir'],
kwargs['brickname'],
kwargs['rowst'],
do_skipids=kwargs['do_skipids'],
do_more=kwargs['do_more'])
if not os.path.exists(metacat_dir):
os.makedirs(metacat_dir)
metafile = os.path.join(metacat_dir, 'metacat' + get_fnsuffix(**kwargs))
log.info('Writing {}'.format(metafile))
if os.path.isfile(metafile):
os.remove(metafile)
metacat.writeto(metafile)
# Store new stuff
kwargs['metacat'] = metacat
kwargs['metacat_dir'] = metacat_dir
def create_ith_simcat(d=None):
"""Write 'simcat' and 'skipped_ids' tables for a given sample of sources
Args:
d: {'Samp': fits_table for the properties of sources in the brick
'brickwcs': WCS object for the brick
'metacat': fits_table with configuration params for the simulated sources
}
Returns:
Nothing, saves the 'simcat' and 'skipped_ids' tables
Adds 'simcat' table to dict 'd'
"""
assert(d is not None)
log = logging.getLogger('decals_sim')
#chunksuffix = '{:02d}'.format(ith_chunk)
# Build and write out the simulated object catalog.
#seed= d['seeds'][ith_chunk]
#simcat = build_simcat(d['nobj'], d['brickname'], d['brickwcs'], d['metacat'], seed)
simcat, skipped_ids = build_simcat(Samp=d['Samp'],brickwcs=d['brickwcs'],meta=d['metacat'])
# Simcat
simcat_dir = get_outdir_runbrick(d['decals_sim_dir'],
d['brickname'],d['rowst'],
do_skipids=d['do_skipids'],do_more=d['do_more'])
if not os.path.exists(simcat_dir):
os.makedirs(simcat_dir)
#simcatfile = os.path.join(simcat_dir, 'simcat-{}-{}-row{}-{}.fits'.format(d['brickname'], d['objtype'],rowstart,rowend)) # chunksuffix))
simcatfile = os.path.join(simcat_dir, 'simcat'+get_fnsuffix(**d))
if os.path.isfile(simcatfile):
os.remove(simcatfile)
simcat.writeto(simcatfile)
log.info('Wrote {}'.format(simcatfile))
# Skipped Ids
if len(skipped_ids) > 0:
skip_table= fits_table()
skip_table.set('ids',skipped_ids)
name= os.path.join(simcat_dir,'skippedids'+get_fnsuffix(**d))
if os.path.exists(name):
os.remove(name)
log.info('Removed %s' % name)
skip_table.writeto(name)
log.info('HUI-TEST:::stuck here?')
log.info('Wrote {}'.format(name))
# add to dict
log.info('HUI-TEST:::or stuck here?')
print('add to dict')
d['simcat']= simcat
d['simcat_dir']= simcat_dir
log.info('HUI-TEST:::or exiting?')
def main(args=None):
"""Main routine which parses the optional inputs."""
t0= Time()
# Command line options
if args is None:
# Read from cmd line
parser= get_parser()
args = parser.parse_args(args=args)
else:
# args is already a argparse.Namespace obj
pass
# Print calling sequence
print('Args:', args)
if args.do_more == 'yes':
assert(not args.minid is None)
# Setup loggers
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s')
log = logging.getLogger('decals_sim')
# Sort through args
#log.info('decals_sim.py args={}'.format(args))
#max_nobj=500
#max_nchunk=1000
#if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1)
#assert(args.nchunk <= max_nchunk)
#assert(args.nobj <= max_nobj)
#if args.ith_chunk is not None:
# assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Exit if expected output already exists
rsdir= get_outdir_runbrick(args.outdir,
args.brick,args.rowstart,
do_skipids=args.do_skipids,
do_more=args.do_more)
rsdir= os.path.basename(rsdir)
tractor_fn= os.path.join(args.outdir,
'tractor',args.brick[:3],args.brick,
rsdir,
'tractor-%s.fits' % args.brick)
if (os.path.exists(tractor_fn) &
(not args.overwrite_if_exists)):
print('Exiting, already finished %s' % tractor_fn)
return 0 #sys.exit(0)
brickname = args.brick
objtype = args.objtype
# Output dir
decals_sim_dir = args.outdir
#nchunk = args.nchunk
#rand = np.random.RandomState(args.seed) # determines seed for all chunks
#seeds = rand.random_integers(0,2**18, max_nchunk)
log.info('Object type = {}'.format(objtype))
#log.info('Number of objects = {}'.format(nobj))
#log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
survey = LegacySurveyData(survey_dir=args.survey_dir)
brickinfo= get_brickinfo_hack(survey,brickname)
#brickinfo = survey.get_brick_by_name(brickname)
#print(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in
[(1,1), (W,1), (W,H), (1,H), (1,1)]])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
t0= ptime('First part of Main()',t0)
# SAMPLE table
sample_kwargs= {"objtype":args.objtype,
"brick":args.brick,
"outdir":args.outdir,
"randoms_db":args.randoms_db,
"minid":args.minid,
"do_skipids":args.do_skipids,
"randoms_from_fits":args.randoms_from_fits,
"dont_sort_sampleid":args.dont_sort_sampleid}
Samp,seed= get_sample(**sample_kwargs)
Samp= Samp[args.rowstart:args.rowstart + args.nobj]
# Performance
#if objtype in ['elg','lrg']:
# Samp=Samp[np.argsort( Samp.get('%s_n' % objtype) )]
print('Max sample size=%d, actual sample size=%d' % (args.nobj,len(Samp)))
assert(len(Samp) <= args.nobj)
t0= ptime('Got randoms sample',t0)
# Store args in dict for easy func passing
kwargs=dict(Samp=Samp,\
brickname=brickname, \
checkpoint=args.checkpoint, \
seed= seed,
decals_sim_dir= decals_sim_dir,\
brickwcs= brickwcs, \
objtype=objtype,\
nobj=len(Samp),\
maxobjs=args.nobj,\
rowst=args.rowstart,\
do_skipids=args.do_skipids,\
do_more=args.do_more,\
minid=args.minid,\
survey_dir=args.survey_dir,\
args=args)
# Stop if starting row exceeds length of radec,color table
if len(Samp) == 0:
fn= get_outdir_runbrick(kwargs['decals_sim_dir'],
kwargs['brickname'],kwargs['rowst'],
do_skipids=kwargs['do_skipids'],do_more=kwargs['do_more'])
fn+= '_exceeded.txt'
junk= os.system('touch %s' % fn)
print('Wrote %s' % fn)
#we want not to add any sample -- obiwan
#raise ValueError('starting row=%d exceeds number of artificial sources, quit' % args.rowstart)
# Create simulated catalogues and run Tractor
create_metadata(kwargs=kwargs)
t0= ptime('create_metadata',t0)
# do chunks
#for ith_chunk in chunk_list:
#log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1))
# Random ra,dec and source properties
create_ith_simcat(d=kwargs)
#log.info('HUI-TEST:::out of create_ith_simcat')
t0= ptime('create_ith_simcat',t0)
# Run tractor
#log.info('HUI-TEST:::running tractor')
do_one_chunk(d=kwargs)
#log.info('HUI-TEST::: checkpoint3i')
t0= ptime('do_one_chunk',t0)
# Clean up output
if args.no_cleanup == False:
do_ith_cleanup(d=kwargs)
#log.info('HUI-TEST::: checkpoint3j')
t0= ptime('do_ith_cleanup',t0)
log.info('All done!')
#log.info('HUI-TEST::: checkpoint3k')
return 0