def auto_extract(root='j023507-040202', maglim=[16.5, 26]): from grizli import utils from grizli.pipeline import auto_script utils.set_warnings() tab = utils.GTable.gread('{0}_footprint.fits'.format(root)) pline = DITHERED_PLINE = { 'kernel': 'point', 'pixfrac': 0.2, 'pixscale': 0.1, 'size': 8, 'wcs': None } args = np.load('fit_args.npy')[0] pline = args['pline'] auto_script.extract(field_root=root, maglim=maglim, ids=None, run_fit=False, MW_EBV=tab.meta['MW_EBV'], pline=pline)
def database_from_query(query): engine = db.get_db_engine() utils.set_warnings() result = db.from_sql(query, engine) return result
def auto_run(root='j023507-040202'): import os import glob import matplotlib.pyplot as plt from grizli import utils, prep from grizli.pipeline import auto_script, photoz utils.set_warnings() tab = utils.GTable.gread('{0}_footprint.fits'.format(root)) HOME_PATH = os.getcwd() auto_script.VALID_FILTERS = ['F098M', 'F105W', 'F110W', 'F125W', 'F127M', 'F139M', 'F140W', 'F153M', 'F160W', 'F410M', 'F435W', 'F438W', 'F439W', 'F450W', 'F467M', 'F475W', 'F475X', 'F547M', 'F550M', 'F555W', 'F569W', 'F600LP', 'F606W', 'F621M', 'F622W', 'F625W', 'F675W', 'F689M', 'F702W', 'F763M', 'F775W', 'F791W', 'F814W', 'F845M', 'F850LP', 'F350LP'] # Include grisms auto_script.VALID_FILTERS += ['G102','G141','G800L'] IS_PARALLEL = utils.column_string_operation(tab['proposal_pi'], 'alkan', method='count', logical='or').sum() > 0 IS_DASH = list(np.unique(np.cast[int](tab['proposal_id']))) == [14114] master_radec = '{0}/{1}_master.radec'.format(os.getcwd(), root) if not os.path.exists(master_radec): if root.startswith('cos-') & os.path.exists('hsc-udeep-i25_corr_cosmos.radec'): master_radec = '{0}/{1}'.format(os.getcwd(), 'hsc-udeep-i25_corr_cosmos.radec') else: master_radec = None parent_radec = '{0}/{1}_parent.radec'.format(os.getcwd(), root) if not os.path.exists(parent_radec): parent_radec = None BKG_PARAMS = {'bw': 1024, 'bh': 1024, 'fw': 3, 'fh': 3} catalogs = ['PS1','DES','NSC', 'SDSS','GAIA','WISE'] # MACS0717 if 'macs0717' in root: catalogs = ['GAIA', 'PS1'] run_fine_alignment=False auto_script.go(root=root, maglim=[16, 23], HOME_PATH=HOME_PATH, inspect_ramps=False, manual_alignment=False, is_parallel_field=IS_PARALLEL, reprocess_parallel=True, only_preprocess=True, run_extractions=True, run_fit=False, s3_sync='cp', catalogs=catalogs, fine_radec=None, run_fine_alignment=run_fine_alignment, combine_all_filters=False, align_mag_limits=[16,24], gaia_by_date=True, align_simple=False, align_clip=120, master_radec=master_radec, parent_radec=parent_radec, is_dash=IS_DASH, run_parse_visits=True, reference_wcs_filters=['F160W','F140W','F125W','F105W','F110W','F098M','F814W','F850LP', 'F606W','F435W'], make_phot=False, make_mosaics=False, align_rms_limit=4, align_min_overlap=2, imaging_bkg_params=BKG_PARAMS)
def auto_run(root='j023507-040202', flag_global_crs=False): import os import glob import numpy as np import astropy.io.fits as pyfits import astropy.wcs as pywcs from drizzlepac import updatehdr from stwcs import updatewcs from grizli import utils, prep from grizli.pipeline import auto_script utils.set_warnings() visit_file = '{0}_visits.npy'.format(root) visits, all_groups, info = np.load(visit_file) # Something wrong with some files with bad shifts, reset wcs for visit in visits: for file in visit['files']: utils.fetch_hst_calibs( file, calib_types=['IDCTAB', 'NPOLFILE', 'IMPHTTAB']) updatewcs.updatewcs(file, verbose=True, use_db=False) # Apply shifts shift_log = '{0}_shifts.log'.format(visit['product']) if os.path.exists(shift_log): sh = utils.read_catalog(shift_log) flt0 = pyfits.open(sh['flt'][0]) wcs_ref = pywcs.WCS(flt0['SCI', 1].header, fobj=flt0, relax=True) shift_dict = {} for i in range(len(sh)): shift_dict[sh['flt'][i]] = [sh['xshift'][i], sh['yshift'][i]] prep.apply_tweak_shifts(wcs_ref, shift_dict, grism_matches={}, verbose=False) # Redrizzle mosaics prep.drizzle_overlaps(visits, check_overlaps=False, skysub=False, static=False, pixfrac=0.5, scale=None, final_wcs=False, fetch_flats=False, final_rot=None, include_saturated=True) ####### Alignment os.system('rm *wcs.*') # Radec master_radec = '{0}/../../{1}_master.radec'.format(os.getcwd(), root) if not os.path.exists(master_radec): master_radec = None ref_catalog = 'USER' if root.startswith('cos-'): hsc = '{0}/../../{1}'.format(os.getcwd(), 'hsc-udeep-i25_corr_cosmos.radec') if os.path.exists(hsc): master_radec = hsc ref_catalog = 'HSC' elif root.startswith('uds-'): hsc = '{0}/../../{1}'.format(os.getcwd(), 'hsc-udeep-sxds_corr_uds.radec') if os.path.exists(hsc): master_radec = hsc ref_catalog = 'HSC' parent_radec = '{0}/../../{1}_parent.radec'.format(os.getcwd(), root) if not os.path.exists(parent_radec): parent_radec = None if master_radec is not None: radec = master_radec elif parent_radec is not None: radec = parent_radec else: radec = None if radec is None: needs_gaia = True else: needs_gaia = False REFERENCE = 'GAIA' REFERENCE = 'PS1' print('master RADEC file: ', radec) thresh = 2.5 for visit in visits: # Clean catalogs files = glob.glob('{0}.*'.format(visit['product'])) for file in files: os.remove(file) # Generate GAIA alignment catalog at the observation epoch clip = 120 clip = -1 if needs_gaia: flt = pyfits.open(visit['files'][0]) h = flt['SCI', 1].header ra_i, dec_i = h['CRVAL1'], h['CRVAL2'] radec, ref_catalog = prep.get_radec_catalog( ra=ra_i, dec=dec_i, product=visit['product'], date=flt[0].header['EXPSTART'], date_format='mjd', reference_catalogs=[REFERENCE], radius=5.) flt.close() if REFERENCE == 'GAIA': mag_limits = [16, 20] else: mag_limits = [18, 22] #clip = 50 if '_flc' in visit['files'][0]: triangle_size_limit = [5, 4000 * np.sqrt(2)] else: triangle_size_limit = [5, 1300] else: mag_limits = [19, 23] triangle_size_limit = [5, 1300] # Remake catalogs cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh) # Redo alignment try: print('XXX clip', clip, mag_limits, triangle_size_limit) result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=mag_limits, simple=False, max_err_percentile=80, clip=clip, outlier_threshold=5, rms_limit=2.5, triangle_size_limit=triangle_size_limit) except: print('First align failed! Relax parameters') try: result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=[10, 20], simple=False, max_err_percentile=99, clip=160, outlier_threshold=20, rms_limit=2.5, triangle_size_limit=triangle_size_limit) except: try: result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=[10, 20], simple=False, max_err_percentile=99, clip=160, outlier_threshold=40, rms_limit=2.5, triangle_size_limit=triangle_size_limit) except: radec = '{0}_ps1.radec'.format(visit['product']) ref_catalog = 'PS1' result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=mag_limits, simple=False, max_err_percentile=80, clip=120, outlier_threshold=5, rms_limit=2.5, triangle_size_limit=triangle_size_limit) #continue orig_wcs, drz_wcs, out_shift, out_rot, out_scale = result # Propagate shifts for file in visit['files']: updatehdr.updatewcs_with_shift(file, str('{0}_wcs.fits'.format( visit['product'])), xsh=out_shift[0], ysh=out_shift[1], rot=out_rot, scale=out_scale, wcsname=ref_catalog, force=True, reusename=True, verbose=True, sciext='SCI') ### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS ### keywords im = pyfits.open(file, mode='update') im[0].header['MJD-OBS'] = im[0].header['EXPSTART'] im.flush() # Redrizzle mosaics again including new shifts prep.drizzle_overlaps(visits, check_overlaps=False, skysub=False, static=False, pixfrac=0.8, scale=None, final_wcs=False, fetch_flats=False, final_rot=None) # Remake catalogs thresh = 2.5 for visit in visits: # Remake catalogs cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh) prep.table_to_regions(cat, '{0}.cat.reg'.format(visit['product'])) prep.table_to_radec(cat, '{0}.cat.radec'.format(visit['product'])) # Update visits file v = auto_script.get_visit_exposure_footprints(visit_file=visit_file, check_paths=['./', '../RAW'], simplify=1.e-6) if flag_global_crs: # Assume everything at same orient pass if False: # Mosaic auto_script.drizzle_overlaps(root, filters=['F160W'], min_nexp=1, pixfrac=0.8, scale=0.1, make_combined=False, ref_image=None, static=False)
# fp = open('{0}.fp.reg'.format(root),'w') # fp.write('fk5\n') # # for weight_image in files: # root_i = '_dr'.join(weight_image.split('_dr')[:-1]) # reg = prep.drizzle_footprint(weight_image, shrink=10, ext=0, # outfile=None, label=root_i) # fp.write(reg+'\n') # # fp.close() if __name__ == "__main__": import sys import time import numpy as np from grizli import utils from grizli.pipeline import auto_script utils.set_warnings() root = sys.argv[1] #auto_run(root=root) try: auto_run(root=root) except: fp = open('{0}.failed'.format(root), 'w') fp.write(time.ctime() + '\n') fp.close()
def auto_run(root='j023507-040202'): import os import matplotlib.pyplot as plt from grizli import utils from grizli.pipeline import auto_script, photoz utils.set_warnings() tab = utils.GTable.gread('{0}_footprint.fits'.format(root)) HOME_PATH = os.getcwd() auto_script.VALID_FILTERS = [ 'F098M', 'F105W', 'F110W', 'F125W', 'F127M', 'F139M', 'F140W', 'F153M', 'F160W', 'F410M', 'F435W', 'F438W', 'F439W', 'F450W', 'F467M', 'F475W', 'F475X', 'F547M', 'F550M', 'F555W', 'F569W', 'F600LP', 'F606W', 'F621M', 'F622W', 'F625W', 'F675W', 'F689M', 'F702W', 'F763M', 'F775W', 'F791W', 'F814W', 'F845M', 'F850LP', 'F350LP' ] IS_PARALLEL = utils.column_string_operation( tab['proposal_pi'], 'alkan', method='count', logical='or').sum() > 0 auto_script.go(root=root, maglim=[19, 23], HOME_PATH=HOME_PATH, inspect_ramps=False, manual_alignment=False, is_parallel_field=IS_PARALLEL, reprocess_parallel=False, only_preprocess=True, run_extractions=False, run_fit=False, s3_sync='cp', fine_radec=None, combine_all_filters=False, gaia_by_date=True, align_simple=False, align_clip=100, master_radec=None, is_dash=False, run_parse_visits=True, reference_wcs_filters=[ 'F160W', 'F140W', 'F125W', 'F105W', 'F110W', 'F098M', 'F814W', 'F850LP', 'F606W', 'F435W' ]) plt.ioff() fig = auto_script.field_rgb(root=root, HOME_PATH=HOME_PATH, xsize=18) plt.close(fig) # Photo-z try: out = photoz.eazy_photoz(root, object_only=False, force=True, aper_ix=1, sys_err=0.05, apply_prior=False, beta_prior=True, external_limits=3, external_sys_err=0.3) except: pass
def extract_beams_from_flt(root, bucket, id, clean=True, silent=False): """ Download GrismFLT files and extract the beams file """ import gc import boto3 import matplotlib.pyplot as plt import grizli from grizli import fitting, utils, multifit from grizli.version import __version__ as grizli__version utils.set_warnings() from grizli.pipeline import auto_script s3 = boto3.resource('s3') s3_client = boto3.client('s3') bkt = s3.Bucket(bucket) # WCS files for ACS files = [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/j'.format(root))] files += [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/i'.format(root))] files += [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/{0}-ir.cat.fits'.format(root))] files += [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/fit_args.npy'.format(root))] download_files = [] for file in np.unique(files): if ('cat.fits' in file) | ('fit_args' in file): if os.path.exists(os.path.basename(file)): continue download_files.append(file) for file in download_files: print(file) bkt.download_file(file, os.path.basename(file), ExtraArgs={"RequestPayer": "requester"}) # Read the catalog ircat = utils.read_catalog('{0}-ir.cat.fits'.format(root)) ix = ircat['NUMBER'] == id object_rd = (ircat['X_WORLD'][ix], ircat['Y_WORLD'][ix]) del(ircat) # One beam at a time beams = None flt_files = [] for file in files: if 'GrismFLT.fits' in file: flt_files.append(file) if not silent: print('Read {0} GrismFLT files'.format(len(flt_files))) if os.path.exists('{0}_fit_args.npy'.format(root)): args_file = '{0}_fit_args.npy'.format(root) else: args_file = 'fit_args.npy' for i, file in enumerate(flt_files): if not silent: print('# Read {0}/{1}'.format(i+1, len(flt_files))) flt, ext, _, _ = os.path.basename(file).split('.') if flt.startswith('i'): fl = 'flt' else: fl = 'flc' out_files = ['{0}_{2}.{1}.wcs.fits'.format(flt, ext, fl), '{0}.{1}.GrismFLT.fits'.format(flt, ext), '{0}.{1}.GrismFLT.pkl'.format(flt, ext)] exp_has_id = False for j, f_j in enumerate(out_files): aws_file = os.path.join(os.path.dirname(file), f_j) if not silent: print(' ', aws_file) if not os.path.exists(f_j): bkt.download_file(aws_file, f_j, ExtraArgs={"RequestPayer": "requester"}) # WCS file, check if object in footprint if f_j.endswith('.wcs.fits'): #exp_has_id = check_object_in_footprint(id, f_j, ircat) exp_has_id = check_object_in_footprint(None, f_j, None, rd=object_rd) if not exp_has_id: if clean: os.remove(f_j) break if not exp_has_id: continue beams_i = auto_script.extract(field_root=root, maglim=[13,24], prior=None, MW_EBV=0.00, ids=id, pline={}, fit_only_beams=True, run_fit=False, poly_order=7, master_files=[os.path.basename(file)], grp=None, bad_pa_threshold=None, fit_trace_shift=False, size=32, diff=True, min_sens=0.02, skip_complete=True, fit_args={}, args_file=args_file, get_only_beams=True) # Remove the GrismFLT file for f_j in out_files: if ('GrismFLT' in f_j) & clean: os.remove(f_j) if beams is None: beams = beams_i else: beams.extend(beams_i) # Garbage collector gc.collect() if not beams: print('No beams found for {0} id={1}'.format(root, id)) return False # Grism Object args = np.load(args_file, allow_pickle=True)[0] mb = multifit.MultiBeam(beams, **args) mb.write_master_fits() # 1D spectrum with R=30 fit if True: bin_steps, step_templ = utils.step_templates(wlim=[5000, 18000.0], R=30, round=10) tfit = mb.template_at_z(z=0, templates=step_templ, fit_background=True, fitter='lstsq', get_uncertainties=2) fig1 = mb.oned_figure(figsize=[5,3], tfit=tfit, show_beams=True, scale_on_stacked=True, ylim_percentile=5) outroot='{0}_{1:05d}.R{2:.0f}'.format(root, id, 30) hdu = mb.oned_spectrum_to_hdu(outputfile=outroot+'.fits', tfit=tfit, wave=bin_steps) fig1.savefig(outroot+'.png') del(hdu) # Drizzled spectrum hdu, fig = mb.drizzle_grisms_and_PAs(fcontam=args['fcontam'], flambda=False, kernel='point', size=32, zfit=tfit, diff=False) hdu[0].header['GRIZLIV'] = (grizli__version, 'Grizli version') fig.savefig('{0}_{1:05d}.stack.png'.format(root, id)) hdu.writeto('{0}_{1:05d}.stack.fits'.format(root, id), overwrite=True) plt.close('all') del(hdu) outfiles = ['{0}_{1:05d}.beams.fits'.format(root, id)] outfiles += glob.glob(outroot+'*') outfiles += glob.glob('{0}_{1:05d}.stack*'.format(root, id)) return(outfiles)
def run_grizli_fit(event): import boto3 import json import shutil import gc import matplotlib.pyplot as plt import grizli from grizli import fitting, utils, multifit try: from grizli.aws import db as grizli_db dbFLAGS = grizli_db.FLAGS except: pass utils.set_warnings() #event = {'s3_object_path':'Pipeline/j001452+091221/Extractions/j001452+091221_00277.beams.fits'} silent = False if 'silent' in event: silent = event['silent'] in TRUE_OPTIONS ### ### Parse event arguments ### event_kwargs = {} for k in event: # Lists if isinstance(event[k], str): # Split lists if ',' in event[k]: try: event_kwargs[k] = np.cast[float](event[k].split(',')) except: event_kwargs[k] = event[k].split(',') else: event_kwargs[k] = event[k] else: try: event_kwargs[k] = json.loads(event[k]) except: event_kwargs[k] = event[k] # Defaults if 'skip_started' not in event_kwargs: event_kwargs['skip_started'] = True for k in ['quasar_fit', 'extract_from_flt', 'fit_stars', 'beam_info_only']: if k not in event_kwargs: event_kwargs[k] = False if event_kwargs['beam_info_only'] in TRUE_OPTIONS: dbtable = 'multibeam' elif event_kwargs['quasar_fit'] in TRUE_OPTIONS: dbtable = 'redshift_fit_quasar' elif event_kwargs['fit_stars'] in TRUE_OPTIONS: dbtable = 'stellar_fit' else: dbtable = 'redshift_fit' if not silent: print('Grizli version: ', grizli.__version__) # Disk space total, used, free = shutil.disk_usage("/") if not silent: print('Disk info: Total = {0:.2f} / Used = {1:.2f} / Free = {2:.2f}'.format(total // (2**20), used // (2**20), free // (2**20))) ## Output path if 'output_path' in event: output_path = event['output_path'] else: output_path = None if 'bucket' in event: event_kwargs['bucket'] = event['bucket'] else: event_kwargs['bucket'] = 'aws-grivam' if 'working_directory' in event: os.chdir(event['working_directory']) else: os.chdir('/tmp/') if not silent: print('Working directory: {0}'.format(os.getcwd())) files = glob.glob('*') files.sort() # Filenames, etc. beams_file = os.path.basename(event['s3_object_path']) root = beams_file.split('_')[0] id = int(beams_file.split('_')[1].split('.')[0]) try: db_status = grizli_db.get_redshift_fit_status(root, id, table=dbtable) except: db_status = -1 # Initial log start_log = '{0}_{1:05d}.start.log'.format(root, id) full_start = 'Pipeline/{0}/Extractions/{1}'.format(root, start_log) if ((start_log in files) | (db_status >= 0)) & event_kwargs['skip_started']: print('Log file {0} found in {1} (db_status={2})'.format(start_log, os.getcwd(), db_status)) return True if not silent: for i, file in enumerate(files): print('Initial file ({0}): {1}'.format(i+1, file)) if os.path.exists('{0}/matplotlibrc'.format(grizli.GRIZLI_PATH)): os.system('cp {0}/matplotlibrc .'.format(grizli.GRIZLI_PATH)) s3 = boto3.resource('s3') s3_client = boto3.client('s3') bkt = s3.Bucket(event_kwargs['bucket']) if event_kwargs['skip_started']: res = [r.key for r in bkt.objects.filter(Prefix=full_start)] if res: print('Already started ({0}), aborting.'.format(start_log)) return True fp = open(start_log,'w') fp.write(time.ctime()+'\n') fp.close() bkt.upload_file(start_log, full_start) # Download fit arguments if 'force_args' in event: force_args = event['force_args'] in TRUE_OPTIONS else: force_args = False args_files = ['{0}_fit_args.npy'.format(root), 'fit_args.npy'] for args_file in args_files: if (not os.path.exists(args_file)) | force_args: aws_file = 'Pipeline/{0}/Extractions/{1}'.format(root, args_file) try: bkt.download_file(aws_file, './{0}'.format(args_file), ExtraArgs={"RequestPayer": "requester"}) print('Use args_file = {0}'.format(args_file)) break except: continue # If no beams file in the bucket, try to generate it put_beams=False try: if not os.path.exists(beams_file): bkt.download_file(event['s3_object_path'], './{0}'.format(beams_file), ExtraArgs={"RequestPayer": "requester"}) put_beams = False except: print('Extract from GrismFLT object!') if 'clean' in event: if isinstance(event['clean'], str): run_clean = event['clean'].lower() in ['true', 'y', '1'] else: run_clean = event['clean'] else: run_clean = True try: # Extracting beams grizli_db.update_redshift_fit_status(root, id, status=dbFLAGS['start_beams'], table=dbtable) except: print('Set DB flag failed: start_beams') pass status = extract_beams_from_flt(root, event_kwargs['bucket'], id, clean=run_clean, silent=silent) # Garbage collector gc.collect() if status is False: return False else: beams_file = status[0] try: # Beams are done grizli_db.update_redshift_fit_status(root, id, status=dbFLAGS['done_beams'], table=dbtable) except: pass put_beams = True # upload it now output_path = 'Pipeline/{0}/Extractions'.format(root) for outfile in status: aws_file = '{0}/{1}'.format(output_path, outfile) print(aws_file) bkt.upload_file(outfile, aws_file, ExtraArgs={'ACL': 'public-read'}) if ('run_fit' in event) & (dbtable == 'redshift_fit'): if event['run_fit'] in FALSE_OPTIONS: res = bkt.delete_objects(Delete={'Objects':[{'Key':full_start}]}) try: grizli_db.update_redshift_fit_status(root, id, status=dbFLAGS['no_run_fit'], table=dbtable) except: pass return True utils.fetch_acs_wcs_files(beams_file, bucket_name=event_kwargs['bucket']) # Update the multibeam/beam_geometry tables if os.path.exists(beams_file): args = np.load(args_file, allow_pickle=True)[0] for arg in event_kwargs: if arg in args: args[arg] = event_kwargs[arg] grizli_db.multibeam_to_database(beams_file, Rspline=15, force=False, **args) if dbtable == 'multibeam': ### Done res = bkt.delete_objects(Delete={'Objects':[{'Key':full_start}]}) return True # Download WCS files # if event_kwargs['check_wcs']: # # WCS files for ACS # files = [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/j'.format(root))] # for file in files: # if 'wcs.fits' in file: # if os.path.exists(os.path.basename(file)): # continue # # bkt.download_file(file, os.path.basename(file), # ExtraArgs={"RequestPayer": "requester"}) # Is zr in the event dict? # if 'zr' in event: # zr = list(np.cast[float](event['zr'])) # else: # try: # zr = np.load('fit_args.npy')[0]['zr'] # except: # zr = np.load('fit_args.npy', allow_pickle=True)[0]['zr'] # Directory listing files = glob.glob('*') files.sort() for i, file in enumerate(files): print('File ({0}): {1}'.format(i+1, file)) try: files = glob.glob('{0}_{1:05d}*R30.fits'.format(root, id)) if (len(files) > 0) & (dbtable == 'redshift_fit'): grizli_db.send_1D_to_database(files=files) except: print('Failed to send R30 to spec1d database') pass ### ### Run the fit try: grizli_db.update_redshift_fit_status(root, id, table=dbtable, status=dbFLAGS['start_redshift_fit']) except: print('Set DB flag failed: start_redshift_fit') pass if event_kwargs['quasar_fit'] in TRUE_OPTIONS: # Don't recopy beams file put_beams = False # Don't make line maps if 'min_line_sn' not in event_kwargs: event_kwargs['min_line_sn'] = np.inf # Don't make drizzled psfs if 'get_ir_psfs' not in event_kwargs: event_kwargs['get_ir_psfs'] = False # Fit line widths if 'get_line_width' not in event_kwargs: event_kwargs['get_line_width'] = True # sys_err if 'sys_err' not in event_kwargs: event_kwargs['sys_err'] = 0.05 # Don't use photometry event_kwargs['phot_obj'] = None event_kwargs['use_phot_obj'] = False event_kwargs['fit_only_beams'] = True event_kwargs['fit_beams'] = False templ_args = {'uv_line_complex': True, 'broad_fwhm':2800, 'narrow_fwhm':1000, 'fixed_narrow_lines':True, 'Rspline':15, 'include_reddened_balmer_lines':False} for k in templ_args: if k in event_kwargs: templ_args[k] = event_kwargs.pop(k) if templ_args['broad_fwhm'] < 0: use_simple_templates=True templ_args['broad_fwhm'] *= -1 else: use_simple_templates = False print('load_quasar_templates(**{0})'.format(templ_args)) q0, q1 = utils.load_quasar_templates(**templ_args) if use_simple_templates: x0 = utils.load_templates(full_line_list=['highO32'], continuum_list=['quasar_lines.txt', 'red_blue_continuum.txt'], line_complexes=False, fwhm=1000) for t in q0: if 'bspl' in t: x0[t] = q0[t] q0 = x0 q1['red_blue_continuum.txt'] = x0['red_blue_continuum.txt'] # Quasar templates with fixed line ratios # q0, q1 = utils.load_quasar_templates(uv_line_complex=True, # broad_fwhm=2800, narrow_fwhm=1000, # fixed_narrow_lines=True, # Rspline=15) if 'zr' not in event_kwargs: event_kwargs['zr'] = [0.03, 6.8] if 'fitter' not in event_kwargs: event_kwargs['fitter'] = ['lstsq', 'lstsq'] print('run_all_parallel: {0}'.format(event_kwargs)) fitting.run_all_parallel(id, t0=q0, t1=q1, args_file=args_file, **event_kwargs) if output_path is None: #output_path = 'Pipeline/QuasarFit'.format(root) output_path = 'Pipeline/{0}/Extractions'.format(root) elif event_kwargs['fit_stars'] in TRUE_OPTIONS: args = np.load(args_file, allow_pickle=True)[0] if 'psf' in event_kwargs: args['psf'] = event_kwargs['psf'] in TRUE_OPTIONS for k in ['fcontam', 'min_sens', 'sys_err']: if k in event_kwargs: print('Set arg {0}={1}'.format(k, event_kwargs[k])) args[k] = event_kwargs[k] # Load MultiBeam mb = multifit.MultiBeam(beams_file, **args) if 'fit_trace_shift' in args: if args['fit_trace_shift']: tr = mb.fit_trace_shift() if 'spline_correction' in event_kwargs: spline_correction = event_kwargs['spline_correction'] in TRUE_OPTIONS else: spline_correction = True if 'fit_background' in event_kwargs: fit_background = event_kwargs['fit_background'] in TRUE_OPTIONS else: fit_background = True if 'fitter' in event_kwargs: fitter = event_kwargs['fitter'] else: fitter = 'lstsq' if 'Rspline' in event_kwargs: Rspline = event_kwargs['Rspline'] else: Rspline = 15 if Rspline == 15: logg_list = [4.5] else: logg_list = utils.PHOENIX_LOGG if 'add_carbon_star' in event_kwargs: add_carbon_star = event_kwargs['add_carbon_star'] else: add_carbon_star = 25 if 'use_phoenix' in event_kwargs: p = event_kwargs.pop('use_phoenix') if p in TRUE_OPTIONS: tstar = utils.load_phoenix_stars(logg_list=logg_list, add_carbon_star=add_carbon_star) else: tstar = utils.load_templates(stars=True, add_carbon_star=add_carbon_star) else: tstar = utils.load_phoenix_stars(logg_list=logg_list, add_carbon_star=add_carbon_star) kws = {'spline_correction':spline_correction, 'fit_background':fit_background, 'fitter':fitter, 'spline_args':{'Rspline':Rspline}} print('kwargs: {0}'.format(kws)) # Fit the stellar templates _res = mb.xfit_star(tstar=tstar, oned_args={}, **kws) _res[0].savefig('{0}_{1:05d}.star.png'.format(root, id)) # Save log info fp = open('{0}_{1:05d}.star.log'.format(root, id), 'w') fp.write(_res[1]) fp.close() if output_path is None: #output_path = 'Pipeline/QuasarFit'.format(root) output_path = 'Pipeline/{0}/Extractions'.format(root) else: # Normal galaxy redshift fit fitting.run_all_parallel(id, fit_only_beams=True, fit_beams=False, args_file=args_file, **event_kwargs) if output_path is None: output_path = 'Pipeline/{0}/Extractions'.format(root) # Output files files = glob.glob('{0}_{1:05d}*'.format(root, id)) for file in files: if ('beams.fits' not in file) | put_beams: aws_file = '{0}/{1}'.format(output_path, file) if event_kwargs['quasar_fit'] in TRUE_OPTIONS: # Don't copy stack if 'stack' in file: continue # Add qso extension on outputs aws_file = aws_file.replace('_{0:05d}.'.format(id), '_{0:05d}.qso.'.format(id)) print('Upload {0} -> {1}'.format(file, aws_file)) bkt.upload_file(file, aws_file, ExtraArgs={'ACL': 'public-read'}) # Put data in the redshift_fit database table try: if dbtable == 'stellar_fit': rowfile = '{0}_{1:05d}.star.log'.format(root, id) else: rowfile = '{0}_{1:05d}.row.fits'.format(root, id) if os.path.exists(rowfile): grizli_db.add_redshift_fit_row(rowfile, table=dbtable, verbose=True) # Add 1D spectra files = glob.glob('{0}_{1:05d}*1D.fits'.format(root, id)) if (len(files) > 0) & (dbtable == 'redshift_fit'): grizli_db.send_1D_to_database(files=files) except: print('Update row failed') pass # Remove start log now that done res = bkt.delete_objects(Delete={'Objects':[{'Key':full_start}]}) # Garbage collector gc.collect()
def auto_run(root='j023507-040202', args=[]): import os import yaml import time import matplotlib.pyplot as plt plt.ioff() from grizli import utils utils.set_warnings() from grizli.pipeline import auto_script, default_params import grizli # Run query again kwargs = auto_script.get_yml_parameters() kwargs['fetch_files_args']['reprocess_parallel'] = True kwargs['preprocess_args']['skip_single_optical_visits'] = False kwargs['run_fine_alignment'] = False kwargs['visit_prep_args']['reference_catalogs'] = [ 'PS1', 'DES', 'NSC', 'SDSS', 'GAIA', 'WISE' ] # params = { # # Preprocessing flags # 's3_sync': True, # Fetch data from AWS # 'inspect_ramps': False, # Visual inspection on ramps # 'remove_bad':True, # Remove bad EXPFLAG visits # 'reprocess_parallel': True, # Reprocess ramps in parallel # 'is_dash': False, # Flag if visits are in DASH mode # 'filters': auto_script.VALID_FILTERS, # Only use these filters # 'run_parse_visits': True, # Parse visits to _visits.npy # 'combine_minexp':2, # Try to combine visits with 2 or fewer exp # 'is_parallel_field': False, # If parallels, change visit parsing # 'skip_single_optical_visits': False, # Skip preprocess if single exp # 'fix_stars': True, # Fill DQ flagged centers of stars # 'imaging_bkg_params': {'bh': 128, 'bw': 128, 'fh': 3, 'fw': 3, # 'pixel_scale': 0.06}, # Imaging background # # # Alignment # 'manual_alignment': False, # Interactive alignment # 'align_clip': 120, # Parameter for initial alignment pairs # 'align_mag_limits': [14, 24], # Magnitude range for alignment sources # 'align_min_overlap': 0.2, # Overlap fraction to use HST catalogs # 'align_outlier_threshold': 4, # Parameter for triangle matches # 'align_rms_limit': 2, # Force shifts = 0 if rms > rms limit # 'align_simple': False, # ?? # 'catalogs': ['PS1','DES','NSC','SDSS','GAIA','WISE'], # Ref cat order # 'gaia_by_date': True, # Get GAIA catalogs by visit epoch # 'master_radec': None, # Force reference # 'parent_radec': None, # Reference to use if no HST overlaps # # 'run_fine_alignment': False, # Run Fine alignment script # 'fine_radec': None, # Fine alingment reference, else GAIA # # # Image Mosaics # 'make_mosaics': True, # Make full-field mosaics # 'fill_mosaics': 'grism', # Fill empty parts of IR mosaics # 'mask_spikes': False, # Mask IR diffraction spikes # 'combine_all_filters': False, # Combine ACS & IR # 'mosaic_pixel_scale': None, # Pixel scale of mosaics (0.06) # 'half_optical_pixscale': False, # Make smaller ACS/UVIS pixel scale # 'mosaic_pixfrac': 0.75, # Pixfrac of mosaic images # 'reference_wcs_filters': auto_script.VALID_FILTERS, # These define WCS if they exist # 'make_phot': True, # Make photometric catalog # # # Spectral Extractions # 'only_preprocess': False, # Make grism models # 'run_extractions': False, # Don't extract grism spectra # 'maglim': [17, 26], # Magnitude range of grism spectra # 'run_fit': False # Fit grism spectra # } try: tab = utils.GTable.gread('{0}_footprint.fits'.format(root)) #IS_PARALLEL = utils.column_string_operation(tab['proposal_pi'], ['Malkan', 'Trenti'], method='count', logical='or').sum() > 0 IS_PARALLEL = (tab['target'] == 'ANY').sum() > 0 IS_PARALLEL = bool(IS_PARALLEL) except: IS_PARALLEL = False master_radec = '{0}/{1}_master.radec'.format(os.getcwd(), root) if not os.path.exists(master_radec): master_radec = None parent_radec = '{0}/{1}_parent.radec'.format(os.getcwd(), root) if not os.path.exists(parent_radec): parent_radec = None kwargs['preprocess_args']['parent_radec'] = parent_radec kwargs['preprocess_args']['master_radec'] = master_radec # kwargs['is_dash'] = '14114' in tab['proposal_id'] # if kwargs['is_dash']: # print('\n\n!!! Process as DASH !!!!\n\n') # Limited filters kwargs['only_preprocess'] = False kwargs['filters'] = default_params.IR_W_FILTERS + default_params.IR_GRISMS kwargs['filters'] += default_params.IR_M_FILTERS # Optical filters. Bluer than F555W often fail for low source counts? kwargs['filters'] += [ 'F814W', 'F850LP', 'F775W', 'F625W', 'F606W', 'F555W', 'F350LP', 'F600LP' ] #, 'G800L'] kwargs['is_parallel_field'] = IS_PARALLEL pixel_scale = 0.06 + 0.02 * IS_PARALLEL kwargs['mosaic_args']['wcs_params']['pixel_scale'] = pixel_scale kwargs['mosaic_args']['mosaic_pixfrac'] = pixel_scale / 0.12 ### Force conservative pixel scale kwargs['mosaic_args']['wcs_params']['pixel_scale'] = 0.1 kwargs['mosaic_args']['mosaic_pixfrac'] = 0.33 #pixel_scale/0.12 kwargs['mosaic_args']['half_optical_pixscale'] = True # Try less aggressive background if CLASH # IS_BRIGHT_CLUSTER = utils.column_string_operation(tab['proposal_pi'], ['Postman', 'Lotz'], method='count', logical='or').sum() > 0 # if IS_BRIGHT_CLUSTER: # print('CLUSTER!') # Force conservative background conservative_background = { 'bh': 256, 'bw': 256, 'fh': 3, 'fw': 3, 'pixel_scale': 0.128 } # Imaging background kwargs['visit_prep_args']['imaging_bkg_params'] = conservative_background print('BKG PARAMS: {0}'.format(conservative_background)) # Command line arguments if args: for arg in args: if arg.startswith('--'): if arg in [ '--grism', '--sync', '--noclean', '--lambda_verbose' ]: continue pspl = arg.strip('--').split('=')[0] val = arg.split('=')[1] if pspl == 'redo_query': if val.lower() in ['true']: redo_query(root=root) # Next argument continue if pspl == 'extra_filters': kwargs['filters'] += [f.upper() for f in val.split(',')] kwargs['filters'] = list(np.unique(kwargs['filters'])) kwargs['filters'] = [str(f) for f in kwargs['filters']] print('Extra filters: {0}'.format(val.split(','))) continue if pspl == 'remove_filters': pop_filters = [f.upper() for f in val.split(',')] for f in pop_filters: if f in kwargs['filters']: kwargs['filters'].pop(kwargs['filters'].index(f)) print('Filters after pop: {0}'.format(kwargs['filters'])) continue # Split nested dictionaries by '.' if '.' in pspl: valid = False ps = pspl.split('.') d = kwargs for p in ps: if p in d: valid = True if isinstance(d[p], dict): d = d[p] else: d = kwargs valid = pspl in kwargs p = pspl if valid: if val.strip() in ['None', 'null']: d[p] = None print('Runtime argument: {0} = {1}/[None]'.format( p, val)) elif isinstance(d[p], list): lval = val.replace('[', '').replace(']', '').split(',') # Item shoud be a list if (len(lval) < len(d[p])) & ('filter' not in arg): msg = 'Parameter {0} should be a list like {1}' raise (ValueError(msg.format(arg, d[p]))) try: lval = list(np.cast[float](lval)) except: pass d[p] = lval print('Runtime argument: {0} = {1}'.format(p, lval)) elif '.ids' in arg: print(arg, p, val) if ',' in val: val = np.cast[int](val.split(',')) d[p] = val else: if isinstance(d[p], bool): if val.isdigit(): d[p] = int(val) else: d[p] = val.lower() == 'true' else: try: d[p] = d[p].__class__(val) except: try: d[p] = float(val) except: d[p] = val print('Runtime argument: {0} = {1}'.format(p, d[p])) # Save YAML parameter file # Need copies of a few things that will break yaml.dump # phot_apertures = kwargs['multiband_catalog_args']['phot_apertures'] # filter_kernel = kwargs['multiband_catalog_args']['detection_params']['filter_kernel'] # # kwargs['multiband_catalog_args']['phot_apertures'] = None # kwargs['multiband_catalog_args']['detection_params']['filter_kernel'] = None # # fp = open('{0}.run.yml'.format(root),'w') # fp.write('# {0}\n'.format(time.ctime())) # fp.write('# Grizli version = {0}\n'.format(grizli.__version__)) # # for k in kwargs: # try: # d = {k:kwargs[k].copy()} # except: # d = {k:kwargs[k]} # # yaml.dump(d, stream=fp, default_flow_style=False) # # fp.close() # # kwargs['multiband_catalog_args']['phot_apertures'] = phot_apertures # kwargs['multiband_catalog_args']['detection_params']['filter_kernel'] = filter_kernel # Master radec if '--preprocess_args.parent_radec' not in [a.split('=')[0] for a in args]: if 'j021732m0512' in root: radec = 'gaia_sxds-dud-HSCdr2_corr_uds.radec' radec = 'uds_prelim_hst.radec' elif 'j033236m2748' in root: #radec = 'gaia-pm_mast-hlf_corr_cdfs.radec' radec = 'cdfs_prelim_hst.radec' elif 'j100012p0210' in root: radec = 'hsc_cosmos-dud_dr2.radec' elif 'j141956p5255' in root: #radec = 'gaia_ps1+hsc_corr_aegis.radec' radec = 'egs_prelim_hst.radec' elif 'j123656p6215' in root: radec = 'goodsn_prelim_hst.radec' elif ('xxxj001352m3023' in root) | ('xxxj001420m3024' in root): radec = 'gaia_subaru_corr_abell2744.radec' elif root in [ 'j224844m4432', 'j224916m4432', 'j024012m0138', 'j023952m0135' ]: radec = '{0}_f814w_mag23.radec'.format(root) kwargs['preprocess_args']['master_radec'] = '../../' + radec elif root in ['j114936p2222']: radec = '{0}_f160w_mag23.radec'.format(root) kwargs['preprocess_args']['master_radec'] = '../../' + radec else: radec = None if radec is not None: os.system( 'aws s3 cp s3://grizli/AlignmentCatalogs/{0} .'.format(radec)) kwargs['preprocess_args']['parent_radec'] = '../../' + radec for k in kwargs: if kwargs[k] == 'NoneType': kwargs[k] = None elif isinstance(kwargs[k], dict): for k_i in kwargs[k]: if kwargs[k][k_i] == 'NoneType': kwargs[k][k_i] = None output_yml = '{0}.auto_script.yml'.format(root) auto_script.write_params_to_yml(kwargs, output_file=output_yml) auto_script.go(root=root, **kwargs) #auto_script.go(root=root, maglim=[19, 23], HOME_PATH=HOME_PATH, inspect_ramps=False, manual_alignment=False, is_parallel_field=IS_PARALLEL, reprocess_parallel=True, only_preprocess=False, run_extractions=False, run_fit=False, s3_sync='cp', fine_radec=None, combine_all_filters=False, gaia_by_date=True, align_simple=False, align_clip=100, master_radec=master_radec, parent_radec=parent_radec, is_dash=False, run_parse_visits=True, reference_wcs_filters=['F160W','F140W','F125W','F105W','F110W','F098M','F814W','F850LP', 'F606W','F435W']) os.chdir('../Prep/') auto_script.make_report(root, make_rgb=True) # Done without errors os.system('date > /tmp/{0}.success'.format(root))
Needs 'fit_args.py' created by `auto_script.generate_fit_params`. """ import time import os import glob import numpy as np import matplotlib.pyplot as plt plt.ioff() from grizli.fitting import run_all_parallel from grizli import utils utils.set_warnings() def find_ids(): # Find objects that with extarcted spectra and that need to be fit all_files=glob.glob('*beams.fits') files = [] for file in all_files: if not os.path.exists(file.replace('beams.fits', 'full.fits')): files.append(file) print('{0} files to fit'.format(len(files))) ids = [int(file.split('_')[1].split('.')[0]) for file in files] return ids
def run_grizli_fit(event): import boto3 import json import shutil import gc import grizli from grizli import fitting, utils, multifit utils.set_warnings() #event = {'s3_object_path':'Pipeline/j001452+091221/Extractions/j001452+091221_00277.beams.fits'} silent = False if 'silent' in event: silent = event['silent'] in TRUE_OPTIONS ### ### Parse event arguments ### event_kwargs = {} for k in event: # Lists if isinstance(event[k], str): # Split lists if ',' in event[k]: try: event_kwargs[k] = np.cast[float](event[k].split(',')) except: event_kwargs[k] = event[k].split(',') else: event_kwargs[k] = event[k] else: try: event_kwargs[k] = json.loads(event[k]) except: event_kwargs[k] = event[k] # Defaults if 'skip_started' not in event_kwargs: event_kwargs['skip_started'] = True for k in ['quasar_fit', 'extract_from_flt']: if k not in event_kwargs: event_kwargs[k] = False if not silent: print('Grizli version: ', grizli.__version__) # Disk space total, used, free = shutil.disk_usage("/") if not silent: print('Disk info: Total = {0:.2f} / Used = {1:.2f} / Free = {2:.2f}'. format(total // (2**20), used // (2**20), free // (2**20))) ## Output path if 'output_path' in event: output_path = event['output_path'] else: output_path = None if 'bucket' in event: event_kwargs['bucket'] = event['bucket'] else: event_kwargs['bucket'] = 'aws-grivam' if 'working_directory' in event: os.chdir(event['working_directory']) else: os.chdir('/tmp/') if not silent: print('Working directory: {0}'.format(os.getcwd())) files = glob.glob('*') files.sort() # Filenames, etc. beams_file = os.path.basename(event['s3_object_path']) root = beams_file.split('_')[0] id = int(beams_file.split('_')[1].split('.')[0]) # Initial log start_log = '{0}_{1:05d}.start.log'.format(root, id) full_start = 'Pipeline/{0}/Extractions/{1}'.format(root, start_log) if (start_log in files) & event_kwargs['skip_started']: print('Log file {0} found in {1}'.format(start_log, os.getcwd())) return True if not silent: for i, file in enumerate(files): print('Initial file ({0}): {1}'.format(i + 1, file)) os.system('cp {0}/matplotlibrc .'.format(grizli.GRIZLI_PATH)) s3 = boto3.resource('s3') s3_client = boto3.client('s3') bkt = s3.Bucket(event_kwargs['bucket']) if event_kwargs['skip_started']: res = [r.key for r in bkt.objects.filter(Prefix=full_start)] if res: print('Already started ({0}), aborting.'.format(start_log)) return True fp = open(start_log, 'w') fp.write(time.ctime() + '\n') fp.close() bkt.upload_file(start_log, full_start) # Download fit arguments args_file = 'fit_args.npy' needs_args = False if not os.path.exists(args_file): needs_args = True else: if 'force_args' in event: needs_args = event['force_args'] in TRUE_OPTIONS if needs_args: aws_args = 'Pipeline/{0}/Extractions/fit_args.npy'.format(root) bkt.download_file(aws_args, './fit_args.npy', ExtraArgs={"RequestPayer": "requester"}) # If no beams file in the bucket, try to generate it put_beams = False try: if not os.path.exists(beams_file): bkt.download_file(event['s3_object_path'], './{0}'.format(beams_file), ExtraArgs={"RequestPayer": "requester"}) put_beams = False except: print('Extract from GrismFLT object!') if 'clean' in event: if isinstance(event['clean'], str): run_clean = event['clean'].lower() in ['true', 'y', '1'] else: run_clean = event['clean'] else: run_clean = True status = extract_beams_from_flt(root, event_kwargs['bucket'], id, clean=run_clean, silent=silent) # Garbage collector gc.collect() if status is False: return False else: beams_file = status[0] put_beams = True # upload it now output_path = 'Pipeline/{0}/Extractions'.format(root) for outfile in status: aws_file = '{0}/{1}'.format(output_path, outfile) print(aws_file) bkt.upload_file(outfile, aws_file, ExtraArgs={'ACL': 'public-read'}) if 'run_fit' in event: if event['run_fit'] in FALSE_OPTIONS: return True utils.fetch_acs_wcs_files(beams_file, bucket_name=event_kwargs['bucket']) # Download WCS files # if event_kwargs['check_wcs']: # # WCS files for ACS # files = [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/j'.format(root))] # for file in files: # if 'wcs.fits' in file: # if os.path.exists(os.path.basename(file)): # continue # # bkt.download_file(file, os.path.basename(file), # ExtraArgs={"RequestPayer": "requester"}) # Is zr in the event dict? # if 'zr' in event: # zr = list(np.cast[float](event['zr'])) # else: # try: # zr = np.load('fit_args.npy')[0]['zr'] # except: # zr = np.load('fit_args.npy', allow_pickle=True)[0]['zr'] # Directory listing files = glob.glob('*') files.sort() for i, file in enumerate(files): print('File ({0}): {1}'.format(i + 1, file)) ### ### Run the fit if event_kwargs['quasar_fit']: # Quasar templates uv_lines = True #zr[1] > 3.5 t0, t1 = utils.load_quasar_templates(uv_line_complex=uv_lines, broad_fwhm=2800, narrow_fwhm=1000, fixed_narrow_lines=True, nspline=13) fitting.run_all_parallel(id, t0=t0, t1=t1, fit_only_beams=True, fit_beams=False, phot_obj=None, **event_kwargs) if output_path is None: output_path = 'Pipeline/QuasarFit'.format(root) else: # Normal galaxy redshift fit fitting.run_all_parallel(id, fit_only_beams=True, fit_beams=False, **event_kwargs) if output_path is None: output_path = 'Pipeline/{0}/Extractions'.format(root) # Output files files = glob.glob('{0}_{1:05d}*'.format(root, id)) for file in files: if ('beams.fits' not in file) | put_beams: aws_file = '{0}/{1}'.format(output_path, file) print(aws_file) bkt.upload_file(file, aws_file, ExtraArgs={'ACL': 'public-read'}) # Remove start log now that done res = bkt.delete_objects(Delete={'Objects': [{'Key': full_start}]}) # Garbage collector gc.collect()