def update(datadir, outdir):
ref_file = os.path.join(datadir, 'ref_obs_data.dat')
if os.path.exists(ref_file):
print "Reading reference data from {}".format(ref_file)
ref_data = Table.read(ref_file, format='ascii')
else:
ref_data = get_obs_table('2015:100', '2016:100', msf='ENAB')
ref_data.write(ref_file, format='ascii')
data_file = os.path.join(datadir, 'msd_data.dat')
if os.path.exists(data_file):
print "Reading previous data from {}".format(data_file)
last_data = Table.read(data_file, format='ascii')
new_data = get_obs_table(last_data[-5]['date'], DateTime(), msf='DISA')
if new_data['time'][0] > last_data['time'][-1]:
msd_data = vstack([last_data, new_data])
else:
idx_old_data = np.flatnonzero(last_data['time'] >= new_data['time'][0])[0]
msd_data = vstack([last_data[0:idx_old_data], new_data])
msd_data.write(data_file, format='ascii')
# but only use the last year for making these plots
msd_data = msd_data[msd_data['date'] >= (DateTime() - 365).date]
else:
msd_data = get_obs_table(DateTime() - 365, DateTime(), msf='DISA')
msd_data.write(data_file, format='ascii')
print msd_data[-1]['date']
# Filter known bad obsids (MUPS test fires)
for obsid in [50702, 49802]:
msd_data = msd_data[msd_data['obsid'] != obsid]
one_shot_plot(outdir=outdir)
att_err_time_plots(ref_data, msd_data, outdir=outdir)
att_err_hist(ref_data, msd_data, outdir=outdir)
def do_it(comm):
rand = np.random.RandomState()
rank = comm.Get_rank()
nproc = comm.Get_size()
mockpaths = ["/project/projectdirs/desi/mocks/GaussianRandomField/v0.0.4/",
"/project/projectdirs/desi/mocks/GaussianRandomField/v0.0.4/",
"/project/projectdirs/desi/mocks/bgs/MXXL/desi_footprint/v0.0.3/BGS_new_footprint.hdf5"]
targetnames = ["ELG", "LRG", "BGS"]
mockformats = ["gaussianfield", "gaussianfield", "durham_mxxl_hdf5"]
#open fits files
targetsfile = 'targets_proc_{}.fits'.format(rank)
alltargets = list()
alltruth = list()
alltrueflux = list()
for mock_path, target_name, mock_format in zip(mockpaths, targetnames, mockformats):
targets, truth, trueflux, targkeep = get_mock_spectra(mock_path, target_name, mock_format, rand=rand, comm=comm)
print(rank, target_name, 'RA', len(targets['RA']), targets['RA'][0])
alltargets.append(targets)
alltruth.append(truth)
alltrueflux.append(trueflux)
targets = vstack(alltargets)
truth = vstack(alltruth)
trueflux = np.concatenate(alltrueflux)
targets.write(targetsfile, overwrite=True)
return 0
def test_vstack():
"""
Vstack tables with mixin cols.
"""
t1 = QTable(MIXIN_COLS)
t2 = QTable(MIXIN_COLS)
with pytest.raises(NotImplementedError):
vstack([t1, t2])
def combine_isochrones(isoc_list):
""" combine isochrone Tables into 1 Table"""
if isinstance(isoc_list[0], Table):
# assume that these data are all Table
comb_isoc = vstack(isoc_list)
else:
for i in xrange(isoc_list):
isoc_list[i] = Table(isoc_list[i])
comb_isoc = vstack(isoc_list)
return comb_isoc
def match_two_cats(ref_cats_file,test_cats_file):
# Set the debugging level
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
#get lists of tractor cats to compare
fns_1= read_lines(ref_cats_file)
fns_2= read_lines(test_cats_file)
log.info('Comparing tractor catalogues: ')
for one,two in zip(fns_1,fns_2): log.info("%s -- %s" % (one,two))
#if fns_1.size == 1: fns_1,fns_2= [fns_1],[fns_2]
#object to store concatenated matched tractor cats
ref_matched = []
ref_missed = []
test_matched = []
test_missed = []
d_matched= 0.
deg2= dict(ref=0.,test=0.,matched=0.)
#for cnt,cat1,cat2 in zip(range(len(fns_1)),fns_1,fns_2):
for cnt,cat1,cat2 in zip([0],[fns_1[0]],[fns_2[0]]):
log.info('Reading %s -- %s' % (cat1,cat2))
ref_tractor = Table(fits.getdata(cat1, 1))
test_tractor = Table(fits.getdata(cat2, 1))
m1, m2, d12 = match_radec(ref_tractor['ra'].copy(), ref_tractor['dec'].copy(),\
test_tractor['ra'].copy(), test_tractor['dec'].copy(), \
1.0/3600.0)
miss1 = np.delete(np.arange(len(ref_tractor)), m1, axis=0)
miss2 = np.delete(np.arange(len(test_tractor)), m2, axis=0)
log.info('matched %d/%d' % (len(m2),len(test_tractor['ra'])))
# Build combined catalogs
if len(ref_matched) == 0:
ref_matched = ref_tractor[m1]
ref_missed = ref_tractor[miss1]
test_matched = test_tractor[m2]
test_missed = test_tractor[miss2]
d_matched= d12
else:
ref_matched = vstack((ref_matched, ref_tractor[m1]))
ref_missed = vstack((ref_missed, ref_tractor[miss1]))
test_matched = vstack((test_matched, test_tractor[m2]))
test_missed = vstack((test_missed, test_tractor[miss2]))
d_matched= np.concatenate([d_matched, d12])
deg2['ref']+= deg2_lower_limit(ref_tractor['ra'],ref_tractor['dec'])
deg2['test']+= deg2_lower_limit(test_tractor['ra'],test_tractor['dec'])
deg2['matched']+= deg2_lower_limit(ref_matched['ra'],ref_matched['dec'])
return dict(ref_matched = ref_matched,
ref_missed = ref_missed,
test_matched = test_matched,
test_missed = test_missed,
d_matched= d_matched,
deg2= deg2)
def join_starcheck_telem(fids_starcheck, fids_telem):
"""
Remake dict of tables into a single table for each structure
"""
# Stack the dict of tables into a single table
t_fids_starcheck = table.vstack([fids_starcheck[obsid] for obsid in sorted(fids_starcheck)])
t_fids_telem = table.vstack([fids_telem[obsid] for obsid in sorted(fids_telem)])
# Join on obsid and slot columns into a single table
starcheck_telem = table.join(t_fids_starcheck, t_fids_telem, keys=['obsid', 'slot'])
# Reject unacquired fids
ok = starcheck_telem['aoacyan'] > -3276
return starcheck_telem[ok]
def update_mass_table(drpall, mass_table_old=None, limit=None, mlband='i'):
'''
'''
# what galaxies are available to aggregate?
res_fnames = glob(os.path.join(basedir, 'results/*-*/*-*_res.fits'))[:limit]
# filter out whose that have not been done
if mass_table_old is None:
already_aggregated = [False for _ in range(len(res_fnames))]
else:
already_aggregated = [os.path.split(fn)[1].split('_')[0] in old_mass_table['plateifu']
for fn in res_fnames]
res_fnames = [fn for done, fn in zip(already_aggregated, res_fnames)]
# aggregate individual galaxies, and stack them
mass_tables_new = list(ProgressBar.map(
partial(mass_agg_onegal, mlband=mlband), res_fnames, multiprocess=False, step=5))
mass_table_new = t.vstack(mass_tables_new)
# if there was an old mass table, stack it with the new one
if mass_table_old is None:
mass_table = mass_table_new
else:
mass_table = t.vstack([mass_table_old, mass_table_new], join_type='inner')
cmlr = totalmass.cmlr_kwargs
missing_flux = (mass_table['nsa_absmag'].to(m.Mgy) - \
mass_table['ifu_absmag'].to(m.Mgy)).clip(
a_min=0.*m.Mgy, a_max=np.inf*m.Mgy)
mag_missing_flux = missing_flux.to(u.ABmag)
cb1, cb2 = cmlr['cb1'], cmlr['cb2']
color_missing_flux = mag_missing_flux[:, totalmass.StellarMass.bands_ixs[cb1]] - \
mag_missing_flux[:, totalmass.StellarMass.bands_ixs[cb2]]
color_missing_flux[~np.isfinite(color_missing_flux)] = np.inf
mass_table['outer_ml_cmlr'] = np.polyval(cmlr['cmlr_poly'], color_missing_flux.value) * \
u.dex(m.m_to_l_unit)
mass_table['outer_lum'] = mag_missing_flux.to(
u.dex(m.bandpass_sol_l_unit),
totalmass.bandpass_flux_to_solarunits(totalmass.StellarMass.absmag_sun))
mass_table['outer_mass_ring'] = \
(mass_table['outer_lum'][:, totalmass.StellarMass.bands_ixs['i']] + \
mass_table['outer_ml_ring']).to(u.Msun)
mass_table['outer_mass_cmlr'] = \
(mass_table['outer_lum'][:, totalmass.StellarMass.bands_ixs['i']] + \
mass_table['outer_ml_cmlr']).to(u.Msun)
return mass_table
def drop_lines(dataset, drop_fractions):
#inserting all supernovae into a single table
for supernova in dataset.data.keys():
new_sna=dataset.data[supernova].copy()
new_col=Table.Column(name='supernova', data=np.repeat(supernova, len(dat.data[supernova])))
new_sna.add_column(new_col, index=0)
if supernova==dataset.data.keys()[0]:
data_table=new_sna
else:
data_table=vstack([data_table, new_sna], metadata_conflicts='silent')
#in each band, determine the number of lines to drop, and drop them
for band in unique(data_table, keys='filter')['filter']:
band_indices=np.where(data_table['filter']==band)
num_drops=int(np.floor(len(data_table[band_indices])*drop_fractions[band]))
#print 'Dropping '+str(num_drops)+' lines of '+str(len(data_table))+' ...'
if num_drops > 0:
lines_to_drop=np.random.choice(band_indices[0], num_drops, replace=False)
else:
lines_to_drop=[]
data_table.remove_rows(lines_to_drop)
#make copy of the original dataset, place rows from data_table in there
new_dataset=copy.deepcopy(dataset)
types=dataset.get_types()
new_types=new_dataset.get_types()
for supernova in dataset.data.keys():
new_dataset.data[supernova]=data_table[data_table['supernova']==supernova]
new_dataset.data[supernova].remove_column('supernova')
return new_dataset
def accessTabZranges(lineconfig = 'wOIII_nHaNIISII', survey='sdss', joinsurveys=True):
"""
access zranges table
PARAMS
----------
lineconfig = 'wOIII_nHaNIISII'
if 'all' than all the tables for all the line cofigs available in the survey is combined and returned
surver = 'sdss'
joinsurveys=True
"""
if lineconfig !='all':
filenames = ['zranges_band_'+lineconfig+'.txt']
tab = accessFile(filename=f, survey=survey, joinsurveys=joinsurveys)
else:
dir_survey = getlocalpath()+survey+'/'
filenames = glob.glob(dir_survey+'zranges_band_*.txt')
tab = at.Table()
for f in filenames:
tabnew = at.Table.read(f, format='ascii')
tab = at.vstack([tab, tabnew])
print(filenames)
return tab
def get_fits_catalog(args, index_table):
"""Makes catalog containing information about parametric fits to the galaxies.
Columns are identical to COSMOS Real Galaxy catalog"""
print "Creating fits catalog"
all_seg_ids = np.loadtxt(args.seg_list_file, delimiter=" ",dtype='S2')
for f, filt in enumerate(args.filter_names):
final_table = fits_table()
for seg_id in all_seg_ids:
file_name = args.main_path + seg_id + '/' + filt + '_with_pstamp.fits'
seg_cat = Table.read(file_name, format='fits')
q, = np.where(index_table['SEG_ID'] == seg_id)
indx_seg = index_table[q]
temp = join(seg_cat, indx_seg, keys='NUMBER')
temp.rename_column('MAG_AUTO', 'mag_auto')
temp.rename_column('FLUX_RADIUS', 'flux_radius')
col = Column(temp['stamp_flux'], name='flux')
temp.add_column(col)
final_table = vstack([final_table,temp], join_type='inner')
path = args.main_path + args.out_dir
index_table.sort('ORDER')
ord_indx = [np.where(i_t==final_table['IDENT'])[0][0] for i_t in index_table['IDENT']]
file_name = args.fits_file_name.replace('filter', args.file_filter_name[f])
print "Savings fits file at ", path + file_name
final_table[ord_indx].write(path + file_name, format='fits',
overwrite=True)
def assign_num(args):
"""Assigns individual identification number to each object"""
seed =122
np.random.seed(seed)
print "Assigning number"
names = ('SEG_ID', 'NUMBER', 'IDENT')
dtype = ('string', 'int', 'int')
index_table = Table(names=names, dtype = dtype)
ident = 0
#objects detected are same in all filters. So getting objects in first filter
#is sufficient
filt = args.filter_names[0]
all_seg_ids = np.loadtxt(args.seg_list_file, delimiter=" ",dtype='S2')
for seg_id in all_seg_ids:
file_name = args.main_path + seg_id + '/' + filt + '_with_pstamp.fits'
catalog = Table.read(file_name, format='fits')
idents = range(ident,ident+len(catalog))
seg_ids = [seg_id]*len(catalog)
numbers = catalog['NUMBER']
temp = Table([seg_ids, numbers,idents],names=names, dtype = dtype)
index_table = vstack([index_table,temp])
ident+=len(catalog)
shuffle_idents = range(len(index_table))
np.random.shuffle(shuffle_idents)
index_table= index_table[shuffle_idents]
order_idents = range(len(index_table))
file_nums = np.array(order_idents)/1000 + 1
hdus= np.zeros(len(order_idents))
names = ('ORDER', 'FILE_NUM', 'HDU')
dtype = ('int' ,'int', 'int')
temp = Table([order_idents,file_nums,hdus], names=names, dtype=dtype)
index_table = hstack([index_table,temp])
cat_name = args.main_path + 'index_table_' + args.cat_name.replace('filter', '')
return index_table
def readps1_coordinates(ra,dec,outfile=None,path='/line12/Pan-STARRS/chunks-qz-star-v2/',silent=True):
"""
This program aims to get PS1 catalog where objects (ra,dec) located
ra,dec: in degrees, scalar
"""
ra=np.array(ra)
dec=np.array(dec)
phi=np.deg2rad(ra)
theta=np.deg2rad(90.0-dec)
ipring=ang2pix(32,theta,phi)
pix = np.unique(ipring)
npix=pix.size
if not silent:
print('Total '+str(npix)+' healpix pixels:')
cat=[]
for ipix in pix:
catname=os.path.join(path,'ps1-'+'%5.5d'%ipix+'.fits')
if not os.path.isfile(catname):
continue
if not silent:
print('reading '+catname)
cat.append(table.Table.read(catname,format='fits'))
totalcat = []
if cat != []:
totalcat=table.vstack(cat)
if outfile is not None:
totalcat.write(outfile,format='fits',overwrite=True)
return totalcat
def _tile_objcat(ad, shifts):
"""
This produces a single Table instance combining all the individual
OBJCATs, with X_IMAGE and Y_IMAGE updated to account for the tiling,
and NUMBER changed to avoid duplications.
Parameters
----------
ad: astrodata
input AD instance (with OBJCATs)
shifts: list of 2-tuples
array shifts (x,y) from original extension into tiled image
Returns
-------
Table: the tiled OBJCAT
"""
tiled_objcat = None
for ext, shift in zip(ad, shifts):
try:
objcat = ext.OBJCAT
except AttributeError:
pass
else:
objcat['X_IMAGE'] += shift[0]
objcat['Y_IMAGE'] += shift[1]
if tiled_objcat:
objcat['NUMBER'] += max(tiled_objcat['NUMBER'])
tiled_objcat = vstack([tiled_objcat, objcat],
metadata_conflicts='silent')
else:
tiled_objcat = objcat
return tiled_objcat
def main():
usage = "usage: %(prog)s [config files]"
description = "Merge tables."
parser = argparse.ArgumentParser(usage=usage,description=description)
parser.add_argument('--output', default = None, required=True)
parser.add_argument('files', nargs='*', default = None,
help='One or more FITS files containing BINTABLEs.')
args = parser.parse_args()
h0 = pyfits.open(args.files[0])
tables = []
for f in sorted(args.files):
print f
tables += [Table.read(f)]
tab = vstack(tables)
tab.write(args.output,format='fits',overwrite=True)
h = pyfits.open(args.output)
h0[1] = h[1]
h0.writeto(args.output,clobber=True)
def dict_to_fits(dict, clusters):
# Recombining table_dict into a single table
fits_table = dict[1][[]]
for i in range(1, clusters + 1):
for key in dict.keys():
fits_table = vstack([fits_table, dict[key]])
return fits_table
def read_aat():
aat_path = SAGA_DROPBOX + '/Spectra/Final/AAT/'
aat_files = glob.glob(aat_path+'*zlog')
n=0
for afile in aat_files:
# ACCEPT ALL GOOD SPECTRA
adata = ascii.read(afile, data_start=0, delimiter=' ')
msk = adata.field('col7') >= 1 # ONLY OBJECTS WITH QUALITY GE 1
aat = adata[msk]
# PLACE HOLDER ARRAYS
one = np.ones(len(aat))
telname = ['AAT' for i in one]
spec_repeat = ['AAT' for i in one]
maskid = [afile for i in one]
maskid = [x.split(SAGA_DROPBOX,1)[1] for x in maskid]
# CREATE MMT SPEC TABLE
aat_table1 = table.table.Table([aat['col2'], aat['col3'], maskid, aat['col8'],\
aat['col5'], aat['col7'], telname,spec_repeat], \
names=('RA', 'DEC', 'MASKNAME','specobjid',\
'SPEC_Z','ZQUALITY','TELNAME','SPEC_REPEAT'))
# CREATE OR APPEND TO AAT TABLE
if (n==0): aat_table = aat_table1
if (n > 0): aat_table = table.vstack([aat_table,aat_table1])
n=n+1
print "Number of AAT Files = ",n
return aat_table
def accessFile(filename='zranges_band_wOIII_nHaNIISII.txt', survey='sdss', joinsurveys=True):
"""
access files in filters/ such as:
HaNIIredshiftrange0.2.txt
OIIIredshiftrange0.6.txt
PARAMS
---------
filename='OIIIredshiftrange0.6.txt'
survey='sdss': (string)
for joined surveys use '-', e.g., hsc-ukirt
joinsurveys=True (string)
if true then if survey/filename does not exist, for joined surveys go into each directories/files and merge file
"""
filepath = getlocalpath()+survey+'/'+filename
if os.path.isfile(filepath):
tab = at.Table.read(filepath, format='ascii')
elif ('-' in survey) and joinsurveys:
surveys = survey.split('-')
tab = at.Table()
for s in surveys:
filepath = getlocalpath()+s+'/'+filename
tabnew = at.Table.read(filepath, format='ascii')
tab = at.vstack([tab, tabnew])
else:
raise NameError('File does not exist: '+filepath)
return tab
def make_total_index_table(self, data_store, modeltype, out_dir_background_model=None,
filename_obs_group_table=None, smooth=False):
"""Create a hdu-index table with a row containing the link to the background model for each observation.
Parameters
----------
data_store : `~gammapy.data.DataStore`
`DataStore` for the runs for which ones we want to compute a background model
modeltype : {'3D', '2D'}
Type of the background modelisation
out_dir_background_model : str
directory where are located the backgrounds models for each group
filename_obs_group_table : str
name of the file containing the `~astropy.table.Table` with the group infos
smooth : bool
True if you want to use the smooth bkg model
Returns
-------
index_table_new : `~astropy.table.Table`
Index hdu table with a background row
"""
index_table_bkg = self.make_bkg_index_table(data_store, modeltype, out_dir_background_model,
filename_obs_group_table, smooth)
index_bkg = np.where(data_store.hdu_table["HDU_CLASS"] == "bkg_3d")[0].tolist()
data_store.hdu_table.remove_rows(index_bkg)
index_table_new = vstack([data_store.hdu_table, index_table_bkg])
return index_table_new
def test_daogroup_three(self):
"""
1 +--+-------+--------+--------+--------+-------+--------+--+
| |
| |
| |
0.5 + +
| |
| |
0 + * * * * * * * * * * +
| |
| |
-0.5 + +
| |
| |
| |
-1 +--+-------+--------+--------+--------+-------+--------+--+
0 0.5 1 1.5 2 2.5 3
"""
first_group = Table([np.linspace(0, 1, 5), np.zeros(5),
np.arange(5) + 1, np.ones(5, dtype=np.int)],
names=('x_0', 'y_0', 'id', 'group_id'))
second_group = Table([np.linspace(2, 3, 5), np.zeros(5),
6 + np.arange(5), 2*np.ones(5, dtype=np.int)],
names=('x_0', 'y_0', 'id', 'group_id'))
starlist = vstack([first_group, second_group])
daogroup = DAOGroup(crit_separation=0.3)
test_starlist = daogroup(starlist['x_0', 'y_0', 'id'])
assert_array_equal(starlist, test_starlist)
def s2n(t,FUV,tnorm=1000):
# load cos etc simulations for flat spectrum 1000s exp, and FUV = 17 mag
cos_g130m = ascii.read('cos_etc_g130m_v24.1.csv')
cos_g160m = ascii.read('cos_etc_g160m_v24.1.csv')
#separate them at 1400 A
cond = cos_g130m['wavelength'] < 1405
cos_g130m = cos_g130m[cond]
cond = cos_g160m['wavelength'] >= 1405
cos_g160m = cos_g160m[cond]
#merge both
cos = vstack([cos_g130m,cos_g160m], join_type='exact')
# Signal
signal = cos['target_counts']*t/tnorm * 10.**((FUV-17.)/(-2.5))
# Noise terms
dark = cos['dark_counts']*t/tnorm
sky = cos['sky_counts']*t/tnorm
# Noise
var = signal + dark + sky
sig = np.sqrt(var)
#append S/N to cos
sn = signal/sig * np.sqrt(6) # per-resolution element of 3 pixels
cos.add_column(Column(name='sn', data=sn))
return cos
def stack(cls, flux_points):
"""Create flux points by stacking list of flux points.
The first `FluxPoints` object in the list is taken as a reference to infer
column names and units for the stacked object.
Parameters
----------
flux_points : list of `FluxPoints`
List of flux points to stack.
Returns
-------
flux_points : `FluxPoints`
Flux points without upper limit points.
"""
reference = flux_points[0].table
tables = []
for _ in flux_points:
table = _.table
for colname in reference.colnames:
column = reference[colname]
if column.unit:
table[colname] = table[colname].quantity.to(column.unit)
tables.append(table[reference.colnames])
table_stacked = vstack(tables)
table_stacked.meta["SED_TYPE"] = reference.meta["SED_TYPE"]
return cls(table_stacked)
def load_raw_bok_aperphot(dataMap,targetName,season=None,old=False):
if old:
# renaming
pfx = {'sdssstarsold':'sdssbright'}.get(targetName,targetName)
aperCatDir = os.environ['HOME'] + \
'/data/projects/SDSS-RM/rmreduce/catalogs_v2b/'
else:
aperCatDir = os.path.join(dataMap.procDir,'catalogs')
pfx = targetName+'_aper'
allTabs = []
for utd in dataMap.iterUtDates():
if ( season is not None and
not ( utd.startswith(season) or
(utd.startswith('2013') and season=='2014') ) ):
continue
print 'loading catalogs from ',utd
for filt in dataMap.iterFilters():
if old and utd=='20131223':
utd = '20131222'
aperCatFn = '.'.join([pfx,utd,filt,'cat','fits'])
aperCatF = os.path.join(aperCatDir,aperCatFn)
if os.path.exists(aperCatF):
if old:
tab = _read_old_catf(dataMap.obsDb,aperCatF)
else:
tab = Table.read(aperCatF)
# tab['filter'] = filt # handy to save this here
allTabs.append(tab)
#
phot = vstack(allTabs)
print 'stacked aperture phot catalogs into table with ',len(phot),' rows'
#self.phot.sort(['objId','frameIndex'])
phot.sort('objId')
return phot
def grab_meta():
""" Grab GGG meta Table
Returns
-------
"""
# This table has units in it!
meta = Table.read(os.getenv('RAW_IGMSPEC')+'/GGG/GGG_catalog.fits.gz')
nqso = len(meta)
# Turn off RA/DEC units
for key in ['RA', 'DEC']:
meta[key].unit = None
meta.rename_column('RA', 'RA_GROUP')
meta.rename_column('DEC', 'DEC_GROUP')
#
# Add zem
meta['zem_GROUP'] = meta['z_gmos']
meta['sig_zem'] = meta['zerror_gmos']
meta['flag_zem'] = [str('GGG')]*nqso
meta.add_column(Column([2000.]*nqso, name='EPOCH'))
#
meta['STYPE'] = [str('QSO')]*nqso
# Double up for the two gratings
ggg_meta = vstack([meta,meta])
# Check
assert chk_meta(ggg_meta, chk_cat_only=True)
# Return
return ggg_meta
def __init__(self, name, FPorbname, tt2tdb_mode = 'pint'):
if FPorbname.startswith('@'):
# Read multiple orbit files names
FPlist = []
fnames = [ll.strip() for ll in open(FPorbname[1:]).readlines()]
for fn in fnames:
FPlist.append(load_FPorbit(fn))
self.FPorb = vstack(FPlist)
# Make sure full table is sorted
self.FPorb.sort('MJD_TT')
else:
self.FPorb = load_FPorbit(FPorbname)
# Now build the interpolator here:
self.X = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['X'])
self.Y = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Y'])
self.Z = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Z'])
self.Vx = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Vx'])
self.Vy = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Vy'])
self.Vz = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Vz'])
super(NICERObs, self).__init__(name=name, tt2tdb_mode=tt2tdb_mode)
# Print this warning once, mainly for @paulray
if self.tt2tdb_mode.lower().startswith('pint'):
log.debug('Using location=None for TT to TDB conversion (pint mode)')
elif self.tt2tdb_mode.lower().startswith('geo'):
log.warning('Using location geocenter for TT to TDB conversion')
def concatenate_line_tables(filelist,outtablefile='compiledVPoutputs.dat'):
'''
Compiles the output from several fitting runs into a single table
Parameters
----------
filelist : list of strings or str
This should be a list containing the names of VP input files or a string referring to a file simply
listing the input files.
See joebvpfit.readpars for details of file format
outtablefile : str
Name of compiled model parameter output file
'''
if isinstance(filelist, str):
lstarr=np.genfromtxt(filelist,dtype=None)
listofiles=lstarr.tolist()
else:
listofiles=filelist
tabs = []
for i, ff in enumerate(listofiles):
tabs.append(ascii.read(ff))
bigpartable = vstack(tabs)
ascii.write(bigpartable, output=outtablefile, delimiter='|') # write out compiled table
def DoPhotometryofNight(PC,night):
""" Does photometry of all images in the night as well as create the final magnitude catlog table """
with open(os.path.join(PC.OUTDIR,night,PC.OUTFITSFILELIST),'r') as outfitsfilelist:
# Skip blank lines and Commented out lines with #
imgfilterlist = [tuple(imageLINE.rstrip().split()) for imageLINE in outfitsfilelist
if ((imageLINE.strip() is not '') and (imageLINE[0] !='#'))]
for OutFinalImage,Filtr in imgfilterlist:
MagTable = RunPhotometryofImage(OutFinalImage,Filtr)
## Append differential 2MASS magnitude of all sources to table
TableHeaders = ['ra','dec','mag','magerror','Qflag']#filter,epoch,ImgFile
filter_col = Column(name='Filter', data=[Filtr]*len(MagTable))
epoch_col = Column(name='Epoch', data=[epoch]*len(MagTable))
imgfile_col = Column(name='ImgFile', data=[OutFinalImage]*len(MagTable))
TableToOutput = MagTable[TableHeaders]
TableToOutput.add_columns([filter_col, epoch_col, imgfile_col])
# Now append the Full output table also to an ascii file.
OutputTableFilename = os.path.join(PC.OUTDIR,PC.OUTPUTFILE)
try :
PreviousFullTable = ascii.read(OutputTableFilename,delimiter=',',
format='commented_header',fill_values=('--','0'))
except IOError :
logger.info('No previous photometry output found, hence we will be creating'
' a new file {0}.'.format(OutputTableFilename))
OutputTableToWrite = TableToOutput
else :
OutputTableToWrite = table.vstack([PreviousFullTable,TableToOutput], join_type='outer')
# Writing the final appended table
ascii.write(OutputTableToWrite, OutputTableFilename,delimiter=',',format='commented_header')
logger.info("Photometry of {0} over.".format(OutFinalImage))
def __init__(self,start_sci=48,end_sci=301,raw_dir='bigdog',
start_sky=35,end_sky=44,edit_dir='edited',
start_flat=23,end_flat=34,
start_arc=11,end_arc=22,
start_dark=1,end_dark=10):
"""
This Class Takes Raw IRTF and pre-processes them for the IRAF/IDL
Pipeline
Parameters
--------------
start_sci: int
The starting image number for science data
end_sci: int
The ending image number for science data
raw_dir: str
The raw directory for SpeX Spectrograph (Bigdog) images
"""
self.start_sci = start_sci
self.end_sci = end_sci
self.raw_dir = raw_dir
self.edit_dir = edit_dir
self.sciTable = self.clean_names('run',start_ind=start_sci,end_ind=end_sci,
usedName='spc')
self.skyTable = self.clean_names('runsky',start_ind=start_sky,end_ind=end_sky,
usedName='spc')
self.flatTable = self.clean_names('flat',start_ind=start_flat,end_ind=end_flat)
self.arcTable = self.clean_names('arc',start_ind=start_arc,end_ind=end_arc)
self.darkTable = self.clean_names('dark',start_ind=start_dark,end_ind=end_dark,
usedName='spc')
self.allTables = vstack([self.sciTable,self.skyTable,self.flatTable,
self.arcTable,self.darkTable])
def s2n_COS(t, FUV, tnorm=1000, v="27.1"):
# load cos etc simulations for flat spectrum 1000s exp, and FUV = 17 mag
data_path('cos_etc_g130m_v26.1.csv')
if v == '26.1':
cos_g130m = ascii.read(data_path('cos_etc_g130m_v26.1.csv'))
cos_g160m = ascii.read(data_path('cos_etc_g160m_v26.1.csv'))
elif v=='27.1':
cos_g130m = ascii.read(data_path('cos_etc_g130m_v27.1.csv'))
cos_g160m = ascii.read(data_path('cos_etc_g160m_v27.1.csv'))
else:
raise ValueError("The current version is not implemented or its out of date.")
#separate them at ~1450 A
cond = cos_g130m['wavelength'] < 1440
cos_g130m = cos_g130m[cond]
cond = cos_g160m['wavelength'] >= 1440
cos_g160m = cos_g160m[cond]
# merge both
cos = vstack([cos_g130m, cos_g160m], join_type='exact')
# Signal
signal = cos['target_counts'] * t / tnorm * 10.**((FUV- 17.)/(-2.5))
# Noise terms
dark = cos['dark_counts'] * t / tnorm
sky = cos['sky_counts'] * t / tnorm
# Noise
var = signal + dark + sky
sig = np.sqrt(var)
#append S/N to cos
sn = signal/sig * np.sqrt(6) # per-resolution element of 6 pixels
cos['sn'] = sn
return cos
def read_imacs():
imacs_path = SAGA_DROPBOX + '/Spectra/Final/IMACS/'
imacs_files = glob.glob(imacs_path+'*zlog')
n=0
for ifile in imacs_files:
# ACCEPT ALL GOOD SPECTRA
idata = ascii.read(ifile,guess=False,format='no_header', delimiter=' ',names=['specid','ra','dec','col4','z','col6','zq','col8','col9','col10','slitid','col12'])
msk = idata.field('zq') >= 1
imacs = idata[msk]
# PLACE HOLDER ARRAYS
one = np.ones(len(imacs))
telname = ['IMACS' for i in one]
spec_repeat = ['IMACS' for i in one]
maskid = [ifile for i in one]
maskid = [x.split(SAGA_DROPBOX,1)[1] for x in maskid]
# CREATE MMT SPEC TABLE
imacs_table1 = table.table.Table([imacs['ra'], imacs['dec'], imacs['slitid'],imacs['col8'],\
imacs['z'], imacs['zq'], telname,spec_repeat], \
names=('RA', 'DEC', 'MASKNAME','specobjid',\
'SPEC_Z','ZQUALITY','TELNAME','SPEC_REPEAT'))
# CREATE OR APPEND TO imacs TABLE
if (n==0): imacs_table = imacs_table1
if (n > 0): imacs_table = table.vstack([imacs_table,imacs_table1])
n=n+1
print "Number of IMACS Files = ",n
return imacs_table
def read_snana_ascii_multi(fnames, default_tablename=None):
"""Like ``read_snana_ascii()``, but read from multiple files containing
the same tables and glue results together into big tables.
Parameters
----------
fnames : list of str
List of filenames.
Returns
-------
tables : dictionary of `~astropy.table.Table`
Tables indexed by table names.
Examples
--------
>>> tables = read_snana_ascii_multi(['data1.txt', 'data1.txt'])
"""
alltables = {}
for fname in fnames:
meta, tables = read_snana_ascii(fname,
default_tablename=default_tablename)
for key, table in six.iteritems(tables):
if key in alltables:
alltables[key].append(table)
else:
alltables[key] = [table]
for key in alltables.keys():
alltables[key] = vstack(alltables[key])
return alltables
def to_table(self,
sed_type="likelihood",
format="gadf-sed",
formatted=False):
"""Create table for a given SED type.
Parameters
----------
sed_type : {"likelihood", "dnde", "e2dnde", "flux", "eflux"}
Sed type to convert to. Default is `likelihood`
format : {"gadf-sed", "lightcurve", "binned-time-series", "profile"}
Format specification. The following formats are supported:
* "gadf-sed": format for sed flux points see :ref:`gadf:flux-points`
for details
* "lightcurve": Gammapy internal format to store energy dependent
lightcurves. Basically a generalisation of the "gadf" format, but
currently there is no detailed documentation available.
* "binned-time-series": table format support by Astropy's
`~astropy.timeseries.BinnedTimeSeries`.
* "profile": Gammapy internal format to store energy dependent
flux profiles. Basically a generalisation of the "gadf" format, but
currently there is no detailed documentation available.
formatted : bool
Formatted version with column formats applied. Numerical columns are
formatted to .3f and .3e respectively.
Examples
--------
This is how to read and plot example flux points:
>>> from gammapy.estimators import FluxPoints
>>> fp = FluxPoints.read("$GAMMAPY_DATA/hawc_crab/HAWC19_flux_points.fits")
>>> table = fp.to_table(sed_type="flux", format="gadf-sed", formatted=True)
>>> print(table[:2])
e_ref e_min e_max flux flux_err flux_ul ts sqrt_ts is_ul
TeV TeV TeV 1 / (cm2 s) 1 / (cm2 s) 1 / (cm2 s)
----- ----- ----- ----------- ----------- ----------- -------- ------- -----
1.334 1.000 1.780 1.423e-11 3.135e-13 nan 2734.000 52.288 False
2.372 1.780 3.160 5.780e-12 1.082e-13 nan 4112.000 64.125 False
Returns
-------
table : `~astropy.table.Table`
Flux points table
"""
if format == "gadf-sed":
# TODO: what to do with GTI info?
if not self.geom.axes.names == ["energy"]:
raise ValueError("Only flux points with a single energy axis "
"can be converted to 'gadf-sed'")
idx = (Ellipsis, 0, 0)
table = self.energy_axis.to_table(format="gadf-sed")
table.meta["SED_TYPE"] = sed_type
if self.n_sigma_ul:
table.meta["UL_CONF"] = np.round(
1 - 2 * stats.norm.sf(self.n_sigma_ul), 7)
if sed_type == "likelihood":
table["ref_dnde"] = self.dnde_ref[idx]
table["ref_flux"] = self.flux_ref[idx]
table["ref_eflux"] = self.eflux_ref[idx]
for quantity in self.all_quantities(sed_type=sed_type):
data = getattr(self, quantity, None)
if data:
table[quantity] = data.quantity[idx]
if self.has_stat_profiles:
norm_axis = self.stat_scan.geom.axes["norm"]
table["norm_scan"] = norm_axis.center.reshape((1, -1))
table["stat"] = self.stat.data[idx]
table["stat_scan"] = self.stat_scan.data[idx]
table["is_ul"] = self.is_ul.data[idx]
elif format == "lightcurve":
time_axis = self.geom.axes["time"]
tables = []
for idx, (time_min,
time_max) in enumerate(time_axis.iter_by_edges):
table_flat = Table()
table_flat["time_min"] = [time_min.mjd]
table_flat["time_max"] = [time_max.mjd]
fp = self.slice_by_idx(slices={"time": idx})
table = fp.to_table(sed_type=sed_type, format="gadf-sed")
for column in table.columns:
table_flat[column] = table[column][np.newaxis]
tables.append(table_flat)
table = vstack(tables)
elif format == "binned-time-series":
message = (
"Format 'binned-time-series' support a single time axis "
f"only. Got {self.geom.axes.names}")
if not self.geom.axes.is_unidimensional:
raise ValueError(message)
axis = self.geom.axes.primary_axis
if not isinstance(axis, TimeMapAxis):
raise ValueError(message)
table = Table()
table["time_bin_start"] = axis.time_min
table["time_bin_size"] = axis.time_delta
for quantity in self.all_quantities(sed_type=sed_type):
data = getattr(self, quantity, None)
if data:
table[quantity] = data.quantity.squeeze()
elif format == "profile":
x_axis = self.geom.axes["projected-distance"]
tables = []
for idx, (x_min, x_max) in enumerate(x_axis.iter_by_edges):
table_flat = Table()
table_flat["x_min"] = [x_min]
table_flat["x_max"] = [x_max]
table_flat["x_ref"] = [(x_max + x_min) / 2]
fp = self.slice_by_idx(slices={"projected-distance": idx})
table = fp.to_table(sed_type=sed_type, format="gadf-sed")
for column in table.columns:
table_flat[column] = table[column][np.newaxis]
tables.append(table_flat)
table = vstack(tables)
else:
raise ValueError(f"Not a supported format {format}")
if formatted:
table = self._format_table(table=table)
return table
def main(argv=None):
""" Main Function """
parser = get_parser()
args = parser.parse_args(argv)
args.log = setup_logging()
if args.pickle:
args.fits = False
if args.merge:
if args.fits:
master_table = Table()
files = glob.glob(op.join(args.mergepath, "*.fits"))
args.log.info("Merging all fits files in " + args.mergepath)
for file in files:
file_table = Table.read(open(file, "rb"))
if np.size(file_table) > 0:
master_table = vstack([master_table, file_table])
outfile = args.outfile + ".fits"
master_table.write(outfile, format="fits", overwrite=True)
else:
all_source_dict = {}
files = glob.glob(op.join(args.mergepath, "*.pkl"))
args.log.info("Merging all pickle files in " + args.mergepath)
for file in files:
file_dict = pickle.load(open(file, "rb"))
if len(file_dict) > 0:
all_source_dict = merge(all_source_dict, file_dict)
outfile = args.outfile + ".pkl"
pickle.dump(all_source_dict, open(outfile, "wb"))
args.log.info("Saved output file to " + outfile)
sys.exit("Exiting")
if args.infile:
args.log.info("Loading External File")
try:
try:
table_in = Table.read(args.infile, format="ascii")
if table_in.colnames == ["col1", "col2", "col3"]:
table_in["col1"].name = "ID"
table_in["col2"].name = "ra"
table_in["col3"].name = "dec"
elif np.size(table_in.colnames) != 3:
args.log.info("Input file not in three column format")
except Exception:
pass
try:
table_in = Table.read(args.infile, format="fits")
except Exception:
pass
except Exception:
if op.exists(args.infile):
args.log.warning("Could not open input file")
sys.exit("Exiting")
else:
args.log.warning("Input file not found")
sys.exit("Exiting")
try:
args.ID = table_in["ID"]
except:
args.ID = table_in["id"]
try:
args.ra = table_in["ra"]
args.dec = table_in["dec"]
except:
args.ra = table_in["RA"]
args.dec = table_in["DEC"]
else:
if args.ID == None:
if np.size(args.ra) > 1:
args.ID = str(np.arange(1, np.size(table_in) + 1)).zfill(9)
else:
args.ID = 1
args.log.info("Extracting for ID: %s" % args.ID)
# generate astropy coordinates object for searching
if re.search(":", str(args.ra)):
args.coords = SkyCoord(args.ra, args.dec, unit=(u.hourangle, u.deg))
else:
args.coords = SkyCoord(args.ra, args.dec, unit=u.deg)
S = Survey(args.survey)
if args.keep_bad_shots:
ind_good_shots = np.ones_like(S.shotid, dtype=bool)
else:
ind_good_shots = S.remove_shots()
if args.tpmin:
ind_tp = S.response_4540 > args.tpmin
args.survey_class = S[ind_good_shots * ind_tp]
else:
args.survey_class = S[ind_good_shots]
# if args.shotidid exists, only select those shots
if args.shotid:
try:
sel_shot = args.survey_class.shotid == int(args.shotid)
except Exception:
sel_shot = args.survey_class.datevobs == str(args.shotid)
args.survey_class = args.survey_class[sel_shot]
else:
pass
# main function to retrieve spectra dictionary
Source_dict = get_spectra_dictionary(args)
args.survey_class.close()
if args.pickle:
outfile = args.outfile + ".pkl"
pickle.dump(Source_dict, open(outfile, "wb"))
if args.single:
# loop over every ID/observation combo:
fluxden_u = 1e-17 * u.erg * u.s**(-1) * u.cm**(-2) * u.AA**(-1)
for ID in Source_dict.keys():
for shotid in Source_dict[ID].keys():
wave_rect = 2.0 * np.arange(1036) + 3470.0
spec = Source_dict[ID][shotid][0]
spec_err = Source_dict[ID][shotid][1]
weights = Source_dict[ID][shotid][2]
sel = np.isfinite(spec)
if np.sum(sel) > 0:
output = Table()
output.add_column(
Column(wave_rect, name="wavelength", unit=u.AA))
output.add_column(Column(spec, name="spec",
unit=fluxden_u))
output.add_column(
Column(spec_err, name="spec_err", unit=fluxden_u))
output.add_column(Column(weights, name="weights"))
output.write("spec_" + str(ID) + "_" + str(shotid) +
".tab",
format="ascii")
if args.fits:
output = return_astropy_table(Source_dict,
fiberweights=args.fiberweights)
if args.fiberweights:
# cannot save fiberweights to a fits file
output.remove_column('fiber_weights')
output.write(args.outfile + ".fits", format="fits", overwrite=True)
def main():
logging.basicConfig(level=logging.INFO)
logging.getLogger("pyirf").setLevel(logging.DEBUG)
for k, p in particles.items():
log.info(f"Simulated {k.title()} Events:")
p["events"], p["simulation_info"] = read_eventdisplay_fits(p["file"])
p["simulated_spectrum"] = PowerLaw.from_simulation(
p["simulation_info"], T_OBS)
p["events"]["weight"] = calculate_event_weights(
p["events"]["true_energy"], p["target_spectrum"],
p["simulated_spectrum"])
p["events"]["source_fov_offset"] = calculate_source_fov_offset(
p["events"])
# calculate theta / distance between reco and assuemd source positoin
# we handle only ON observations here, so the assumed source pos
# is the pointing position
p["events"]["theta"] = calculate_theta(
p["events"],
assumed_source_az=p["events"]["pointing_az"],
assumed_source_alt=p["events"]["pointing_alt"],
)
log.info(p["simulation_info"])
log.info("")
gammas = particles["gamma"]["events"]
# background table composed of both electrons and protons
background = table.vstack(
[particles["proton"]["events"], particles["electron"]["events"]])
log.info(
f"Using fixed G/H cut of {INITIAL_GH_CUT} to calculate theta cuts")
# event display uses much finer bins for the theta cut than
# for the sensitivity
theta_bins = add_overflow_bins(
create_bins_per_decade(
10**(-1.9) * u.TeV,
10**2.3005 * u.TeV,
50,
))
# theta cut is 68 percent containmente of the gammas
# for now with a fixed global, unoptimized score cut
mask_theta_cuts = gammas["gh_score"] >= INITIAL_GH_CUT
theta_cuts = calculate_percentile_cut(
gammas["theta"][mask_theta_cuts],
gammas["reco_energy"][mask_theta_cuts],
bins=theta_bins,
min_value=0.05 * u.deg,
fill_value=np.nan * u.deg,
percentile=68,
)
# evaluate the theta cut
gammas["selected_theta"] = evaluate_binned_cut(gammas["theta"],
gammas["reco_energy"],
theta_cuts, operator.le)
# we make the background region larger by a factor of ALPHA,
# so the radius by sqrt(ALPHA) to get better statistics for the background
theta_cuts_bg = get_bg_cuts(theta_cuts, ALPHA)
background["selected_theta"] = evaluate_binned_cut(
background["theta"], background["reco_energy"], theta_cuts_bg,
operator.le)
# same bins as event display uses
sensitivity_bins = add_overflow_bins(
create_bins_per_decade(10**-1.9 * u.TeV,
10**2.31 * u.TeV,
bins_per_decade=5))
log.info("Optimizing G/H separation cut for best sensitivity")
sensitivity_step_2, gh_cuts = optimize_gh_cut(
gammas[gammas["selected_theta"]],
background[background["selected_theta"]],
bins=sensitivity_bins,
cut_values=np.arange(-1.0, 1.005, 0.05),
op=operator.ge,
alpha=ALPHA,
)
# now that we have the optimized gh cuts, we recalculate the theta
# cut as 68 percent containment on the events surviving these cuts.
for tab in (gammas, background):
tab["selected_gh"] = evaluate_binned_cut(tab["gh_score"],
tab["reco_energy"], gh_cuts,
operator.ge)
theta_cuts_opt = calculate_percentile_cut(
gammas["theta"],
gammas["reco_energy"],
theta_bins,
fill_value=np.nan * u.deg,
percentile=68,
min_value=0.05 * u.deg,
)
theta_cuts_opt_bg = get_bg_cuts(theta_cuts_opt, ALPHA)
for tab, cuts in zip([gammas, background],
[theta_cuts_opt, theta_cuts_opt_bg]):
tab["selected_theta"] = evaluate_binned_cut(tab["theta"],
tab["reco_energy"], cuts,
operator.le)
tab["selected"] = tab["selected_theta"] & tab["selected_gh"]
signal_hist = create_histogram_table(gammas[gammas["selected"]],
bins=sensitivity_bins)
background_hist = create_histogram_table(
background[background["selected"]], bins=sensitivity_bins)
sensitivity = calculate_sensitivity(signal_hist,
background_hist,
alpha=ALPHA)
# scale relative sensitivity by Crab flux to get the flux sensitivity
for s in (sensitivity_step_2, sensitivity):
s["flux_sensitivity"] = s["relative_sensitivity"] * CRAB_HEGRA(
s["reco_energy_center"])
# write OGADF output file
hdus = [
fits.PrimaryHDU(),
fits.BinTableHDU(sensitivity, name="SENSITIVITY"),
fits.BinTableHDU(sensitivity_step_2, name="SENSITIVITY_STEP_2"),
fits.BinTableHDU(theta_cuts, name="THETA_CUTS"),
fits.BinTableHDU(theta_cuts_opt, name="THETA_CUTS_OPT"),
fits.BinTableHDU(gh_cuts, name="GH_CUTS"),
]
masks = {
"": gammas["selected"],
"_NO_CUTS": slice(None),
"_ONLY_GH": gammas["selected_gh"],
"_ONLY_THETA": gammas["selected_theta"],
}
# binnings for the irfs
true_energy_bins = add_overflow_bins(
create_bins_per_decade(
10**-1.9 * u.TeV,
10**2.31 * u.TeV,
10,
))
reco_energy_bins = add_overflow_bins(
create_bins_per_decade(
10**-1.9 * u.TeV,
10**2.31 * u.TeV,
10,
))
fov_offset_bins = [0, 0.5] * u.deg
source_offset_bins = np.arange(0, 1 + 1e-4, 1e-3) * u.deg
energy_migration_bins = np.geomspace(0.2, 5, 200)
for label, mask in masks.items():
effective_area = point_like_effective_area(
gammas[mask],
particles["gamma"]["simulation_info"],
true_energy_bins=true_energy_bins,
)
hdus.append(
create_aeff2d_hdu(
effective_area[...,
np.newaxis], # add one dimension for FOV offset
true_energy_bins,
fov_offset_bins,
extname="EFFECTIVE_AREA" + label,
))
edisp = energy_dispersion(
gammas[mask],
true_energy_bins=true_energy_bins,
fov_offset_bins=fov_offset_bins,
migration_bins=energy_migration_bins,
)
hdus.append(
create_energy_dispersion_hdu(
edisp,
true_energy_bins=true_energy_bins,
migration_bins=energy_migration_bins,
fov_offset_bins=fov_offset_bins,
extname="ENERGY_DISPERSION" + label,
))
bias_resolution = energy_bias_resolution(
gammas[gammas["selected"]],
true_energy_bins,
)
ang_res = angular_resolution(
gammas[gammas["selected_gh"]],
true_energy_bins,
)
psf = psf_table(
gammas[gammas["selected_gh"]],
true_energy_bins,
fov_offset_bins=fov_offset_bins,
source_offset_bins=source_offset_bins,
)
background_rate = background_2d(
background[background['selected_gh']],
reco_energy_bins,
fov_offset_bins=np.arange(0, 11) * u.deg,
t_obs=T_OBS,
)
hdus.append(
create_background_2d_hdu(
background_rate,
reco_energy_bins,
fov_offset_bins=np.arange(0, 11) * u.deg,
))
hdus.append(
create_psf_table_hdu(
psf,
true_energy_bins,
source_offset_bins,
fov_offset_bins,
))
hdus.append(
create_rad_max_hdu(theta_bins,
fov_offset_bins,
rad_max=theta_cuts_opt["cut"][:, np.newaxis]))
hdus.append(fits.BinTableHDU(ang_res, name="ANGULAR_RESOLUTION"))
hdus.append(
fits.BinTableHDU(bias_resolution, name="ENERGY_BIAS_RESOLUTION"))
fits.HDUList(hdus).writeto("pyirf_eventdisplay.fits.gz", overwrite=True)
uvb=uvb)
# to efficiently save numpy array
save_file_name = outpath + '/' + name
np.save(save_file_name, flat_samples)
out = [[q]]
for i in range(ndim):
mcmc = np.percentile(flat_samples[:, i], [16, 50, 84])
q = np.diff(mcmc)
out.append([mcmc[1]])
out.append([q[0]])
out.append([q[1]])
print(out)
t = tab.Table(out, names=('Q', 'nH', 'n16', 'n84', 'Z', 'Z16', 'Z84'))
out_tab = tab.vstack((out_tab, t))
uvb_column = [
'Q14', 'Q15', 'Q16', 'Q17', 'Q18', 'Q19', 'Q20', 'P19', 'FG20', 'HM12'
]
out_tab.add_column(uvb_column, name='uvb')
out_tab.write(outfile, overwrite=True)
"""
ions_to_use= ['C+3', 'N+3', 'Si+3', 'O+5', 'C+2']
true_Q =18
outpath = '/home/vikram/cloudy_run/figures/2DLLS'
model_path = '/home/vikram/cloudy_run/metal_NH19'
outfile = outpath + '/NH19_metal_2D.fits'
if len(t) == 0:
continue
if args.min_good_time is not None:
tgti = Table.read(fn, hdu='gti')
good_time = np.sum(tgti['STOP'] - tgti['START'])
if good_time < args.min_good_time:
print('Ignoring file {0} with good time {1:0.2f}.'.format(
fn, good_time))
continue
tlist.append(t)
log.info('Concatenating files')
if len(tlist) == 1:
etable = tlist[0]
else:
etable = vstack(tlist, metadata_conflicts='silent')
del tlist
m = args.maxharm
ts_func = cached_zm if args.ztest else cached_hm
phasesinitial = etable['PULSE_PHASE'].astype(np.float32)
hbest = z2m(phasesinitial, m=m)[-1] if args.ztest else hm(phasesinitial, m=m)
eminbest = 0.0
emaxbest = 100.0
ts_name = "Ztest" if args.ztest else "Htest"
if args.ztest:
print("Initial {0} = {1}".format(ts_name, np.round(hbest, 3)))
else:
print("Initial {0} = {1} ({2} sigma)".format(ts_name, np.round(hbest, 3),
np.round(h2sig(hbest), 3)))
def do_photometry(self, image, init_guesses=None):
"""
Perform PSF photometry in ``image``.
This method assumes that ``psf_model`` has centroids and flux
parameters which will be fitted to the data provided in
``image``. A compound model, in fact a sum of ``psf_model``,
will be fitted to groups of stars automatically identified by
``group_maker``. Also, ``image`` is not assumed to be background
subtracted. If ``init_guesses`` are not ``None`` then this method
uses ``init_guesses`` as initial guesses for the centroids. If
the centroid positions are set as ``fixed`` in the PSF model
``psf_model``, then the optimizer will only consider the flux as
a variable.
Parameters
----------
image : 2D array-like, `~astropy.io.fits.ImageHDU`, `~astropy.io.fits.HDUList`
Image to perform photometry.
init_guesses: `~astropy.table.Table`
Table which contains the initial guesses (estimates) for the set
of parameters. Columns 'x_0' and 'y_0' which represent the
positions (in pixel coordinates) for each object must be present.
'flux_0' can also be provided to set initial fluxes.
If 'flux_0' is not provided, aperture photometry is used to
estimate initial values for the fluxes. Additional columns of the
form '<parametername>_0' will be used to set the initial guess for
any parameters of the ``psf_model`` model that are not fixed.
Returns
-------
output_table : `~astropy.table.Table` or None
Table with the photometry results, i.e., centroids and
fluxes estimations and the initial estimates used to start
the fitting process.
None is returned if no sources are found in ``image``.
"""
if init_guesses is not None:
table = super(IterativelySubtractedPSFPhotometry,
self).do_photometry(image, init_guesses)
table['iter_detected'] = np.ones(table['x_fit'].shape,
dtype=np.int32)
# n_start = 2 because it starts in the second iteration
# since the first iteration is above
output_table = self._do_photometry(init_guesses.colnames,
n_start=2)
output_table = vstack([table, output_table])
else:
if self.bkg_estimator is not None:
self._residual_image = image - self.bkg_estimator(image)
if self.aperture_radius is None:
if hasattr(self.psf_model, 'fwhm'):
self.aperture_radius = self.psf_model.fwhm.value
elif hasattr(self.psf_model, 'sigma'):
self.aperture_radius = (self.psf_model.sigma.value *
gaussian_sigma_to_fwhm)
output_table = self._do_photometry(['x_0', 'y_0', 'flux_0'])
return output_table
def from_horizons(cls,
targetids,
id_type='smallbody',
epochs=None,
center='500@10',
**kwargs):
"""Load target orbital elements from
`JPL Horizons <https://ssd.jpl.nasa.gov/horizons.cgi>`_ using
`astroquery.jplhorizons.HorizonsClass.elements`
Parameters
----------
targetids : str or iterable of str
Target identifier, i.e., a number, name, designation, or JPL
Horizons record number, for one or more targets.
id_type : str, optional
The nature of ``targetids`` provided; possible values are
``'smallbody'`` (asteroid or comet), ``'majorbody'`` (planet or
satellite), ``'designation'`` (asteroid or comet designation),
``'name'`` (asteroid or comet name), ``'asteroid_name'``,
``'comet_name'``, ``'id'`` (Horizons id).
Default: ``'smallbody'``
epochs : `~astropy.time.Time` or dict, optional
Epochs of elements to be queried; requires a
`~astropy.time.Time` object with a single or multiple epochs. A
dictionary including keywords ``start`` and ``stop``, as well
as either ``step`` or ``number``, can be used to generate a range
of epochs. ``start`` and ``stop`` have to be
`~astropy.time.Time` objects (see :ref:`epochs`).
If ``step`` is provided, a range
of epochs will be queries starting at ``start`` and ending at
``stop`` in steps of ``step``; ``step`` has to be provided as
a `~astropy.units.Quantity` object with integer value and a
unit of either minutes, hours, days, or years. If
``number`` is provided as an integer, the
interval defined by
``start`` and ``stop`` is split into ``number`` equidistant
intervals. If ``None`` is
provided, current date and time are
used. All epochs should be provided in TDB; if not, they will be
converted to TDB and a `~sbpy.data.TimeScaleWarning` will be
raised. Default: ``None``
center : str, optional, default ``'500@10'`` (center of the Sun)
Elements will be provided relative to this position.
**kwargs : optional
Arguments that will be provided to
`astroquery.jplhorizons.HorizonsClass.elements`.
Notes
-----
* For detailed explanations of the queried fields, refer to
`astroquery.jplhorizons.HorizonsClass.elements` and the
`JPL Horizons documentation <https://ssd.jpl.nasa.gov/?horizons_doc>`_.
* By default, elements are provided in the J2000.0 reference
system and relative to the ecliptic and mean equinox of the
reference epoch. Different settings can be chosen using
additional keyword arguments as used by
`astroquery.jplhorizons.HorizonsClass.elements`.
Returns
-------
`~Orbit` object
Examples
--------
>>> from sbpy.data import Orbit
>>> from astropy.time import Time
>>> epoch = Time('2018-05-14', scale='tdb')
>>> eph = Orbit.from_horizons('Ceres', epochs=epoch) # doctest: +REMOTE_DATA
"""
# modify epoch input to make it work with astroquery.jplhorizons
# maybe this stuff should really go into that module....
if epochs is None:
epochs = [Time.now().tdb.jd]
elif isinstance(epochs, Time):
if epochs.scale is not 'tdb':
warn(('converting {} epochs to tdb for use in '
'astroquery.jplhorizons').format(epochs.scale),
TimeScaleWarning)
epochs = epochs.tdb.jd
elif isinstance(epochs, dict):
if 'start' in epochs and 'stop' in epochs and 'number' in epochs:
epochs['step'] = epochs['number'] * u.dimensionless_unscaled
# convert to tdb and iso for astroquery.jplhorizons
epochs['start'] = epochs['start'].tdb.iso
epochs['stop'] = epochs['stop'].tdb.iso
if 'step' in epochs:
if epochs['step'].unit is not u.dimensionless_unscaled:
epochs['step'] = '{:d}{:s}'.format(
int(epochs['step'].value), {
u.minute: 'm',
u.hour: 'h',
u.d: 'd',
u.year: 'y'
}[epochs['step'].unit])
else:
epochs['step'] = '{:d}'.format(
int(epochs['step'].value - 1))
# if targetids is a list, run separate Horizons queries and append
if not isinstance(targetids, (list, ndarray, tuple)):
targetids = [targetids]
# append elements table for each targetid
all_elem = None
for targetid in targetids:
# load elements using astroquery.jplhorizons
obj = Horizons(id=targetid,
id_type=id_type,
location=center,
epochs=epochs)
try:
elem = obj.elements(**kwargs)
except ValueError as e:
raise QueryError(
('Error raised by astroquery.jplhorizons: {:s}\n'
'The following query was attempted: {:s}').format(
str(e), obj.uri))
# workaround for current version of astroquery to make
# column units compatible with astropy.table.QTable
# should really change '---' units to None in
# astroquery.jplhorizons.__init__.py
for column_name in elem.columns:
if elem[column_name].unit == '---':
elem[column_name].unit = None
if all_elem is None:
all_elem = elem
else:
all_elem = vstack([all_elem, elem])
# turn epochs into astropy.time.Time and apply timescale
# https://ssd.jpl.nasa.gov/?horizons_doc
all_elem['epoch'] = Time(all_elem['datetime_jd'],
format='jd',
scale='tdb')
all_elem.remove_column('datetime_jd')
all_elem.remove_column('datetime_str')
return cls.from_table(all_elem)
def save_scan(times,
ra,
dec,
channels,
filename='out.fits',
other_columns=None,
scan_type=None,
src_ra=None,
src_dec=None,
srcname='Dummy',
counts_to_K=0.03):
"""Save a simulated scan in fitszilla format.
Parameters
----------
times : iterable
times corresponding to each bin center, in seconds
ra : iterable
RA corresponding to each bin center
dec : iterable
Dec corresponding to each bin center
channels : {'Ch0': array([...]), 'Ch1': array([...]), ...}
Dictionary containing the count array. Keys represent the name of the
channel
filename : str
Output file name
srcname : str
Name of the source
counts_to_K : float, array or dict
Conversion factor between counts and K. If array, it has to be the same
length as channels.keys()
"""
if src_ra is None:
src_ra = np.mean(ra)
if src_dec is None:
src_dec = np.mean(dec)
# If it's a single value, make it into a list
if not isinstance(counts_to_K, collections.Iterable):
counts_to_K = counts_to_K * np.ones(len(list(channels.keys())))
# If it's a list, make it into a dict
if not hasattr(counts_to_K, 'keys'):
counts_to_K = dict([(ch, counts_to_K[i])
for i, ch in enumerate(channels.keys())])
curdir = os.path.abspath(os.path.dirname(__file__))
template = os.path.abspath(
os.path.join(curdir, 'data', 'scan_template.fits'))
lchdulist = fits.open(template)
datahdu = lchdulist['DATA TABLE']
temphdu = lchdulist['ANTENNA TEMP TABLE']
lchdulist[0].header['SOURCE'] = "Dummy"
lchdulist[0].header['ANTENNA'] = "SRT"
lchdulist[0].header['HIERARCH RIGHTASCENSION'] = np.radians(src_ra)
lchdulist[0].header['HIERARCH DECLINATION'] = np.radians(src_dec)
if scan_type is not None:
lchdulist[0].header['HIERARCH SubScanType'] = scan_type
data_table_data = Table(datahdu.data)
obstimes = Time((times / 86400 + 57000) * u.day, format='mjd', scale='utc')
coords = SkyCoord(ra,
dec,
unit=u.degree,
location=locations['srt'],
obstime=obstimes)
altaz = coords.altaz
el = altaz.alt.rad
az = altaz.az.rad
newtable = Table(
names=['time', 'raj2000', 'decj2000', "el", "az"],
data=[obstimes.value,
np.radians(ra),
np.radians(dec), el, az])
for ch in channels.keys():
newtable[ch] = channels[ch]
if other_columns is None:
other_columns = {}
for col in other_columns.keys():
newtable[col] = other_columns[col]
data_table_data = vstack([data_table_data, newtable])
nrows = len(data_table_data)
hdu = fits.BinTableHDU.from_columns(datahdu.data.columns, nrows=nrows)
for colname in datahdu.data.columns.names:
hdu.data[colname][:] = data_table_data[colname]
datahdu.data = hdu.data
temptable = Table()
for ch in channels.keys():
temptable[ch] = newtable[ch] * counts_to_K[ch]
thdu = fits.BinTableHDU.from_columns(temphdu.data.columns, nrows=nrows)
for colname in temphdu.data.columns.names:
thdu.data[colname][:] = temptable[colname]
temphdu.data = thdu.data
lchdulist[0].header['SOURCE'] = srcname
lchdulist.writeto(filename, overwrite=True)
lchdulist.close()
idx, d2d, _ = match_coordinates_sky(xmmcoord, bestmfcoord)
mask = d2d <= 5 * u.arcsec #make sure match is within 5 arcsec (like in topcat)
idx = idx[mask]
xmmmf = bestmf[idx]
## match with chandra
print('Matching Chandra')
chan = Table.read('UDS_catalogues/chandra_catalogue.fits')
chan['RA'].unit = u.deg
chan['Dec'].unit = u.deg
chancoord = SkyCoord(chan['RA'], chan['Dec'])
idx, d2d, _ = match_coordinates_sky(chancoord, bestmfcoord)
mask = d2d <= 1 * u.arcsec #make sure match is within 1 arcsec (like in topcat)
idx = idx[mask]
chanmf = bestmf[idx]
# combine chandra and xmm
print('Joining xray table')
xraymf = vstack([chanmf, xmmmf])
#%%
# boolean whether a source is seen in x-rays
xray = np.isin(bestmf['NUMBER_05B'], xraymf['NUMBER_05B'])
xraycol = Column(xray, 'X-ray')
bestmf.add_column(xraycol)
#%% Save the tables
extra = '_extra_clean'
semcom.write('mag_flux_tables/J/mag_flux_table_J' + extra + '.fits')
bestmf.write('mag_flux_tables/J/mag_flux_table_best_J' + extra + '.fits')
starsmf.write('mag_flux_tables/J/stars_mag_flux_table_J' + extra + '.fits')
xraymf.write('mag_flux_tables/J/xray_mag_flux_table_best_J' + extra + '.fits')
#~ plt.show()
# =============================================================================
# Apartado B
# =============================================================================
d10 = ascii.read('../ej1b/L_10_Tvar.dat')
d20 = ascii.read('../ej1b/L_20_Tvar.dat')
d40 = ascii.read('../ej1b/L_40_Tvar.dat')
dneg10 = ascii.read('../neg_modL_10_Tvar.dat')
dneg20 = ascii.read('../neg_modL_20_Tvar.dat')
dneg40 = ascii.read('../neg_modL_40_Tvar.dat')
sd10 = vstack([d10, dneg10])
sd20 = vstack([d20, dneg20])
sd40 = vstack([d40, dneg40])
plt.plot(sd10['T'], sd10['cv'], '*', label=r'$10\times10$')
plt.plot(sd20['T'], sd20['cv'], 'o', label=r'$20\times20$')
plt.plot(sd40['T'], sd40['cv'], '.', label=r'$40\times40$')
plt.xlabel(r'$T/T_c$')
plt.ylabel(r'$C_v$')
plt.xlim(0.3, 2.8)
plt.ylim(0., 6)
plt.legend(loc='best')
plt.savefig('cv.png')
plt.close()
plt.plot(sd10['T'], sd10['e'], '*', label=r'$10\times10$')
while True:
startra, startdec = ra, dec
tcopy = lt
tcopy['dist'] = np.sqrt((np.cos(dec * np.pi / 180.0) *
(tcopy['RA'] - ra))**2.0 +
(tcopy['DEC'] - dec)**2.0) * 3600.0
tcopy = tcopy[tcopy['dist'] < 180]
print 'Iter', iter, 'found', len(tcopy), 'neighbours'
# make sure the original source is in there
for nr in tcopy:
if sourcename == nr['Source_Name']:
break
else:
if 'Maj' in r.columns:
tcopy = vstack((tcopy, r))
ra = np.mean(tcopy['RA'])
dec = np.mean(tcopy['DEC'])
if startra == ra and startdec == dec:
break
iter += 1
if iter == 10:
break
# now find the bounding box of the resulting collection
ra, dec, size = find_bbox(tcopy)
if np.isnan(size):
ra = r['RA']
outfile = dirmain + '/' + dirout + '/' + fname
if not os.path.isdir(dirmain + '/' + dirout):
os.makedirs(dirmain + '/' + dirout)
tot_data = None
f = h5py.File(fi, 'r')
print('try loading', fi, len(f.keys()))
n_lc = 0
for i, keyb in enumerate(f.keys()):
tab = Table.read(fi, path=keyb)
#print(i,tab)
n_lc += len(tab)
if tot_data is None:
tot_data = tab
else:
tot_data = vstack([tot_data, tab])
print('number of lc', n_lc)
#print(tot_data.colnames)
"""
for name in tot_data.colnames:
if 'Cov' in name:
print name
"""
param_names = ['x0', 'x1', 'c']
outvars = [
'mb_recalc', 'salt2.CovColormb', 'salt2.CovX1mb', 'salt2.CovX0mb',
'salt2.Covmbmb'
]
snutils = SN_Utils()
def from_mpc(cls, targetids, id_type=None, target_type=None, **kwargs):
"""Load latest orbital elements from the
`Minor Planet Center <http://minorplanetcenter.net>`_.
Parameters
----------
targetids : str or iterable of str
Target identifier, resolvable by the Minor Planet
Ephemeris Service. If multiple targetids are provided in a list,
the same format (number, name, or designation) must be used.
id_type : str, optional
``'name'``, ``'number'``, ``'designation'``, or ``None`` to
indicate
type of identifiers provided in ``targetids``. If ``None``,
automatic identification is attempted using
`~sbpy.data.names`. Default: ``None``
target_type : str, optional
``'asteroid'``, ``'comet'``, or ``None`` to indicate
target type. If ``None``, automatic identification is
attempted using
`~sbpy.data.names`. Default: ``None``
**kwargs : optional
Additional keyword arguments are passed to
`~astroquery.mpc.MPC.query_object`
Returns
-------
`~Orbit` object
The resulting object will be populated with most columns as
defined in
`~astroquery.mpc.query_object`; refer
to that document on information on how to modify the list
of queried parameters.
Examples
--------
>>> from sbpy.data import Orbit
>>> orb = Orbit.from_mpc('Ceres') # doctest: +REMOTE_DATA
>>> orb # doctest: +SKIP
<QTable length=1>
absmag Q arc w ... a Tj moid_uranus moid_venus
mag AU d deg ... AU AU AU
float64 float64 float64 float64 ... float64 float64 float64 float64
------- ------- ------- -------- ... --------- ------- ----------- ----------
3.34 2.98 79653.0 73.59764 ... 2.7691652 3.3 15.6642 1.84632
"""
from ..data import Names
# if targetids is a list, run separate Horizons queries and append
if not isinstance(targetids, (list, ndarray, tuple)):
targetids = [targetids]
for targetid in targetids:
if target_type is None:
target_type = Names.asteroid_or_comet(targetid)
if id_type is None:
if target_type == 'asteroid':
ident = Names.parse_asteroid(targetid)
elif target_type == 'comet':
ident = Names.parse_comet(targetid)
if 'name' in ident:
id_type = 'name'
elif 'desig' in ident:
id_type = 'designation'
elif 'number' in ident:
id_type = 'number'
# append ephemerides table for each targetid
all_elem = None
for targetid in targetids:
# get elements
kwargs[id_type] = targetid
e = MPC.query_object(target_type, **kwargs)
# parse results from MPC.query_object
results = {}
for key, val in e[0].items():
# skip if key not in conf.mpc_orbit_fields
if key not in conf.mpc_orbit_fields:
continue
fieldname, fieldunit = conf.mpc_orbit_fields[key]
# try to convert to float
try:
val = float(val)
except (ValueError, TypeError):
pass
if fieldname == 'mpc_orbit_type':
results[fieldname] = [{
0: 'Unclassified',
1: 'Atira',
2: 'Aten',
3: 'Apollo',
4: 'Amor',
5: 'Mars Crosser',
6: 'Hungaria',
7: 'Phocaeas',
8: 'Hilda',
9: 'Jupiter Trojan',
10: 'Distant Object'
}[int(val)]]
elif fieldunit is None:
results[fieldname] = [val]
elif fieldunit == 'time_jd_utc':
results[fieldname] = Time([val], scale='utc', format='jd')
else:
if val is None:
continue
results[fieldname] = [val] * u.Unit(fieldunit)
if all_elem is None:
all_elem = QTable(results)
else:
all_elem = vstack([all_elem, QTable(results)])
return cls.from_table(all_elem)
def merge(self,
new_ll,
thresh=None,
loggf_thresh=None,
raise_exception=True,
skip_exactly_equal_lines=False,
skip_equal_loggf=False,
override_current=False,
in_place=True,
add_new_lines=True,
ignore_conflicts=False):
"""
new_ll:
new LineList object to merge into this one
thresh:
threshold for wavelength check when matching lines
Defaults to self.default_thresh (0.1)
loggf_thresh:
threshold for loggf check when finding identical lines
Defaults to self.default_loggf_thresh (0.01)
raise_exception:
If True (default), finds all the conflicts and raises LineListConflict
Note: if in_place == True, then it merges new lines BEFORE raising the error
If False, uses self.pick_best_line() to pick a line to overwrite
skip_exactly_equal_lines:
If True, skips lines that have equal hashes during the merge
If False (default), raises exception for duplicate lines
skip_equal_loggf:
If True, skips lines that are almost exactly equal during the merge
If False (default), raises exception for duplicate lines
override_current:
If True, uses new lines whenever duplicate lines are found.
If False (default), keep current lines whenever duplicate lines are found.
Ignored if raise_exception == True
in_place:
If True (default), merge new lines into this object. It will do so BEFORE throwing any LineListConflict exceptions!
If False, return a new LineList
add_new_lines:
If True (default), add new lines when merging.
If False, do not add new lines. This is to replace lines from a list without adding them.
ignore_conflicts:
If True, merge the linelists without checking for conflicts
If False (default), check for conflicts during merge
"""
if thresh == None: thresh = self.default_thresh
if loggf_thresh == None: loggf_thresh = self.default_loggf_thresh
if len(self) == 0:
if not in_place:
return new_ll.copy()
else:
n_cols = len(new_ll.colnames)
names = new_ll.colnames
dtype = [None] * n_cols
self._init_indices = self._init_indices and new_ll._copy_indices
self._init_from_table(new_ll, names, dtype, n_cols, True)
return None
if ignore_conflicts:
if self.verbose:
print("Ignoring conflicts: adding {} lines".format(
len(new_ll)))
if not in_place:
return table.vstack([self, new_ll])
else:
#combined = table.vstack([self, new_ll])
#names = combined.colnames
#dtype = [None] * n_cols
#self._init_indices = self._init_indices and combined._copy_indices
#self._init_from_table(combined, names, dtype, n_cols, True)
#return None
raise NotImplementedError
num_in_list = 0
num_with_multiple_conflicts = 0
lines_to_add = []
for j, new_line in enumerate(new_ll):
index = self.find_match(new_line, thresh)
if index == -1: # New Line
lines_to_add.append(new_line)
elif raise_exception: # Record all conflicts later
pass
else: # use self.pick_best_line to find best line
if index < -1:
num_with_multiple_conflicts += 1
index = self.pick_best_line(new_line, thresh)
# index < 0 is the convention that you should just skip the line rather than overwriting
if index < 0: continue
if override_current:
self[index] = new_line
elif index >= 0:
num_in_list += 1
if override_current:
self[index] = new_line
num_lines_added = len(lines_to_add)
if add_new_lines and len(lines_to_add) > 0:
if in_place:
for line in lines_to_add:
self.add_row(line)
else:
new_lines = Table(rows=lines_to_add,
names=lines_to_add[0].colnames)
old_lines = self.copy()
# During the vstack creates an empty LineList and warns
new_data = table.vstack([old_lines, new_lines])
else:
if not in_place:
new_data = self.copy()
# Note: if in_place == True, then it merges new lines BEFORE raising the exception
if raise_exception:
conflicts1, conflicts2 = self.identify_conflicts(
self,
new_ll,
skip_exactly_equal_lines=skip_exactly_equal_lines,
skip_equal_loggf=skip_equal_loggf,
dwl_thresh=thresh,
dgf_thresh=loggf_thresh)
if len(conflicts1) > 0:
raise LineListConflict(conflicts1, conflicts2)
if self.verbose:
print("Num lines added: {}".format(num_lines_added))
print("Num lines {}: {}".format(
'replaced' if override_current else 'ignored', num_in_list))
print("Num lines with multiple matches: {}".format(
num_with_multiple_conflicts))
if not in_place:
return LineList(new_data)
else:
return None
def nstar(self, image, star_groups):
"""
Fit, as appropriate, a compound or single model to the given
``star_groups``. Groups are fitted sequentially from the
smallest to the biggest. In each iteration, ``image`` is
subtracted by the previous fitted group.
Parameters
----------
image : numpy.ndarray
Background-subtracted image.
star_groups : `~astropy.table.Table`
This table must contain the following columns: ``id``,
``group_id``, ``x_0``, ``y_0``, ``flux_0``. ``x_0`` and
``y_0`` are initial estimates of the centroids and
``flux_0`` is an initial estimate of the flux. Additionally,
columns named as ``<param_name>_0`` are required if any other
parameter in the psf model is free (i.e., the ``fixed``
attribute of that parameter is ``False``).
Returns
-------
result_tab : `~astropy.table.Table`
Astropy table that contains photometry results.
image : numpy.ndarray
Residual image.
"""
result_tab = Table()
for param_tab_name in self._pars_to_output.keys():
result_tab.add_column(Column(name=param_tab_name))
y, x = np.indices(image.shape)
star_groups = star_groups.group_by('group_id')
for n in range(len(star_groups.groups)):
group_psf = get_grouped_psf_model(self.psf_model, star_groups.groups[n],
self._pars_to_set)
usepixel = np.zeros_like(image, dtype=np.bool)
for row in star_groups.groups[n]:
usepixel[overlap_slices(large_array_shape=image.shape,
small_array_shape=self.fitshape,
position=(row['y_0'], row['x_0']),
mode='trim')[0]] = True
fit_model = self.fitter(group_psf, x[usepixel], y[usepixel],
image[usepixel])
param_table = self._model_params2table(fit_model,
len(star_groups.groups[n]))
result_tab = vstack([result_tab, param_table])
try:
from astropy.nddata.utils import NoOverlapError
except ImportError:
raise ImportError("astropy 1.1 or greater is required in "
"order to use this class.")
# do not subtract if the fitting did not go well
try:
image = subtract_psf(image, self.psf_model, param_table,
subshape=self.fitshape)
except NoOverlapError:
pass
return result_tab, image
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
setup_envs()
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / args.n_fibers
# Read fibermapfile to get object type, night and expid
fibermap, objtype, night, expid = get_fibermap(args.fibermap,
log=log,
nspec=args.nspec)
# Initialize the spectral simulator
log.info("Initializing SpecSim with config {}".format(args.config))
lvmparams = load_lvmparams(config=args.config, telescope=args.telescope)
qsim = get_simulator(args.config, num_fibers=1, params=lvmparams)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = lvmsim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
# - TODO: do we need quickgen to support arcs? For full pipeline
# - arcs are used to measure PSF but aren't extracted except for
# - debugging.
# - TODO: if we do need arcs, this needs to be redone.
# - conversion from phot to flux doesn't include throughput,
# - and arc lines are rebinned to nearest 0.2 A.
# Create full wavelength and flux arrays for arc exposure
wave_b = np.array(simspec.wave['b'])
wave_r = np.array(simspec.wave['r'])
wave_z = np.array(simspec.wave['z'])
phot_b = np.array(simspec.phot['b'][0])
phot_r = np.array(simspec.phot['r'][0])
phot_z = np.array(simspec.phot['z'][0])
sim_wave = np.concatenate((wave_b, wave_r, wave_z))
sim_phot = np.concatenate((phot_b, phot_r, phot_z))
wavelengths = np.arange(3533., 9913.1, 0.2)
phot = np.zeros(len(wavelengths))
for i in range(len(sim_wave)):
wavelength = sim_wave[i]
flux_index = np.argmin(abs(wavelength - wavelengths))
phot[flux_index] = sim_phot[i]
# Convert photons to flux: following specter conversion method
dw = np.gradient(wavelengths)
exptime = 5. # typical BOSS exposure time in s
fibarea = const.pi * (1.07e-2 /
2)**2 # cross-sectional fiber area in cm^2
hc = 1.e17 * const.h * const.c # convert to erg A
spectra = (hc * exptime * fibarea * dw * phot) / wavelengths
else:
wavelengths = simspec.wave['brz']
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = lvmsim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = lvmsim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = lvmsim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = lvmsim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
fstd = lvmsim.templates.FSTD(wave=wavelengths)
flux, tmpwave, meta1 = fstd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = lvmsim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = lvmsim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = lvmsim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# ---------- end simspec
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or lvmparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set site location
if args.location is not None:
qsim.observation.observatory = args.location
else:
qsim.observation.observatory = 'APO'
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = lvmparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = lvmparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse format used in lvmspec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
dw = np.gradient(simspec.wave[camera.name])
else:
sflux = np.empty((args.nspec, npix))
# - Check if input simspec is for a continuum flat lamp instead of science
# - This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
dw = np.gradient(simspec.wave[channel])
meanspec = resample_flux(
waves[channel], simspec.wave[channel],
np.average(simspec.phot[channel] / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) //
args.n_fibers + 1):
camera = channel + str(kk)
outfile = lvmspec.io.findfile('fiberflat', night, expid,
camera)
start = max(args.n_fibers * kk, args.nstart)
end = min(args.n_fibers * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = lvmspec.io.findfile("fiberflat", night, expid,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
scale = 1e-17
fluxunits = scale * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
# Before writing, convert from counts/bin to counts/A (as in Pixsim output)
# Quicksim Default:
# FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
# COUNTS_OBJ - object counts in 0.5 Ang bin
# COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than args.n_fibers spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // args.n_fibers + 1):
start = max(args.n_fibers * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(args.n_fibers * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = lvmspec.io.findfile("frame", night, expid,
camera)
frame_flux = nobj[start:end, armName[channel], :num_pixels] + \
nsky[start:end, armName[channel], :num_pixels] + \
frame_rand_noise[start:end, armName[channel], :num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
# required for slicing the resolution metric, resolusion matrix has (nspec, ndiag, wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
sh1 = frame_flux.shape[0]
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create lvmspec.Frame object
frame = Frame(waves[channel],
frame_flux,
frame_ivar,
resolution_data=resol,
spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor))
lvmspec.io.write_frame(framefileName, frame)
framefilePath = lvmspec.io.findfile("frame", night, expid,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = lvmspec.io.findfile("cframe", night, expid,
camera)
cframeFlux = cframe_observedflux[start:end, armName[channel], :num_pixels] + \
cframe_rand_noise[start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create lvmspec.Frame object
cframe = Frame(waves[channel],
cframeFlux,
cframeIvar,
resolution_data=resol,
spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor))
lvmspec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = lvmspec.io.findfile("cframe", night, expid,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = lvmspec.io.findfile("sky", night, expid, camera)
skyflux = nsky[start:end, armName[channel], :num_pixels] + \
sky_rand_noise[start:end, armName[channel], :num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create lvmspec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
lvmspec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = lvmspec.io.findfile("sky", night, expid, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = lvmspec.io.findfile("calib", night, expid,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
# - TODO: what should calibivar be?
# - For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
# mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = lvmspec.io.findfile("calib", night, expid,
camera)
log.info("Wrote file {}".format(calibfilePath))
def _do_photometry(self, param_tab, n_start=1):
"""
Helper function which performs the iterations of the photometry process.
Parameters
----------
param_names : list
Names of the columns which represent the initial guesses.
For example, ['x_0', 'y_0', 'flux_0'], for intial guesses on the
center positions and the flux.
n_start : int
Integer representing the start index of the iteration.
It is 1 if init_guesses are None, and 2 otherwise.
Returns
-------
output_table : `~astropy.table.Table` or None
Table with the photometry results, i.e., centroids and
fluxes estimations and the initial estimates used to start
the fitting process.
None is returned if no sources are found in ``image``.
"""
output_table = Table()
self._define_fit_param_names()
for (init_param_name, fit_param_name) in zip(self._pars_to_set.keys(),
self._pars_to_output.keys()):
output_table.add_column(Column(name=init_param_name))
output_table.add_column(Column(name=fit_param_name))
sources = self.finder(self._residual_image)
n = n_start
while(len(sources) > 0 and
(self.niters is None or n <= self.niters)):
apertures = CircularAperture((sources['xcentroid'],
sources['ycentroid']),
r=self.aperture_radius)
sources['aperture_flux'] = aperture_photometry(self._residual_image,
apertures)['aperture_sum']
init_guess_tab = Table(names=['id', 'x_0', 'y_0', 'flux_0'],
data=[sources['id'], sources['xcentroid'],
sources['ycentroid'],
sources['aperture_flux']])
for param_tab_name, param_name in self._pars_to_set.items():
if param_tab_name not in (['x_0', 'y_0', 'flux_0']):
init_guess_tab.add_column(Column(name=param_tab_name,
data=getattr(self.psf_model,
param_name)*np.ones(len(sources))))
star_groups = self.group_maker(init_guess_tab)
table, self._residual_image = super(IterativelySubtractedPSFPhotometry,
self).nstar(self._residual_image, star_groups)
star_groups = star_groups.group_by('group_id')
table = hstack([star_groups, table])
table['iter_detected'] = n*np.ones(table['x_fit'].shape, dtype=np.int32)
output_table = vstack([output_table, table])
sources = self.finder(self._residual_image)
n += 1
return output_table
def do_sdss(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
Mnames, Mradec = [], []
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mnames.append(name)
Mradec.append(
str(catalog.entries[name][FASTSTARS.RA][0]['value']) +
str(catalog.entries[name][FASTSTARS.DEC][0]['value']))
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
# We must step through the data to query it, >100 is too many
maxstep = 100 # stepsize
Nentries = len(Mradec)
roundindex = np.zeros(
0
) # the stepping round that this star's data was acquired in, needed to connect the obj_id in the query to the name of the star
for i in range(int(Nentries / maxstep) + 1):
result_tmp = SDSS.query_crossid(
c[maxstep * i:min(maxstep * (i + 1), Nentries)],
timeout=200.,
photoobj_fields=[
'u', 'err_u', 'g', 'err_g', 'r', 'err_r', 'i', 'err_i', 'z',
'err_z', 'MJD'
])
roundindex = np.concatenate(
[roundindex, i * np.ones(len(result_tmp['obj_id']))])
if i == 0:
result = result_tmp
else:
result = vstack([result, result_tmp])
flagsuccess = result['obj_id']
listfilter = ['u', 'g', 'r', 'i', 'z']
for i in range(len(flagsuccess)):
Mi = int(flagsuccess[i].strip('obj_')) + maxstep * int(roundindex[i])
name = Mnames[Mi]
source = catalog.entries[name].add_source(
bibcode='2015ApJS..219...12A')
for j in range(5):
catalog.entries[name].add_photometry(
time=str(result[i]['MJD']),
u_time='MJD',
telescope='SDSS',
band=listfilter[j],
magnitude=str(result[i][listfilter[j]]),
e_magnitude=str(result[i]['err_' + listfilter[j]]),
source=source)
catalog.journal_entries()
return
def get_simspec(simspecfile, log=None, nspec=None):
''' Get the simspec object
The simspec table holds the "truth" spectra and the intrinsic properties
of each object (redshift, noiseless photometry, [OII] flux, etc.).
(Input spectra to simulate with pixsim.)
http://desidatamodel.readthedocs.io/en/latest/DESI_SPECTRO_SIM/PIXPROD/NIGHT/simspec-EXPID.html
Parameters:
simspecfile (str):
The filename of the input simspec file
'''
minwave = 3533.
maxwave = 9913.1
stepwave = 0.2
scale = 1.e17
exptime = 5. # typical BOSS exposure time in s
if simspecfile:
if log:
log.info('Reading input file {}'.format(args.simspec))
# create SimSpec object
simspec = lvmsim.io.read_simspec(args.simspec)
# number of spectra to simulate from SimSpec
sim_nspec = simspec.nspec
# get the spectra and wavelengths arrays for different flavors
if simspec.flavor == 'arc':
# - TODO: do we need quickgen to support arcs? For full pipeline
# - arcs are used to measure PSF but aren't extracted except for
# - debugging.
# - TODO: if we do need arcs, this needs to be redone.
# - conversion from phot to flux doesn't include throughput,
# - and arc lines are rebinned to nearest 0.2 A.
# Create full wavelength and flux arrays for arc exposure
wave_b = np.array(simspec.wave['b'])
wave_r = np.array(simspec.wave['r'])
wave_z = np.array(simspec.wave['z'])
phot_b = np.array(simspec.phot['b'][0])
phot_r = np.array(simspec.phot['r'][0])
phot_z = np.array(simspec.phot['z'][0])
sim_wave = np.concatenate((wave_b, wave_r, wave_z))
sim_phot = np.concatenate((phot_b, phot_r, phot_z))
wavelengths = np.arange(minwave, maxwave, stepwave)
phot = np.zeros(len(wavelengths))
for i in range(len(sim_wave)):
wavelength = sim_wave[i]
flux_index = np.argmin(abs(wavelength - wavelengths))
phot[flux_index] = sim_phot[i]
# Convert photons to flux: following specter conversion method
dw = np.gradient(wavelengths)
fibarea = const.pi * (1.07e-2 /
2)**2 # cross-sectional fiber area in cm^2
hc = scale * const.h * const.c # convert to erg A
spectra = (hc * exptime * fibarea * dw * phot) / wavelengths
else:
wavelengths = simspec.wave['brz']
spectra = simspec.flux
# check there's enough spectra to simulate from what we ask for
if sim_nspec < nspec:
log.info("Only {} spectra in input file".format(sim_nspec))
nspec = sim_nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
if log:
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = lvmsim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = lvmsim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = lvmsim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = lvmsim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
fstd = lvmsim.templates.FSTD(wave=wavelengths)
flux, tmpwave, meta1 = fstd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = lvmsim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = lvmsim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = lvmsim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
raise RuntimeError('Unknown object type')
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
return spectra, wavelengths, nspec
Bd0.sort('MJD')
Vd0=Vd[Vd['DET']=='Y']
Vd0.sort('MJD')
Rd0=Rd[Rd['DET']=='Y']
Rd0.sort('MJD')
Id0=Id[Id['DET']=='Y']
Id0.sort('MJD')
gd0=gd[gd['DET']=='Y']
gd0.sort('MJD')
rd0=rd[rd['DET']=='Y']
rd0.sort('MJD')
iid0=iid[iid['DET']=='Y']
iid0.sort('MJD')
vsd=vstack([Bd0['MJD','MAG','MAGERR'], Vd0['MJD','MAG','MAGERR'],
Rd0['MJD','MAG','MAGERR'], Id0['MJD','MAG','MAGERR'],
gd0['MJD','MAG','MAGERR'], rd0['MJD','MAG','MAGERR'], iid0['MJD','MAG','MAGERR']])
vsd.write('SN2018kp_snpy.dat',format='ascii.commented_header', overwrite=True)
# edit file incuding filter names
# filter set check
snpy.fset.list_filters()
'''
APO
SDSS
'u_s': sloan u at APO
'g_s': sloan g at APO
'r_s': sloan r at APO
'i_s': sloan i at APO
'z_s': sloan z at APO
CTIO
def color_table(self,band1,band2,time_delays=None,magnifications=None,referenceImage='image_1',ignore_images=[]):
"""Takes the multiple images in self.images and combines
the data into a single color curve using defined
time delays and magnifications or best (quick) guesses.
Parameters
----------
band1: str
The first band for color curve
band2: str
The second band for color curve
time_delays : :class:`dict`
Dictionary with image names as keys and relative time
delays as values (e.g. {'image_1':0,'image_2':20})
magnifications : dict
Dictionary with image names as keys and relative
magnifications as values (e.g.
{'image_1':1,'image_2':1.1})
referenceImage: str
The image you want to be the reference (e.g. iamge_1, image_2, etc.)
ignore_images: list
List of images you do not want to include in the color curve.
Returns
-------
self: :class:`sntd.curve_io.curveDict`
"""
ignore_images=list(ignore_images) if not isinstance(ignore_images,(list,tuple)) else ignore_images
names=['time','image','zpsys']
dtype=[self.table.dtype[x] for x in names]
names=np.append(names,[band1+'-'+band2,band1+'-'+band2+'_err'])
dtype=np.append(dtype,[dtype[0],dtype[0]])
self.color.table=Table(names=names,dtype=dtype)
if not time_delays:
time_delays=_guess_time_delays(self,referenceImage) #TODO fix these guessing functions
if not magnifications:
magnifications=_guess_magnifications(self,referenceImage)
for im in [x for x in self.images.keys() if x not in ignore_images]:
temp2=copy(self.images[im].table[self.images[im].table['band']==band2])
temp1=copy(self.images[im].table[self.images[im].table['band']==band1])
temp1=temp1[temp1['flux']>0]
temp2=temp2[temp2['flux']>0]
temp1['flux']/=magnifications[im]
temp2['flux']/=magnifications[im]
temp1['time']-=time_delays[im]
temp2['time']-=time_delays[im]
temp2['mag']=-2.5*np.log10(temp2['flux'])+temp2['zp']
temp2['magerr']=1.0857*temp2['fluxerr']/temp2['flux']
temp1['mag']=-2.5*np.log10(temp1['flux'])+temp1['zp']
temp1['magerr']=1.0857*temp1['fluxerr']/temp1['flux']
temp1_remove=[i for i in range(len(temp1)) if temp1['time'][i] not in temp2['time']]
temp1.remove_rows(temp1_remove)
temp2_remove=[i for i in range(len(temp2)) if temp2['time'][i] not in temp1['time']]
temp2.remove_rows(temp2_remove)
temp1['magerr']=np.sqrt(temp2['magerr']**2+temp1['magerr']**2)
temp1['mag']-=temp2['mag']
temp1.rename_column('mag',band1+'-'+band2)
temp1.rename_column('magerr',band1+'-'+band2+'_err')
to_remove=[x for x in temp1.colnames if x not in names]
temp1.remove_columns(to_remove)
self.color.table=vstack([self.color.table,copy(temp1)])
self.color.table.sort('time')
return(self)
def voronoi_binning(image,
obj_name,
targetSN=50,
largest_bin=5,
smallest_bin=0,
minimumSN=7,
quiet=True,
plot=True):
"""
Function to bin an image using the Voronoi binning method by Cappellari & Copin (2003)
Input as 'image' the target with the filter where S/N is highest
"""
import numpy as np
import astropy.io.fits as pyfits
from grizli import utils
from grizli import prep
import matplotlib.pyplot as plt
from astropy.table import vstack
im = pyfits.open(image)
sci = np.cast[np.float32](im['SCI'].data)
sh = sci.shape
ivar = np.cast[np.float32](im['WHT'].data)
var = 1 / ivar
orig_mask = (ivar > 0)
sci[~orig_mask] = 0
var[~orig_mask] = 0
orig_var = var * 1
# Simple background
bkg = np.median(sci[orig_mask])
sci -= bkg * orig_mask
cps = sci * im[0].header['EXPTIME']
shot_err = np.sqrt(np.maximum(cps, 4)) / im[0].header['EXPTIME']
var2 = var + shot_err**2
var = var2
full_bin_seg = np.zeros(sh, dtype=np.int) - 1
yp, xp = np.indices(sci.shape)
xpf = xp.flatten()
ypf = yp.flatten()
# Initialize mask
mask = orig_mask & True
bin_min = 1
full_image = sci * 0.
full_err = sci * 0.
idx = np.arange(sci.size, dtype=np.int)
full_image = full_image.flatten()
full_err = full_err.flatten()
# id, bin, xmin, xmax, ymin, ymax, npix
full_bin_data = []
SKIP_LAST = False
bin_iter = 0
bin_factor = largest_bin
NO_NEWLINE = '\x1b[1A\x1b[1M'
for bin_iter, bin_factor in enumerate(range(largest_bin + 1)[::-1]):
bin = 2**bin_factor
if bin_factor < smallest_bin:
break
if (bin_factor == 0) & SKIP_LAST:
continue
ypb = yp[mask] // bin
xpb = xp[mask] // bin
if bin_factor > 0:
binned_sci = np.zeros((sh[0] // bin + 1, sh[1] // bin + 1))
binned_npix = binned_sci * 0
binned_var = binned_sci * 0
ypi, xpi = np.indices(binned_sci.shape)
# Only consider unmasked bins
ij = np.unique(xpb + sh[0] // bin * ypb)
yarr = ij // (sh[0] // bin)
xarr = ij - (sh[0] // bin) * yarr
for xi, yi in zip(xarr, yarr):
if not quiet:
print(NO_NEWLINE + '{0} {1}/{2} {3}/{4}'.format(
bin_factor, xi, xarr.max(), yi, yarr.max()))
slx = slice(xi * bin, xi * bin + bin)
sly = slice(yi * bin, yi * bin + bin)
mslice = mask[sly, slx]
# Straight average
binned_sci[yi, xi] = sci[sly, slx][mslice].sum()
binned_npix[yi, xi] = mslice.sum()
binned_var[yi, xi] = var[sly, slx][mslice].sum()
binned_err = np.sqrt(binned_var) / binned_npix
binned_avg = binned_sci / binned_npix
mask_i = (binned_npix > 0) & (binned_avg / binned_err > minimumSN)
xpi = xpi[mask_i]
ypi = ypi[mask_i]
binned_avg = binned_avg[mask_i]
binned_err = binned_err[mask_i]
binned_npix = binned_npix[mask_i]
else:
mask_i = mask_j
xpi = xp[mask]
ypi = yp[mask]
binned_avg = sci[mask]
binned_err = np.sqrt(var)[mask]
binned_npix = mask[mask] * 1
if True:
# Mask pixels in that don't satisfy S/N cutoff as they are
# unreliable for vorbin
clip_mask = mask < 0
for xi, yi in zip(xpi, ypi):
slx = slice(xi * bin, xi * bin + bin)
sly = slice(yi * bin, yi * bin + bin)
clip_mask[sly, slx] = True
mask &= clip_mask
# Identify blobs (usually the main central galaxies) and
# only consider blobs larger than 20% of the largest blob
if bin_factor == largest_bin:
label_image = label(mask)
label_ids = np.unique(label_image)[1:]
label_sizes = np.array([(label_image == id_i).sum()
for id_i in label_ids])
keep_ids = label_ids[label_sizes > 0.2 * label_sizes.max()]
keep_mask = mask < -1
for i in keep_ids:
keep_mask |= label_image == i
mask &= keep_mask
# In binned_coords
in_blob = keep_mask[ypi * bin, xpi * bin]
msg = 'Drop {0} bins not in main blob'
print(msg.format((~in_blob).sum()))
xpi = xpi[in_blob]
ypi = ypi[in_blob]
binned_avg = binned_avg[in_blob]
binned_err = binned_err[in_blob]
binned_npix = binned_npix[in_blob]
ypb = yp[mask] // bin
xpb = xp[mask] // bin
print('Run voronoi_2d_binning, bin_factor={0}'.format(bin_factor))
res = voronoi_2d_binning(xpi,
ypi,
binned_avg,
binned_err,
targetSN,
quiet=True,
plot=False,
pixelsize=0.1 * bin,
cvt=True,
wvt=True)
binNum, xBin, yBin, xBar, yBar, sn, nPixels, scale = res
# Put Voronoi bins with nPixels > 1 back in the original image
NBINS = len(nPixels)
if (bin_factor == smallest_bin) & (smallest_bin > 0):
valid_bins = nPixels > 0
else:
valid_bins = nPixels > 1
large_bin_ids = np.arange(NBINS)[valid_bins]
# Put bin in original 2D array and store info
for b0, bin_id in enumerate(large_bin_ids):
m_i = binNum == bin_id
xi_bin = xpi[m_i]
yi_bin = ypi[m_i]
for xi, yi in zip(xi_bin, yi_bin):
slx = slice(xi * bin, xi * bin + bin)
sly = slice(yi * bin, yi * bin + bin)
mslice = mask[sly, slx]
full_bin_seg[sly, slx][mslice] = b0 + bin_min
# Bin properties
id_i = b0 + bin_min
xmin = xi_bin.min() * bin - 1
xmax = xi_bin.max() * bin + 1
ymin = yi_bin.min() * bin - 1
ymax = yi_bin.max() * bin + 1
npix = m_i.sum() * bin**2
bin_data_i = [id_i, bin, xmin, xmax, ymin, ymax, npix]
full_bin_data.append(bin_data_i)
# Update the mask
not_in_a_bin = full_bin_seg == -1
mask &= not_in_a_bin
bin_min = full_bin_data[-1][0] + 1
if not quiet:
print('\n\n\n\n\n bin_factor: {0}, bin_min: {1}\n\n\n\n'.format(
bin_factor, bin_min))
## Bin information
bin_data = np.array(full_bin_data)
# bin_data_i = [id_i, bin, xmin, xmax, ymin, ymax, npix]
tab = utils.GTable()
for i, c in enumerate(
['id', 'bin', 'xmin', 'xmax', 'ymin', 'ymax', 'npix']):
tab[c] = bin_data[:, i]
if 'min' in c:
tab[c] -= tab['bin']
elif 'max' in c:
tab[c] += tab['bin']
# Make a table for the individual pixels
if mask.sum() > 0:
single_table = single_pixel_table(mask, start_id=1 + tab['id'].max())
full_bin_seg[mask] = single_table['id']
tab = vstack([tab, single_table])
tab['flux'], tab['err'], tab['area'] = prep.get_seg_iso_flux(
sci, full_bin_seg, tab, err=np.sqrt(var))
binned_flux = prep.get_seg_iso_flux(sci,
full_bin_seg,
tab,
fill=tab['flux'] / tab['area'])
binned_err = prep.get_seg_iso_flux(sci,
full_bin_seg,
tab,
fill=tab['err'] / tab['area'])
binned_area = prep.get_seg_iso_flux(sci,
full_bin_seg,
tab,
fill=tab['area'])
binned_bin = prep.get_seg_iso_flux(sci, full_bin_seg, tab, fill=tab['bin'])
binned_flux[mask] = sci[mask]
binned_err[mask] = np.sqrt(var)[mask]
binned_area[mask] = 1
binned_bin[mask] = 1
if plot:
plt.figure(figsize=(12, 12))
plt.imshow(binned_flux, vmin=-0.5, vmax=2)
# Save output into image fits file
primary_extn = pyfits.PrimaryHDU()
sci_extn = pyfits.ImageHDU(data=binned_flux.astype(np.float32), name='SCI')
err_extn = pyfits.ImageHDU(data=binned_err.astype(np.float32), name='ERR')
hdul = pyfits.HDUList([primary_extn, sci_extn, err_extn])
for ext in [0, 1]:
for k in im[ext].header:
if k not in hdul[ext].header:
if k in ['COMMENT', 'HISTORY', '']:
continue
hdul[ext].header[k] = im[ext].header[k]
hdul.writeto('binned_{0}_image.fits'.format(obj_name),
output_verify='fix',
overwrite=True)
tab.write('binned_{0}_table.fits'.format(obj_name), overwrite=True)
pyfits.writeto('binned_{0}_seg.fits'.format(obj_name),
data=full_bin_seg,
overwrite=True)
pyfits.writeto('binned_{0}_mask.fits'.format(obj_name),
data=mask * 1,
overwrite=True)
return tab, full_bin_seg, mask
def generate_full_table(self, gen_small_table=False, load=None):
"""
Generates the full table from the gaia, tmass, and allwise queries.
Xmatch Gaia and tmass
Xmatch Gaia with allwise
Xmatch tmass with allwise
Xmatch (Gaia x tmass) with allwise
We then append:
sources in all three catalogs
sources in two of the catalogs
sources in just one catalog
Arguments:
gen_small_table [bool]: Whether to only use a subset of available columns. False by default.
load [string]: Directory from which to load an already generated full_table. None by default.
"""
out("\nGenerating full table...")
out("Extracting specified data columns...")
if gen_small_table:
# Gaia columns in small_table
gaia_cat_table = self.gaia.table["designation", "ra", "dec",
"ra_error", "dec_error",
"parallax", "parallax_error",
"pmra", "pmra_error", "pmdec",
"pmdec_error", "phot_g_mean_mag",
"bp_rp", "bp_g", "g_rp",
"phot_g_n_obs",
"phot_g_mean_flux_over_error",
"phot_g_mean_flux_error",
"phot_g_mean_flux",
"phot_bp_mean_mag",
"phot_rp_mean_mag"]
gaia_cat = CatalogTable(["gaia"], gaia_cat_table)
# 2Mass columns in small_table
tmass_cat_table = self.tmass.table["designation", "ra", "dec",
"err_maj", "err_min", "j_m",
"h_m", "k_m", "j_cmsig",
"h_cmsig", "k_cmsig", "k_snr"]
tmass_cat = CatalogTable(["2mass"], tmass_cat_table)
# Allwise columns in small_table
allwise_cat_table = self.allwise.table["designation", "ra", "dec",
"sigra", "sigdec", "w1mpro",
"w2mpro", "w3mpro",
"w4mpro", "w1sigmpro",
"w2sigmpro", "w3sigmpro",
"w4sigmpro", "w4snr"]
allwise_cat = CatalogTable(["allwise"], allwise_cat_table)
else:
gaia_cat = self.gaia
tmass_cat = self.tmass
allwise_cat = self.allwise
# rename columns to avoid name collisions later
out("Renaming columns...")
gaia_cat.table.rename_column("designation", "gaia_designation")
gaia_cat.table.rename_column("ra", "gaia_ra")
gaia_cat.table.rename_column("dec", "gaia_dec")
tmass_cat.table.rename_column("designation", "2mass_designation")
tmass_cat.table.rename_column("ra", "2mass_ra")
tmass_cat.table.rename_column("dec", "2mass_dec")
allwise_cat.table.rename_column("designation", "allwise_designation")
allwise_cat.table.rename_column("ra", "allwise_ra")
allwise_cat.table.rename_column("dec", "allwise_dec")
out("Crossmatching Gaia with 2MASS...")
gaia_X_tmass = self.merge_tables(gaia_cat, tmass_cat)
out("Crossmatching Gaia with AllWISE...")
gaia_X_allwise = self.merge_tables(gaia_cat, allwise_cat)
out("Crossmatching 2MASS with AllWISE...")
tmass_X_allwise = self.merge_tables(tmass_cat, allwise_cat)
out("Crossmatching all three catalogs...")
gaia_X_tmass_X_allwise = self.merge_tables(gaia_X_tmass, allwise_cat)
info_out(
str(len(gaia_X_tmass_X_allwise.table['gaia_designation'])) +
" sources in all three catalogs.")
full_table = gaia_X_tmass_X_allwise
out("Adding objects that do not appear in all catalogs...")
# sources in gaia_X_tmass that are not in gaia_X_tmass_X_allwise
# i.e. sources in Gaia and Tmass but not Allwise
diff1 = self.table_difference(gaia_X_tmass, full_table,
"gaia_designation", ["allwise"],
gaia_cat, tmass_cat, allwise_cat)
info_out(
str(len(diff1['gaia_designation'])) +
" sources in Gaia and 2MASS but not AllWISE.")
full_table.table = vstack([full_table.table, diff1])
# sources in gaia_X_allwise that are not in gaia_X_tmass_X_allwise
# i.e. sources in Gaia and Allwise but not Tmass.
diff2 = self.table_difference(gaia_X_allwise, full_table,
"gaia_designation", ["2mass"], gaia_cat,
tmass_cat, allwise_cat)
info_out(
str(len(diff2['gaia_designation'])) +
" sources in Gaia and AllWISE but not 2MASS.")
full_table.table = vstack([full_table.table, diff2])
# objects in tmass_X_allwise that are not in gaia_X_tmass_X_allwise,
# i.e. sources in tmass and allwise but not gaia.
diff3 = self.table_difference(tmass_X_allwise, full_table,
"allwise_designation", ["gaia"],
gaia_cat, tmass_cat, allwise_cat)
info_out(
str(len(diff3['allwise_designation'])) +
" sources in 2MASS and AllWISE but not Gaia.")
full_table.table = vstack([full_table.table, diff3])
# objects in gaia but not yet in full_table
diff4 = self.table_difference(gaia_cat, full_table, "gaia_designation",
["2mass", "allwise"], gaia_cat,
tmass_cat, allwise_cat)
info_out(
str(len(diff4['gaia_designation'])) + " sources in Gaia only.")
full_table.table = vstack([full_table.table, diff4])
# objects in tmass but not yet in full_table
diff5 = self.table_difference(tmass_cat, full_table,
"2mass_designation", ["gaia", "allwise"],
gaia_cat, tmass_cat, allwise_cat)
info_out(
str(len(diff5['2mass_designation'])) + " sources in 2mass only.")
full_table.table = vstack([full_table.table, diff5])
# objects in allwise but not yet in full_table
diff6 = self.table_difference(allwise_cat, full_table,
"allwise_designation", ["gaia", "2mass"],
gaia_cat, tmass_cat, allwise_cat)
info_out(
str(len(diff6['allwise_designation'])) +
" sources in AllWISE only.")
full_table.table = vstack([full_table.table, diff6])
out("Computing user-defined columns...")
out("Variability...")
v = np.sqrt(full_table.table['phot_g_n_obs'] /
full_table.table['phot_g_mean_flux_over_error'])
v.name = "variability"
v.unit = "(n_obs / mag)^0.5"
full_table.table.add_column(v)
out("Radial distance...")
d = 1000 / full_table.table['parallax']
d.name = "radial_distance"
d.unit = "pc"
full_table.table.add_column(d)
info_out("Full table generated.")
fname = tpath + "/full_table.dat"
full_table.table.write(fname, format='ascii.ecsv')
return (full_table)
def phot(images,
positions=None,
aperture_radius=6.0,
annulus=8.0,
dannulus=10.0,
method='exact',
bg_method='median',
bg_box_size=50,
gain=1.1,
output=None):
"""
Basic circular aperture photometry of given images.
Parameters
----------
images : generator or list of 'ccdproc.CCDData'
Images to be combined.
positions : list
The positions should be either a single tuple of (x, y), a list of (x, y) tuples, or
an array with shape Nx2, where N is the number of positions.
Default is 'None'.
aperture_radius : list or float
The aperture radius of a source.
Default is '3.0'.
annulus : float
The circular inner annulus aperture radius of a source.
Default is '6.0'.
dannulus : float
The circular outer annulus aperture radius of a source.
Default is '8.0'.
method : {'exact', 'center', 'subpixel'}, optional
The method used to determine the overlap of the aperture on the
pixel grid. Not all options are available for all aperture
types. Note that the more precise methods are generally slower.
The following methods are available:
* ``'exact'`` (default):
The the exact fractional overlap of the aperture and
each pixel is calculated. The returned mask will
contain values between 0 and 1.
* ``'center'``:
A pixel is considered to be entirely in or out of the
aperture depending on whether its center is in or out of
the aperture. The returned mask will contain values
only of 0 (out) and 1 (in).
* ``'subpixel'``:
A pixel is divided into subpixels (see the ``subpixels``
keyword), each of which are considered to be entirely in
or out of the aperture depending on whether its center
is in or out of the aperture. If ``subpixels=1``, this
method is equivalent to ``'center'``. The returned mask
will contain values between 0 and 1.
bg_method: {'mean', 'median'}, optional
The statistic used to calculate the background.
All measurements are sigma clipped.
bg_box_size: int
Selecting the box size requires some care by the user. The box size
should generally be larger than the typical size of sources in the
image, but small enough to encapsulate any background variations. For
best results, the box size should also be chosen so that the data are
covered by an integer number of boxes in both dimensions. More information:
https://github.com/Gabriel-p/photpy/blob/master/IRAF_compare/aperphot.py
gain: float
CCD gain value.
output : None or str
If it is None, function returns just Table.
If it is 'str', function creates photometry result file.
Returns
-------
'A table object or file'
Examples
--------
>>> from tuglib.io import FitsCollection
>>> from tuglib.analysis import phot
>>> location = '/Users/ykilic/Desktop/data/'
>>> images = FitsCollection(location, file_extension="fit")
>>> fits_images = images.ccds(IMAGETYP='object')
>>> phot(fits_images, positions=[(1400.65, 1545.78)], aperture_radius=10.0, annulus=12.0, dannulus=15)
"""
if not isinstance(images, (list, types.GeneratorType)):
raise TypeError("'images' should be a 'ccdproc.CCDData' object.")
if positions is not None:
if not isinstance(positions, (list, type(None))):
raise TypeError("'positions' should be 'list' or 'None' object.")
if not isinstance(aperture_radius, (list, float, int)):
raise TypeError("'radius' should be a 'list' or 'float' object.")
if not isinstance(annulus, (float, int)):
raise TypeError("'annulus' should be a 'int' or 'float' object.")
if not isinstance(dannulus, (float, int)):
raise TypeError("'dannulus' should be a 'int' or 'float' object.")
if not isinstance(method, str):
raise TypeError("'method' should be a 'str' object.")
if not isinstance(bg_method, str):
raise TypeError("'bg_method' should be a 'str' object.")
if not isinstance(bg_box_size, int):
raise TypeError("'bg_box_size' should be a 'int' or 'float' object.")
if not isinstance(gain, (float, int)):
raise TypeError("'gain' should be a 'float' object.")
if not isinstance(output, (type(None), type(str))):
raise TypeError("'output' should be 'None' or 'str' objects.")
if isinstance(images, types.GeneratorType):
try:
ccds = list(images)
except IndexError:
return None
else:
ccds = images
ccds_phot_list = list()
for ccd in ccds:
exptime = ccd.meta['EXPTIME']
box_xy = (bg_box_size, bg_box_size)
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = MedianBackground()
bkg = Background2D(ccd,
box_xy,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
ccd.uncertainty = StdDevUncertainty(
calc_total_error(ccd.data, bkg.background, gain))
aperture = CircularAperture(positions, r=aperture_radius)
annulus_aperture = CircularAnnulus(positions,
r_in=annulus,
r_out=dannulus)
annulus_masks = annulus_aperture.to_mask(method=method)
bkg_phot = []
for mask in annulus_masks:
annulus_data = mask.multiply(ccd)
annulus_data_1d = annulus_data[mask.data > 0]
mean_sigclip, median_sigclip, stddev_sigclip = sigma_clipped_stats(
annulus_data_1d)
if bg_method == "mean":
sigclip = mean_sigclip
elif bg_method == "median":
sigclip = median_sigclip
else:
sigclip = median_sigclip
bkg_phot.append(sigclip)
bkg_phot = np.array(sigclip)
phot_table = aperture_photometry(ccd, aperture)
phot_table['annulus_{}'.format(bg_method)] = bkg_phot * u.adu
phot_table['aper_bkg'] = bkg_phot * aperture.area * u.adu
phot_table['residual_aperture_sum'] = phot_table[
'aperture_sum'] - phot_table['aper_bkg']
phot_table['flux'] = phot_table['residual_aperture_sum'] / float(
exptime)
phot_table['flux'].info.format = '%.3f'
phot_table['JD'] = float(ccd.meta['JD'])
phot_table['JD'].info.format = '%.8f'
phot_table['JD'].unit = 'd'
phot_table['aperture_sum'].info.format = '%.3f'
phot_table['aperture_sum_err'].info.format = '%.3f'
phot_table['annulus_{}'.format(bg_method)].info.format = '%.3f'
phot_table['aper_bkg'].info.format = '%.3f'
phot_table['residual_aperture_sum'].info.format = '%.3f'
phot_table = flux2mag(phot_table)
phot_table['mag_err'] = 1.0857 * (
phot_table['aperture_sum_err'] /
phot_table['residual_aperture_sum']) * u.mag
phot_table['mag_err'].info.format = '%.3f'
ccds_phot_list.append(phot_table)
return vstack(ccds_phot_list)
import os
import glob
import numpy as np
from astropy.table import Table, vstack
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
t_all = []
for filename in glob.glob(os.path.join('tables', '*.fits')):
print(filename)
t = Table.read(filename)
sub = t[~np.isnan(t['altitude'])]['timestamp', 'callsign', 'longitude',
'latitude', 'altitude']
if len(sub) > 0:
t_all.append(sub)
t_all = vstack(t_all)
t_all.write('summary.fits', overwrite=True)
star1 = stmass_pc2(tab1['mass_ssp_sm'],
dist=dist.loc[gal]['caDistP3d'],
name='sigstar_sm')
avstar0 = stmass_pc2(tab0['mass_Avcor_ssp_rg'],
dist=dist.loc[gal]['caDistP3d'],
name='sigstar_Avcor_rg')
avstar1 = stmass_pc2(tab1['mass_Avcor_ssp_sm'],
dist=dist.loc[gal]['caDistP3d'],
name='sigstar_Avcor_sm')
tab0.add_column(star0)
tab1.add_column(star1)
tab0.add_column(avstar0)
tab1.add_column(avstar1)
if len(rglist) > 0:
rg_merge = vstack(rglist)
rg_merge.meta['date'] = datetime.today().strftime('%Y-%m-%d')
print(rg_merge.colnames)
print('There are', len(rg_merge), 'rows in native table')
if len(smlist) > 0:
sm_merge = vstack(smlist)
sm_merge.meta['date'] = datetime.today().strftime('%Y-%m-%d')
print(sm_merge.colnames)
print('There are', len(sm_merge), 'rows in smoothed table')
if (len(filelist) > 1):
outname = 'edge'
else:
outname = gal
if prod == prodtype[0]:
magl_lim = lambda mag, mag_lim : (mag < mag_lim)
selection = lambda t, mag_lim : area_ero(t['DEC'], t['g_lat'], t['g_lon']) & magl_lim( t['MAG'], mag_lim )
nl = lambda sel : len(sel.nonzero()[0])
#t = t1
s1 = selection(t1, 24.5)
print(nl(s1))
s2 = selection(t2, 22.0)
print(nl(s2))
s3 = selection(t3, 20.5)
print(nl(s3))
path_2_output = os.path.join(root_dir, 'S5_4MOST_RFIB215.fit')
t1_b = Table(t1[s1])
t2_b = Table(t2[s2])
t3_b = Table(t3[s3])
t1_b.remove_columns(['ra','dec','g_lat','g_lon','ecl_lat','ecl_lon','redshift_R','redshift_S','dL_cm','galactic_NH','galactic_ebv','galaxy_stellar_mass','galaxy_star_formation_rate','galaxy_LX_hard','is_quiescent','HALO_M200c','HALO_M500c','HALO_Mvir','HALO_Acc_Rate_1Tdyn','HALO_rs','HALO_rvir','HALO_vmax','Vrms','HALO_id','HALO_host_id','angular_distance_to_cluster_center_in_rvir','comoving_distance_to_cluster_in_rvir','redshift_R_distance_to_cluster','redshift_S_distance_to_cluster','galaxy_gr','galaxy_ri','galaxy_iz','richness','galaxy_mag_abs_r','galaxy_mag_r','sdss_g','sdss_r','sdss_i','sdss_z','sdss_g_err','sdss_r_err','sdss_i_err','sdss_z_err'])
t2_b.remove_columns(['ra','dec','g_lat','g_lon','ecl_lat','ecl_lon','redshift_R','redshift_S','dL_cm','galactic_NH','galactic_ebv','galaxy_stellar_mass','galaxy_star_formation_rate','galaxy_LX_hard','is_quiescent','HALO_M200c','HALO_M500c','HALO_Mvir','HALO_Acc_Rate_1Tdyn','HALO_rs','HALO_rvir','HALO_vmax','Vrms','HALO_id','HALO_host_id','angular_distance_to_cluster_center_in_rvir','comoving_distance_to_cluster_in_rvir','redshift_R_distance_to_cluster','redshift_S_distance_to_cluster','galaxy_gr','galaxy_ri','galaxy_iz','richness','galaxy_mag_abs_r','galaxy_mag_r','sdss_g','sdss_r','sdss_i','sdss_z','sdss_g_err','sdss_r_err','sdss_i_err','sdss_z_err'])
t3_b.remove_columns(['Z','Mstar','SFR','EBV','K_mag_abs','rtot','rfib','ug','gr','ri','iz','zy','yj','jh','hks','g_lat','g_lon'])
t3_c=Table(t3_b)
t3_c.remove_columns('NAME')
t3_c.add_column(Column(name='NAME', data=t3_b['NAME']), index=0)
test = vstack((t1_b, t2_b, t3_c))
test['MAG_TYPE'][:]="SDSS_r_AB"
test.write (path_2_output , overwrite=True)
def efosc2_pol_phot(folder_path, apermul, fwhm):
""" Script runs photometry for EFOSC2 polarimetry images """
# Read in four wave plate angle files and set up arrays for later
file_ang0 = os.path.join(folder_path, '0ang.fits')
file_ang225 = os.path.join(folder_path, '225ang.fits')
file_ang45 = os.path.join(folder_path, '45ang.fits')
file_ang675 = os.path.join(folder_path, '675ang.fits')
files = [file_ang0, file_ang225, file_ang45, file_ang675]
angle = ['0', '225', '45', '675']
ang_dec = ['0', '22.5', '45', '67.5']
label = [
'$0^{\circ}$ image', '$22.5^{\circ}$ image', '$45^{\circ}$ image',
'$67.5^{\circ}$ image'
]
# Set up array to store the number of sources per half-wave plate
numsource = []
# Loop over files for the four wave plate files
for k in range(0, len(angle), 1):
# Open fits file, extract pixel flux data and remove saturated pixels
try:
hdulist = fits.open(files[k])
image_data = hdulist[0].data
except FileNotFoundError:
raise FileNotFoundError("Cannot find the input fits file(s).")
# Remove bad pixels and mask edges
image_data[image_data > 60000] = 0
image_data[image_data < 0] = 0
rows = len(image_data[:, 0])
cols = len(image_data[0, :])
hdulist.close()
# Calculate estimate of background using sigma-clipping and calculate
# number of pixels used in the background region that were not
# clipped! This is done in a small area near the optical axis.
go_bmean, go_bmedian, go_bstd = sigma_clipped_stats(
image_data[510:568, 520:580], sigma=3.0, maxiters=5)
ge_bmean, ge_bmedian, ge_bstd = sigma_clipped_stats(
image_data[446:504, 520:580], sigma=3.0, maxiters=5)
mask_o = sigma_clip(image_data[510:568, 520:580],
sigma=3.0,
maxiters=5,
masked=True)
mask_e = sigma_clip(image_data[446:504, 520:580],
sigma=3.0,
maxiters=5,
masked=True)
ann_area_o = np.ma.MaskedArray.count(mask_o)
ann_area_e = np.ma.MaskedArray.count(mask_e)
# Detect sources using DAO star finder
daofind_o = DAOStarFinder(fwhm=5,
threshold=5 * go_bstd,
exclude_border=True)
daofind_e = DAOStarFinder(fwhm=5,
threshold=5 * ge_bstd,
exclude_border=True)
sources_o = daofind_o(image_data[522:552, 535:565])
sources_e = daofind_e(image_data[462:492, 535:565])
if (sources_o is None or sources_e is None):
raise ValueError("No source detected in image")
if len(sources_o) != len(sources_e):
raise ValueError("Unequal number of sources in o and e images!")
glob_bgm = [go_bmean, ge_bmean]
glob_bgerr = [go_bstd, ge_bstd]
# Convert the source centroids back into detector pixels
sources_o['xcentroid'] = sources_o['xcentroid'] + 535
sources_o['ycentroid'] = sources_o['ycentroid'] + 522
sources_e['xcentroid'] = sources_e['xcentroid'] + 535
sources_e['ycentroid'] = sources_e['ycentroid'] + 462
# Estimate the FWHM of the source by simulating a 2D Gaussian
# This is only done on the 0 angle image ensuring aperture sizes
# are equal for all half-wave plate angles. If a user specified
# FWHM is given, then the estimation is not used.
if fwhm == 0.0:
xpeaks_o = []
xpeaks_e = []
ypeaks_o = []
ypeaks_e = []
fwhm = []
for i in range(0, len(sources_o), 1):
data_o = image_data[525:550, 535:565]
xpeaks_o.append(int(sources_o[i]['xcentroid']) - 535)
ypeaks_o.append(int(sources_o[i]['ycentroid']) - 525)
data_e = image_data[465:490, 535:560]
xpeaks_e.append(int(sources_e[i]['xcentroid']) - 535)
ypeaks_e.append(int(sources_e[i]['ycentroid']) - 465)
min_count_o = np.min(data_o)
min_count_e = np.min(data_e)
max_count_o = data_o[ypeaks_o[i], xpeaks_e[i]]
max_count_e = data_e[ypeaks_o[i], xpeaks_e[i]]
half_max_o = (max_count_o + min_count_o) / 2
half_max_e = (max_count_e + min_count_e) / 2
# Crude calculation for each source
nearest_above_x_o = ((
np.abs(data_o[ypeaks_o[i], xpeaks_o[i]:-1] -
half_max_o)).argmin())
nearest_below_x_o = ((
np.abs(data_o[ypeaks_o[i], 0:xpeaks_o[i]] -
half_max_o)).argmin())
nearest_above_x_e = ((
np.abs(data_e[ypeaks_e[i], xpeaks_e[i]:-1] -
half_max_e)).argmin())
nearest_below_x_e = ((
np.abs(data_e[ypeaks_e[i], 0:xpeaks_e[i]] -
half_max_e)).argmin())
nearest_above_y_o = ((
np.abs(data_o[ypeaks_o[i]:-1, xpeaks_o[i]] -
half_max_o)).argmin())
nearest_below_y_o = ((
np.abs(data_o[0:ypeaks_o[i], xpeaks_o[i]] -
half_max_o)).argmin())
nearest_above_y_e = ((
np.abs(data_e[ypeaks_e[i]:-1, xpeaks_e[i]] -
half_max_e)).argmin())
nearest_below_y_e = ((
np.abs(data_e[0:ypeaks_e[i], xpeaks_e[i]] -
half_max_e)).argmin())
fwhm.append(
(nearest_above_x_o + (xpeaks_o[i] - nearest_below_x_o)))
fwhm.append(
(nearest_above_y_o + (ypeaks_o[i] - nearest_below_y_o)))
fwhm.append(
(nearest_above_x_e + (xpeaks_e[i] - nearest_below_x_e)))
fwhm.append(
(nearest_above_y_e + (ypeaks_e[i] - nearest_below_y_e)))
fwhm = np.mean(fwhm)
# Stack both ord and exord sources together
tot_sources = vstack([sources_o, sources_e])
# Store the ordinary and extraordinary beam source images and
# create apertures for aperture photometry
positions = np.swapaxes(
np.array((tot_sources['xcentroid'], tot_sources['ycentroid']),
dtype='float'), 0, 1)
aperture = CircularAperture(positions, r=0.5 * apermul * fwhm)
phot_table = aperture_photometry(image_data, aperture)
# Set up arrays of ord and exord source parameters
s_id = np.zeros([len(np.array(phot_table['id']))])
xp = np.zeros([len(s_id)])
yp = np.zeros([len(s_id)])
fluxbgs = np.zeros([len(s_id)])
mean_bg = np.zeros([len(s_id)])
bg_err = np.zeros([len(s_id)])
s_area = []
for i in range(0, len(np.array(phot_table['id'])), 1):
s_id[i] = np.array(phot_table['id'][i])
xpos = np.array(phot_table['xcenter'][i])
ypos = np.array(phot_table['ycenter'][i])
xp[i] = xpos
yp[i] = ypos
s_area.append(np.pi * (0.5 * apermul * fwhm)**2)
j = i % 2
fluxbgs[i] = (phot_table['aperture_sum'][i] -
aperture.area * glob_bgm[j])
mean_bg[i] = glob_bgm[j]
bg_err[i] = glob_bgerr[j]
# Create and save the image in z scale and overplot the ordinary and
# extraordinary apertures and local background annuli if applicable
fig = plt.figure()
zscale = ZScaleInterval(image_data)
norm = ImageNormalize(stretch=SqrtStretch(), interval=zscale)
image = plt.imshow(image_data, cmap='gray', origin='lower', norm=norm)
bg_annulus_o = RectangularAnnulus((550, 539),
w_in=0.1,
w_out=60,
h_out=58,
theta=0)
bg_annulus_e = RectangularAnnulus((550, 475),
w_in=0.1,
w_out=60,
h_out=58,
theta=0)
bg_annulus_o.plot(color='skyblue', lw=1.5, alpha=0.5)
bg_annulus_e.plot(color='lightgreen', lw=1.5, alpha=0.5)
for i in range(0, len(np.array(phot_table['id'])), 1):
aperture = CircularAperture((xp[i], yp[i]), r=0.5 * apermul * fwhm)
if i < int(len(np.array(phot_table['id'])) / 2):
aperture.plot(color='blue', lw=1.5, alpha=0.5)
else:
aperture.plot(color='green', lw=1.5, alpha=0.5)
plt.xlim(500, 600)
plt.ylim(425, 575)
plt.title(label[k])
image_fn = folder_path + angle[k] + '_image.png'
fig.savefig(image_fn)
# Create dataframes for photometry results
cols = [
'xpix', 'ypix', 'fluxbgs', 'sourcearea', 'meanbg', 'bgerr',
'bgarea'
]
df_o = pd.DataFrame(columns=cols)
df_e = pd.DataFrame(columns=cols)
for i in range(0, len(np.array(phot_table['id'])), 1):
if 0 <= i < int(len(np.array(phot_table['id'])) / 2):
df_o = df_o.append(
{
cols[0]: xp[i],
cols[1]: yp[i],
cols[2]: fluxbgs[i],
cols[3]: s_area[i],
cols[4]: mean_bg[i],
cols[5]: bg_err[i],
cols[6]: ann_area_o
},
ignore_index=True)
else:
df_e = df_e.append(
{
cols[0]: xp[i],
cols[1]: yp[i],
cols[2]: fluxbgs[i],
cols[3]: s_area[i],
cols[4]: mean_bg[i],
cols[5]: bg_err[i],
cols[6]: ann_area_e
},
ignore_index=True)
# Save dataframes to text files
df_o.to_string(folder_path + 'angle' + angle[k] + '_ord.txt',
index=False,
justify='left')
df_e.to_string(folder_path + 'angle' + angle[k] + '_exord.txt',
index=False,
justify='left')
# Save the number of sources in each beam to a list
numsource.append(int(len(np.array(phot_table['id'])) / 2))
# Print number of sources per half-wave plate image and FWHM
print("FWHM =", fwhm, "pixels")
for i in range(0, len(numsource), 1):
print("No of sources detected at", ang_dec[i], "degrees:",
numsource[i])
return 0
for it in range(0,len(tilet)):
date = str(datet[it])
tile = str(tilet[it])
tt=Table.read(dirvi+tp[:3]+'/'+'desi-vi_'+tp[:3]+'_tile'+tile+'_nightdeep_merged_all_'+date+'.csv',format='pandas.csv')
tt.keep_columns(['TARGETID','best_z','best_quality','best_spectype','all_VI_issues','all_VI_comments','merger_comment','N_VI'])
tz = Table.read(dirz+'/'+tp+'/'+tile+'_'+tp+'zinfo.fits')
tj = join(tz,tt,join_type='left',keys='TARGETID')
tj['N_VI'].fill_value = 0
tj['N_VI'] = tj['N_VI'].filled() #should easily be able to select rows with N_VI > 0 to get desired info
tj.write(dirz+'/'+tp+'/'+tile+'_'+tp+'zinfo_wVI.fits',format='fits',overwrite=True)
print('wrote file with VI info to '+dirz+'/'+tp+'/'+tile+'_'+tp+'zinfo_wVI.fits')
if len(tilet) > 1:
dt = Table.read(dirz+'/'+tp+'/'+str(tilet[0])+'_'+tp+'zinfo_wVI.fits')
for it in range(1,len(tilet)):
dtn = Table.read(dirz+'/'+tp+'/'+str(tilet[it])+'_'+tp+'zinfo_wVI.fits')
dt = vstack([dt,dtn])
cols = ['z','zwarn','chi2','deltachi2','spectype','subtype']
for i in range(1,5):
dt['z_'+str(i)]=np.zeros(len(dt))
dt['zwarn_'+str(i)]=np.zeros(len(dt))
dt['chi2_'+str(i)]=np.zeros(len(dt))
dt['deltachi2_'+str(i)]=np.zeros(len(dt))
dt['spectype_'+str(i)] = 'GALAXY'
dt['subtype_'+str(i)] = 'GALAXY'
for ii in range(0,len(dt)):
ln = dt[ii]
zfits = zinfo.get_zfits(ln['TILEID'],ln['PETAL_LOC'],ln['subset'],ln['TARGETID'],release)
for jj in range(1,5):
def vstack(tables, **kwargs):
"""
Wraps astropy.table.vstack and returns a LineList
"""
return LineList(table.vstack(tables, **kwargs))
