def saveTableToFile(table,
saveFile=None,
fileType='hdf5',
table_name='table',
overwrite=False,
**kwargs):
'''
Save a grid or source table to a file.
This is much faster than atpy or astropy for creating hdf5 files.
'''
if (fileType == 'hdf5'):
kwargs['path'] = table_name
if saveFile is None:
print('!!> No file name given, not saving to any file.')
else:
if (not overwrite) and (os.path.exists(saveFile)):
print(
'!!> Output file %s already exists.\n Use the overwrite option if needed.'
% saveFile)
else:
if (os.path.exists(saveFile)):
os.remove(saveFile)
print("@@> Saving table %s to file %s as type %s" %
(table_name, saveFile, fileType))
table.write(saveFile, format=fileType, **kwargs)
def makeCartoon(seed=1):
"""Create a cartoon jitter timeseries"""
np.random.seed(seed)
rmsat2s = 2.0 / 3.0
rmsat120s = 0.21
# share across two dimensions
rmsat120s1d = rmsat120s / np.sqrt(2)
rmsat2s1d = rmsat2s / np.sqrt(2)
cadence = 2.0
smoothscale = 10.0
nsmooth = int(smoothscale / cadence)
t = np.arange(0, 30 * 24 * 60 * 60, 2)
n = len(t)
d = {}
d['t'] = t
for k in ['x', 'y']:
v = np.random.normal(0, 1, n)
for i in range(2):
v = np.convolve(v, np.ones(nsmooth), mode='same')
d[k] = v / np.std(v) * rmsat2s1d
table = astropy.table.Table(d, names=['t', 'x', 'y'])
table.write(os.path.join(settings.inputs, 'cartoon.jitter'), format='ascii.fixed_width', bookend=False)
def save(output_file_path, data_array, header_list, dtype_list):
print(output_file_path)
if output_file_path.lower().endswith('.csv'):
# See http://docs.astropy.org/en/stable/io/fits/usage/table.html
np.savetxt(
output_file_path,
data_array,
fmt=
"%s", # See http://stackoverflow.com/questions/16621351/how-to-use-python-numpy-savetxt-to-write-strings-and-float-number-to-an-ascii-fi
delimiter=",",
header=",".join(header_list),
comments=
"" # String that will be prepended to the ``header`` and ``footer`` strings, to mark them as comments. Default: '# '.
)
elif output_file_path.lower().endswith(('.fits', '.fit')):
table = astropy.table.Table(names=header_list, dtype=dtype_list)
for row in data_array:
table.add_row(row)
#print(table)
table.write(output_file_path, overwrite=True)
else:
raise Exception('Unknown output format.')
def makeCartoon(seed=1):
"""Create a cartoon jitter timeseries"""
np.random.seed(seed)
rmsat2s = 2.0 / 3.0
rmsat120s = 0.21
# share across two dimensions
rmsat120s1d = rmsat120s / np.sqrt(2)
rmsat2s1d = rmsat2s / np.sqrt(2)
cadence = 2.0
smoothscale = 10.0
nsmooth = int(smoothscale / cadence)
t = np.arange(0, 30 * 24 * 60 * 60, 2)
n = len(t)
d = {}
d['t'] = t
for k in ['x', 'y']:
v = np.random.normal(0, 1, n)
for i in range(2):
v = np.convolve(v, np.ones(nsmooth), mode='same')
d[k] = v / np.std(v) * rmsat2s1d
table = astropy.table.Table(d, names=['t', 'x', 'y'])
table.write(os.path.join(settings.inputs, 'cartoon.jitter'),
format='ascii.fixed_width',
bookend=False)
def save(output_file_path, data_array, header_list, dtype_list):
print(output_file_path)
if output_file_path.lower().endswith('.csv'):
# See http://docs.astropy.org/en/stable/io/fits/usage/table.html
np.savetxt(output_file_path,
data_array,
fmt="%s", # See http://stackoverflow.com/questions/16621351/how-to-use-python-numpy-savetxt-to-write-strings-and-float-number-to-an-ascii-fi
delimiter=",",
header=",".join(header_list),
comments="" # String that will be prepended to the ``header`` and ``footer`` strings, to mark them as comments. Default: '# '.
)
elif output_file_path.lower().endswith(('.fits', '.fit')):
table = astropy.table.Table(names=header_list, dtype=dtype_list)
for row in data_array:
table.add_row(row)
#print(table)
table.write(output_file_path, overwrite=True)
else:
raise Exception('Unknown output format.')
def clobber_table_write(table, filename, **kwargs):
""" Writes a table, even if it has to clobber an older one. """
try:
table.write(filename, **kwargs)
except Exception, e:
print e
print "Overwriting file."
os.remove(filename)
table.write(filename, **kwargs)
def do_all_h2co():
table = initialize_table()
result = (filaments_right(table=table) + filaments_left(table=table) +
middlechunk(table=table) + maus(table=table) +
w51main(table=table))
table.write(os.path.join(datapath_spectra, "spectralfit_table.ipac"),
format='ascii.ipac')
table2 = add_tex_tau_to_table(table)
table2.write(os.path.join(datapath_spectra, "spectralfit_table_withtextau.ipac"),
format='ascii.ipac')
return result,table
def table_info(table, label):
# Print basic info to console
print('\n*** {0} ***'.format(label))
print('Rows: {0}'.format(len(table)))
print('Columns: {0}'.format(table.colnames))
print('')
# Save to file for debugging
filename = 'crab_mwl_{0}.ipac'.format(label)
print('INFO: Writing {0}'.format(filename))
table.write(filename, format='ascii.ipac')
def from_table(cls,
table,
slice,
mos_point_source,
relative_path='spectra',
fname=None):
"""
class method to add a new spectrum given a table from the extracted
spectrum.
Parameters
----------
table: astropy.table.Table
mos_point_source: MosPointSource
relative_path: str
relative path to work dir (default='spectra')
fname: str
filename of the extracted spectrum (default None)
if None it automatically generates
<raw_fits_fname>_slc<slc_id>_src<src_id>
"""
if not os.path.exists(os.path.join(cls.work_dir, relative_path)):
logger.warn('Relative dir {0} does not exist for adding spectrum '
'- creating'.format(relative_path))
os.mkdir(os.path.join(cls.work_dir, relative_path))
if fname is None:
fname = '{0}_slc{1}_src{2}.h5'.format(
slice.science_set.science.fits.fname.replace('.fits', ''),
slice.id, mos_point_source.id)
full_fname = os.path.join(cls.work_dir, relative_path, fname)
table.write(full_fname,
path=cls.hdf5_path,
format='hdf5',
overwrite=True)
data_file = DataFile.from_file(os.path.join(relative_path, fname))
spectrum = cls(mos_point_source_id=mos_point_source.id,
slice_id=slice.id)
spectrum.data_file = data_file
return data_file, spectrum
def main():
file_ = sys.argv[1]
dest = file_ + '.csv'
table = astropy.table.Table.read(file_, format='ascii.ipac')
table.meta = {}
table.write(dest,
format='csv',
fill_values=[(ascii.masked, '\\N')],
overwrite=True)
return
def write_csv(self, filename=None, path=None, overwrite=None, **kwargs):
''' Write the datamodel to a CSV '''
release = self.parent.aliases[0].lower().replace('-', '')
if not filename:
filename = 'dapmodels_dm_{0}.csv'.format(release)
if not path:
path = os.path.join(os.getenv("MARVIN_DIR"), 'docs', 'sphinx', '_static')
fullpath = os.path.join(path, filename)
table = self.to_table(**kwargs)
table.write(fullpath, format='csv', overwrite=overwrite)
def test_load_bad(tmpdir):
meta = dict(group_name='g', band_name='b')
# Missing wavelength column.
table = astropy.table.QTable(meta=meta)
table['response'] = [1, 1]
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
# Missing response column.
table = astropy.table.QTable(meta=meta)
table['wavelength'] = [1, 2] * u.Angstrom
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
# Missing wavelength units.
table = astropy.table.QTable(meta=meta)
table['wavelength'] = [1, 2]
table['response'] = [1, 1]
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
# Unexpected response units.
table = astropy.table.QTable(meta=meta)
table['wavelength'] = [1, 2] * u.Angstrom
table['response'] = [1, 1] * u.erg
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
def test_load_bad(tmpdir):
meta = dict(group_name='g', band_name='b')
# Missing wavelength column.
table = astropy.table.QTable(meta=meta)
table['response'] = [1, 1]
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
# Missing response column.
table = astropy.table.QTable(meta=meta)
table['wavelength'] = [1, 2] * u.Angstrom
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
# Missing wavelength units.
table = astropy.table.QTable(meta=meta)
table['wavelength'] = [1, 2]
table['response'] = [1, 1]
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
# Unexpected response units.
table = astropy.table.QTable(meta=meta)
table['wavelength'] = [1, 2] * u.Angstrom
table['response'] = [1, 1] * u.erg
name = str(tmpdir.join('bad.ecsv'))
table.write(name, format='ascii.ecsv', overwrite=True)
with pytest.raises(RuntimeError):
load_filter(name)
def from_table(cls, table, slice, mos_point_source, relative_path='spectra',
fname=None):
"""
class method to add a new spectrum given a table from the extracted
spectrum.
Parameters
----------
table: astropy.table.Table
mos_point_source: MosPointSource
relative_path: str
relative path to work dir (default='spectra')
fname: str
filename of the extracted spectrum (default None)
if None it automatically generates
<raw_fits_fname>_slc<slc_id>_src<src_id>
"""
if not os.path.exists(os.path.join(cls.work_dir, relative_path)):
logger.warn('Relative dir {0} does not exist for adding spectrum '
'- creating'.format(relative_path))
os.mkdir(os.path.join(cls.work_dir, relative_path))
if fname is None:
fname = '{0}_slc{1}_src{2}.h5'.format(
slice.science_set.science.fits.fname.replace('.fits', ''),
slice.id, mos_point_source.id)
full_fname = os.path.join(cls.work_dir, relative_path, fname)
table.write(full_fname, path=cls.hdf5_path, format='hdf5',
overwrite=True)
data_file = DataFile.from_file(os.path.join(relative_path, fname))
spectrum = cls(mos_point_source_id=mos_point_source.id,
slice_id=slice.id)
spectrum.data_file = data_file
return data_file, spectrum
def process_db(connection, name, tile, outfile, epochfile, band, blind=True, quiet=True):
sql="select * from {0}_main where tilename='{1}'".format(name, tile)
print sql
table = db.table_from_sql(connection, sql)
if table is None:
print "NO OBJECTS FOR ", name, tile
return
print "Loaded main table"
sql="select * from {0}_epoch where tilename='{1}'".format(name, tile)
print sql
epochs = db.table_from_sql(connection, sql)
print "Loaded epoch table"
table, epochs = process_table(table, epochs, band, blind=blind,quiet=quiet)
print 'Saving to FITS'
table.write(outfile, format='fits')
epochs.write(epochfile, format='fits')
def process_text(main_file, epoch_file, out_main, out_epoch, band, blind=True,quiet=True, report=False):
if os.path.exists(out_main):
print 'Skipping sub-file', out_main
return
table = astropy.table.Table.read(main_file, format='ascii')
epochs = astropy.table.Table.read(epoch_file, format='ascii')
if len(table)==0:
print "No rows found for ", outfile
return
table, epochs = process_table(table, epochs, band, blind=blind,quiet=quiet,report=report)
if report:
print "Not saving output file as we are just reporting cuts"
return
print 'Saving to FITS'
table.write(out_main, format='fits')
epochs.write(out_epoch, format='fits')
def check_write_table_via_table(test_def, table, fast_writer):
out = StringIO()
test_def = copy.deepcopy(test_def)
if 'Writer' in test_def['kwargs']:
format = 'ascii.{0}'.format(test_def['kwargs']['Writer']._format_name)
del test_def['kwargs']['Writer']
else:
format = 'ascii'
try:
table.write(out, format=format, fast_writer=fast_writer, **test_def['kwargs'])
except ValueError as e: # if format doesn't have a fast writer, ignore
if 'not in the list of formats with fast writers' not in str(e):
raise e
return
print('Expected:\n{}'.format(test_def['out']))
print('Actual:\n{}'.format(out.getvalue()))
assert [x.strip() for x in out.getvalue().strip().splitlines()] == [
x.strip() for x in test_def['out'].strip().splitlines()]
def check_write_table_via_table(test_def, table, fast_writer):
out = StringIO()
test_def = copy.deepcopy(test_def)
if 'Writer' in test_def['kwargs']:
format = f"ascii.{test_def['kwargs']['Writer']._format_name}"
del test_def['kwargs']['Writer']
else:
format = 'ascii'
try:
table.write(out, format=format, fast_writer=fast_writer, **test_def['kwargs'])
except ValueError as e: # if format doesn't have a fast writer, ignore
if 'not in the list of formats with fast writers' not in str(e.value):
raise e
return
print(f"Expected:\n{test_def['out']}")
print(f'Actual:\n{out.getvalue()}')
assert [x.strip() for x in out.getvalue().strip().splitlines()] == [
x.strip() for x in test_def['out'].strip().splitlines()]
def process_multi_tile_text(main_file, epoch_file, outdir, band, blind=True, quiet=True):
mkdir(outdir+"/text")
mkdir(outdir+"/text/main")
mkdir(outdir+"/text/epoch")
mkdir(outdir+"/fits/")
mkdir(outdir+"/fits/main")
mkdir(outdir+"/fits/epoch")
filenames = split_by_tile(main_file, epoch_file, outdir+"/text")
for n,(tilename, main_file, epoch_file) in enumerate(filenames):
print "{0} - Working on {0}".format(n,tilename)
table = astropy.table.Table.read(main_file, format='ascii')
epochs = astropy.table.Table.read(epoch_file, format='ascii')
table, epochs = process_table(table, epochs, band, blind=blind,quiet=quiet)
out_main = outdir+"/fits/main/"+tilename+".txt"
out_epoch = outdir+"/fits/epoch/"+tilename+".txt"
print 'Saving to FITS'
table.write(out_main, format='fits')
epochs.write(out_epoch, format='fits')
def write_csv(self, filename=None, path=None, overwrite=None, **kwargs):
''' Write the datamodel to a CSV '''
release = self.parent.release.lower().replace('-', '')
if not filename:
if isinstance(self.parent, DRPRSSDataModel):
filename = 'drp_rss_spectra_dm_{0}.csv'.format(release)
elif isinstance(self.parent, DRPCubeDataModel):
filename = 'drp_cube_spectra_dm_{0}.csv'.format(release)
else:
raise ValueError('invalid parent of type {!r}'.format(
type(self.parent)))
if not path:
path = os.path.join(os.getenv("MARVIN_DIR"), 'docs', 'sphinx',
'_static')
fullpath = os.path.join(path, filename)
table = self.to_table(**kwargs)
table.write(fullpath, format='csv', overwrite=overwrite)
def check_write_table_via_table(test_def, table, fast_writer, out=None):
if out is None:
out = StringIO()
test_def = copy.deepcopy(test_def)
if 'Writer' in test_def['kwargs']:
format = f"ascii.{test_def['kwargs']['Writer']._format_name}"
del test_def['kwargs']['Writer']
else:
format = 'ascii'
try:
table.write(out,
format=format,
fast_writer=fast_writer,
**test_def['kwargs'])
except ValueError as e: # if format doesn't have a fast writer, ignore
if 'not in the list of formats with fast writers' not in str(e.value):
raise e
return
if isinstance(out, StringIO):
# Output went to a buffer
actual = out.getvalue()
else:
# Output went to a file
if str(out).startswith('~'):
# Ensure a file hasn't been accidentally written to a literal tilde
# path
assert not os.path.exists(out)
out = os.path.expanduser(out)
assert os.path.exists(out)
with open(out) as f:
actual = f.read()
os.remove(out)
print(f"Expected:\n{test_def['out']}")
print(f'Actual:\n{actual}')
assert [x.strip() for x in actual.strip().splitlines()
] == [x.strip() for x in test_def['out'].strip().splitlines()]
def writeText(self, filename):
"""Write the defects out to a text file with the specified name.
Parameters
----------
filename : `str`
Name of the file to write. The file extension ".ecsv" will
always be used.
Returns
-------
used : `str`
The name of the file used to write the data (which may be
different from the supplied name given the change to file
extension).
Notes
-----
The file is written to ECSV format and will include any metadata
associated with the `Defects`.
"""
# Using astropy table is the easiest way to serialize to ecsv
afwTable = self.toSimpleTable()
table = afwTable.asAstropy()
metadata = afwTable.getMetadata()
now = datetime.datetime.utcnow()
metadata["DATE"] = now.isoformat()
metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
table.meta = metadata.toDict()
# Force file extension to .ecsv
path, ext = os.path.splitext(filename)
filename = path + ".ecsv"
table.write(filename, format="ascii.ecsv")
return filename
stel.append(stel_pa)
gas.append(gas_pa)
data.append(pa)
gas_error.append(gas_err)
stel_error.append(stel_err)
gal_id.append(plate_id)
stel = np.array(stel)
gas = np.array(gas)
data = np.array(data)
gal_id = np.array(gal_id)
stel_error = np.array(stel_error)
gas_error = np.array(gas_error)
bad = np.array([how_many_bad])
bad_gals = np.array(bad_gals)
t = Table()
t['STEL_PA'] = Column(stel, description='Stellar position angle')
t['STEL_PA_ERR'] = Column(stel_error,
description='Stellar position angle error')
t['GAS_PA'] = Column(gas, description='Position angle calculated from the gas')
t['GAS_PA_ERR'] = Column(
gas_error, description='Position angle calculated from the gas error')
t['PA'] = Column(data, description='Position angle from the MaNGA data')
t['GALAXY_ID'] = Column(gal_id, description='galaxy ID')
t['NUM_BAD_GALS'] = Column(bad, description="For Celeste's testing, ignore")
t['BAD_GALS_NAMES'] = Column(bad_gals,
description="For Celeste's testing, ignore")
t.write('/home/celeste/Documents/astro_research/summer_2018/pa_datav5.fits')
def do_indiv_fits(regfilename, outpfx, ncomp=1, dobaseline=False, table=None,
tableprefix="", **kwargs):
regions = pyregion.open(regfilename)
spectra = [both.get_apspec(r.coord_list, coordsys=r.coord_format,
wunit='degree')
for r in regions]
cont11hdu = fits.open(cont11filename)[0]
pl.ioff()
for ii,(sp,r) in enumerate(zip(spectra,regions)):
if hasattr(ncomp,'__len__'):
nc = ncomp[ii]
else:
nc = ncomp
chi2, parinfo = {},{}
sp.header['REGION'] = "{shape}({ra},{dec},{radius})".format(shape=r.name,
ra=r.coord_list[0],
dec=r.coord_list[1],
radius=r.coord_list[2])
## this is pretty hacky =(
#if dobaseline:
# spdict = fith2co.BigSpectrum_to_H2COdict(sp)
# for n,s in spdict.items():
# s.baseline(exclude=[2,8,43,75])
# sp = pyspeckit.Spectra(spdict.values())
# spectra[ii] = sp
for cont in ('front','back',):
print("Continuum in the %s" % cont)
fig = pl.figure(ii, figsize=(12,8))
fig.clf()
name = r.attr[1]['text']
if cont == 'back':
M = pyregion.ShapeList([r]).get_mask(cont11hdu)
c11 = cont11[M].mean()
c22 = cont22[M].mean()
sp.header['CONT11'] = c11
sp.header['CONT22'] = c22
elif cont == 'front':
c11 = 2.7315
c22 = 2.7315
dofit(sp, c11, c22, nc, **kwargs)
sp.specname = name
sp.header['OBJECT'] = name
print("ap%i %s: ni=%i, X^2%0.1f, X^2/n=%0.1f, n=%0.1f" % (ii,
sp.specname,
sp.specfit.fitter.mp.niter,
sp.specfit.chi2,
sp.specfit.chi2/sp.specfit.dof,
sp.specfit.parinfo.DENSITY0.value))
#if sp.specfit.chi2 / sp.specfit.dof > 2:
# gg[4] = sp.specfit.parinfo.CENTER0.value
# fixed[4] = True
# sp.specfit(fittype='formaldehyde_radex',guesses=gg,
# fixed=fixed, multifit=True,quiet=False,verbose=True,
# limits=limits,
# limited=limited,
# use_window_limits=False, fit_plotted_area=False)
sp.plotter.autorefresh=False
plotitem(sp, ii)
sp.plotter.savefig(os.path.join(datapath,
outpfx+'_aperture_%s_%s.pdf' %
(name,cont)))
plotitem(sp, ii, dolegend=True)
# seriously, something aint'n't right here
pl.figure(sp.plotter.figure.number)
pl.savefig(os.path.join(datapath,
outpfx+'_aperture_%s_%s_legend.pdf' %
(name,cont)),
bbox_extra_artists=[sp.specfit.fitleg])
# for writing to file, select the best-fit
chi2[cont] = sp.specfit.chi2
parinfo[cont] = sp.specfit.parinfo
if cont == 'back': # second one...
if sp.specfit.chi2 > chi2['front']:
print("Back chi^2 > front chi^2. Replacing parameters with Continuum in the Front")
best = 'front'
else:
best = 'back'
# set the previous, unset to match this...
# since we're guaranteed to be in the 2nd of 2 in a loop here,
# no danger
table[-1]['frontbackbest'] = best
if nc == 2:
# or previous *two* if twocomp
table[-2]['frontbackbest'] = best
# at this stage, 'frontbackbest' should be universally assigned....
if np.any(table['frontbackbest'] == ''):
import ipdb; ipdb.set_trace()
else:
best = ''
if table is not None:
add_parinfo_to_table(table, sp.specfit.parinfo, sp.specfit.chi2,
sp.specfit.dof, sp.specfit.optimal_chi2(reduced=False),
sp.specfit.optimal_chi2(reduced=True),
ra=r.coord_list[0],
dec=r.coord_list[1],
radius=r.coord_list[2],
frontback=cont,
frontbackbest=best,
name=tableprefix+sp.specname)
if best != '':
sp.specfit.parinfo = parinfo[best]
sp.specfit.chi2 = chi2[best]
sp.write(os.path.join(datapath,outpfx+"_aperture_%s.fits" % name))
table.write(os.path.join(datapath_spectra,
tableprefix+"spectralfit_table.ipac"),
format='ascii.ipac')
pl.ion()
return spectra
# Documentation:
# - http://docs.astropy.org/en/stable/table/index.html#getting-started
# - http://www.astropy.org/astropy-tutorials/FITS-tables.html
# - http://www.astropy.org/astropy-tutorials/FITS-header.html
import argparse
from astropy.io import fits
import numpy as np
import astropy.table
# PARSE OPTIONS ###############################################################
parser = argparse.ArgumentParser(description="An astropy snippet")
parser.add_argument("filearg", nargs=1, metavar="FILE", help="the output FITS file")
args = parser.parse_args()
file_path = args.filearg[0]
# WRITE DATA ##################################################################
table = astropy.table.Table(names=("column1", "column2", "column3"))
table.add_row([1, 2, 3])
table.add_row([10, 20, 30])
table.add_row([100, 200, 300])
print(table)
table.write(file_path, overwrite=True)
limited=[(True, True)] * 5,
fixed=[False, False, False, False, False])
#only change center parameter from example
#sp.plotter.savefig('H2CO_all_radexfit.pdf')
table.add_row()
table[-1]['x'] = v
table[-1]['y'] = w
table[-1]['temp'] = sp.specfit.modelpars[0]
table[-1]['column'] = sp.specfit.modelpars[1]
table[-1]['density'] = sp.specfit.modelpars[2]
table[-1]['center'] = sp.specfit.modelpars[3]
table[-1]['width'] = sp.specfit.modelpars[4]
table[-1]['temp errors'] = sp.specfit.modelerrs[0]
table[-1]['column errors'] = sp.specfit.modelerrs[1]
table[-1]['density errors'] = sp.specfit.modelerrs[2]
table[-1]['center errors'] = sp.specfit.modelerrs[3]
table[-1]['width errors'] = sp.specfit.modelerrs[4]
table.write('grs1915H2COparameters.fits', overwrite=True)
plt.show()
t = Table.read('grs1915H2COparameters.fits')
plt.scatter(t['x'],
t['y'],
c=t['temp'],
marker='o',
cmap='hot',
edgecolor='none')
plt.colorbar(label='Temperature(K)')
plt.savefig('grs1915H2COtempmap.pdf')
"""Generate an astropy-readable .ecsv files for `butler ingest-files`, to ingest an existing gen2 refcat.
The `refcat_dir` variable needs to be modified for each refcat being converted.
"""
import os
import glob
import astropy.table
refcat_dir = "ref_cats/gaia_dr2_20200414"
out_dir = "."
out_file = f"{out_dir}/{os.path.basename(refcat_dir)}.ecsv"
table = astropy.table.Table(names=("filename", "htm7"), dtype=("str", "int"))
files = glob.glob(f"{refcat_dir}/[0-9]*.fits")
for i, file in enumerate(files):
# running status, overwriting each print statement as it proceeds
print(f"{i}/{len(files)} ({100*i/len(files):0.1f}%)", end="\r")
# extract file index; add row to table
file_index = int(os.path.basename(os.path.splitext(file)[0]))
table.add_row((file, file_index))
table.write(out_file)
print(f"Saving to: {out_file}")
"""Combine all Fermi IRF data into a single file.
"""
from glob import glob
import numpy as np
from astropy import table
from astropy.io import fits
from astropy.table import Table, vstack
import astropy.units as u
from astropy.units import Quantity
filenames = glob('*.txt')
layers = []
primary = fits.PrimaryHDU()
layers.append(primary)
for filename in filenames:
print(filename)
table = Table.read(filename, format='ascii')
table.write('temp.fits', overwrite=True)
hdu = fits.open('temp.fits')[1]
layers.append(hdu)
hdus = fits.HDUList(layers)
hdus.writeto('fermi_irf_data.fits', clobber=True)
def sim_spectra(wave,
flux,
program,
spectra_filename,
obsconditions=None,
sourcetype=None,
targetid=None,
redshift=None,
expid=0,
seed=0,
skyerr=0.0,
ra=None,
dec=None,
meta=None,
fibermap_columns=None,
fullsim=False,
use_poisson=True,
specsim_config_file="desi",
dwave_out=None,
save_resolution=True):
"""
Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can
be used as input to the redshift fitter.
Args:
wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted)
flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size
flux has to be in units of 10^-17 ergs/s/cm2/A
spectra_filename : path to output FITS file in the Spectra format
program : dark, lrg, qso, gray, grey, elg, bright, mws, bgs
ignored if obsconditions is not None
Optional:
obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP
sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star
targetid : list of targetids for each target. default of None has them generated as str(range(nspec))
redshift : list/array with each index being the redshifts for that target
expid : this expid number will be saved in the Spectra fibermap
seed : random seed
skyerr : fractional sky subtraction error
ra : numpy array with targets RA (deg)
dec : numpy array with targets Dec (deg)
meta : dictionnary, saved in primary fits header of the spectra file
fibermap_columns : add these columns to the fibermap
fullsim : if True, write full simulation data in extra file per camera
use_poisson : if False, do not use numpy.random.poisson to simulate the Poisson noise. This is useful to get reproducible random
realizations.
save_resolution : if True it will save the Resolution matrix for each spectra.
If False returns a resolution matrix (useful for mocks to save disk space).
"""
log = get_logger()
if len(flux.shape) == 1:
flux = flux.reshape((1, flux.size))
nspec = flux.shape[0]
log.info("Starting simulation of {} spectra".format(nspec))
if sourcetype is None:
sourcetype = np.array(["star" for i in range(nspec)])
log.debug("sourcetype = {}".format(sourcetype))
tileid = 0
telera = 0
teledec = 0
dateobs = time.gmtime()
night = desisim.obs.get_night(utc=dateobs)
program = program.lower()
frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec)
frame_fibermap.meta["FLAVOR"] = "custom"
frame_fibermap.meta["NIGHT"] = night
frame_fibermap.meta["EXPID"] = expid
# add DESI_TARGET
tm = desitarget.targetmask.desi_mask
frame_fibermap['DESI_TARGET'][sourcetype == "star"] = tm.STD_FAINT
frame_fibermap['DESI_TARGET'][sourcetype == "lrg"] = tm.LRG
frame_fibermap['DESI_TARGET'][sourcetype == "elg"] = tm.ELG
frame_fibermap['DESI_TARGET'][sourcetype == "qso"] = tm.QSO
frame_fibermap['DESI_TARGET'][sourcetype == "sky"] = tm.SKY
frame_fibermap['DESI_TARGET'][sourcetype == "bgs"] = tm.BGS_ANY
if fibermap_columns is not None:
for k in fibermap_columns.keys():
frame_fibermap[k] = fibermap_columns[k]
if targetid is None:
targetid = np.arange(nspec).astype(int)
# add TARGETID
frame_fibermap['TARGETID'] = targetid
# spectra fibermap has two extra fields : night and expid
# This would be cleaner if desispec would provide the spectra equivalent
# of desispec.io.empty_fibermap()
spectra_fibermap = desispec.io.empty_fibermap(nspec)
spectra_fibermap = desispec.io.util.add_columns(
spectra_fibermap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(night), np.int32(expid),
np.int32(tileid)],
)
for s in range(nspec):
for tp in frame_fibermap.dtype.fields:
spectra_fibermap[s][tp] = frame_fibermap[s][tp]
if ra is not None:
spectra_fibermap["TARGET_RA"] = ra
spectra_fibermap["FIBER_RA"] = ra
if dec is not None:
spectra_fibermap["TARGET_DEC"] = dec
spectra_fibermap["FIBER_DEC"] = dec
if obsconditions is None:
if program in ['dark', 'lrg', 'qso']:
obsconditions = desisim.simexp.reference_conditions['DARK']
elif program in ['elg', 'gray', 'grey']:
obsconditions = desisim.simexp.reference_conditions['GRAY']
elif program in ['mws', 'bgs', 'bright']:
obsconditions = desisim.simexp.reference_conditions['BRIGHT']
else:
raise ValueError('unknown program {}'.format(program))
elif isinstance(obsconditions, str):
try:
obsconditions = desisim.simexp.reference_conditions[
obsconditions.upper()]
except KeyError:
raise ValueError('obsconditions {} not in {}'.format(
obsconditions.upper(),
list(desisim.simexp.reference_conditions.keys())))
try:
params = desimodel.io.load_desiparams()
wavemin = params['ccd']['b']['wavemin']
wavemax = params['ccd']['z']['wavemax']
except KeyError:
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
if specsim_config_file == "eboss":
wavemin = 3500
wavemax = 10000
if wave[0] > wavemin:
log.warning(
'Minimum input wavelength {}>{}; padding with zeros'.format(
wave[0], wavemin))
dwave = wave[1] - wave[0]
npad = int((wave[0] - wavemin) / dwave + 1)
wavepad = np.arange(npad) * dwave
wavepad += wave[0] - dwave - wavepad[-1]
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wavepad, wave])
flux = np.hstack([fluxpad, flux])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wave[0] <= wavemin
if wave[-1] < wavemax:
log.warning(
'Maximum input wavelength {}<{}; padding with zeros'.format(
wave[-1], wavemax))
dwave = wave[-1] - wave[-2]
npad = int((wavemax - wave[-1]) / dwave + 1)
wavepad = wave[-1] + dwave + np.arange(npad) * dwave
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wave, wavepad])
flux = np.hstack([flux, fluxpad])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wavemax <= wave[-1]
ii = (wavemin <= wave) & (wave <= wavemax)
flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2)
wave = wave[ii] * u.Angstrom
flux = flux[:, ii] * flux_unit
sim = desisim.simexp.simulate_spectra(
wave,
flux,
fibermap=frame_fibermap,
obsconditions=obsconditions,
redshift=redshift,
seed=seed,
psfconvolve=True,
specsim_config_file=specsim_config_file,
dwave_out=dwave_out)
random_state = np.random.RandomState(seed)
sim.generate_random_noise(random_state, use_poisson=use_poisson)
scale = 1e17
specdata = None
resolution = {}
for camera in sim.instrument.cameras:
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1])
if not save_resolution:
resolution[camera.name] = R.to_fits_array()
skyscale = skyerr * random_state.normal(size=sim.num_fibers)
if fullsim:
for table in sim.camera_output:
band = table.meta['name'].strip()[0]
table_filename = spectra_filename.replace(
".fits", "-fullsim-{}.fits".format(band))
table.write(table_filename, format="fits", overwrite=True)
print("wrote", table_filename)
if specsim_config_file == "eboss":
for table in sim._eboss_camera_output:
wave = table['wavelength'].astype(float)
flux = (table['observed_flux'] + table['random_noise_electrons'] *
table['flux_calibration']).T.astype(float)
if np.any(skyscale):
flux += ((table['num_sky_electrons'] * skyscale) *
table['flux_calibration']).T.astype(float)
ivar = table['flux_inverse_variance'].T.astype(float)
band = table.meta['name'].strip()[0]
flux = flux * scale
ivar = ivar / scale**2
mask = np.zeros(flux.shape).astype(int)
spec = Spectra([band], {band: wave}, {band: flux}, {band: ivar},
resolution_data=None,
mask={band: mask},
fibermap=spectra_fibermap,
meta=meta,
single=True)
if specdata is None:
specdata = spec
else:
specdata.update(spec)
else:
for table in sim.camera_output:
wave = table['wavelength'].astype(float)
flux = (table['observed_flux'] + table['random_noise_electrons'] *
table['flux_calibration']).T.astype(float)
if np.any(skyscale):
flux += ((table['num_sky_electrons'] * skyscale) *
table['flux_calibration']).T.astype(float)
ivar = table['flux_inverse_variance'].T.astype(float)
band = table.meta['name'].strip()[0]
flux = flux * scale
ivar = ivar / scale**2
mask = np.zeros(flux.shape).astype(int)
if not save_resolution:
spec = Spectra([band], {band: wave}, {band: flux},
{band: ivar},
resolution_data=None,
mask={band: mask},
fibermap=spectra_fibermap,
meta=meta,
single=True)
else:
spec = Spectra([band], {band: wave}, {band: flux},
{band: ivar},
resolution_data={band: resolution[band]},
mask={band: mask},
fibermap=spectra_fibermap,
meta=meta,
single=True)
if specdata is None:
specdata = spec
else:
specdata.update(spec)
desispec.io.write_spectra(spectra_filename, specdata)
log.info('Wrote ' + spectra_filename)
# need to clear the simulation buffers that keeps growing otherwise
# because of a different number of fibers each time ...
desisim.specsim._simulators.clear()
desisim.specsim._simdefaults.clear()
if not save_resolution:
return resolution
"""Combine all Fermi IRF data into a single file.
"""
from glob import glob
import numpy as np
from astropy import table
from astropy.io import fits
from astropy.table import Table, vstack
import astropy.units as u
from astropy.units import Quantity
filenames = glob('*.txt')
layers = []
primary = fits.PrimaryHDU()
layers.append(primary)
for filename in filenames:
print(filename)
table = Table.read(filename, format='ascii')
table.write('temp.fits', overwrite=True)
hdu = fits.open('temp.fits')[1]
layers.append(hdu)
hdus = fits.HDUList(layers)
hdus.writeto('fermi_irf_data.fits', clobber=True)
def sim_spectra(wave, flux, program, spectra_filename, obsconditions=None,
sourcetype=None, targetid=None, redshift=None, expid=0, seed=0, skyerr=0.0, ra=None, dec=None, meta=None, fibermap_columns=None, fullsim=False,use_poisson=True):
"""
Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can
be used as input to the redshift fitter.
Args:
wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted)
flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size
flux has to be in units of 10^-17 ergs/s/cm2/A
spectra_filename : path to output FITS file in the Spectra format
program : dark, lrg, qso, gray, grey, elg, bright, mws, bgs
ignored if obsconditions is not None
Optional:
obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP
sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star
targetid : list of targetids for each target. default of None has them generated as str(range(nspec))
redshift : list/array with each index being the redshifts for that target
expid : this expid number will be saved in the Spectra fibermap
seed : random seed
skyerr : fractional sky subtraction error
ra : numpy array with targets RA (deg)
dec : numpy array with targets Dec (deg)
meta : dictionnary, saved in primary fits header of the spectra file
fibermap_columns : add these columns to the fibermap
fullsim : if True, write full simulation data in extra file per camera
use_poisson : if False, do not use numpy.random.poisson to simulate the Poisson noise. This is useful to get reproducible random realizations.
"""
log = get_logger()
if len(flux.shape)==1 :
flux=flux.reshape((1,flux.size))
nspec=flux.shape[0]
log.info("Starting simulation of {} spectra".format(nspec))
if sourcetype is None :
sourcetype = np.array(["star" for i in range(nspec)])
log.debug("sourcetype = {}".format(sourcetype))
tileid = 0
telera = 0
teledec = 0
dateobs = time.gmtime()
night = desisim.obs.get_night(utc=dateobs)
program = program.lower()
frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec)
frame_fibermap.meta["FLAVOR"]="custom"
frame_fibermap.meta["NIGHT"]=night
frame_fibermap.meta["EXPID"]=expid
# add DESI_TARGET
tm = desitarget.targetmask.desi_mask
frame_fibermap['DESI_TARGET'][sourcetype=="star"]=tm.STD_FAINT
frame_fibermap['DESI_TARGET'][sourcetype=="lrg"]=tm.LRG
frame_fibermap['DESI_TARGET'][sourcetype=="elg"]=tm.ELG
frame_fibermap['DESI_TARGET'][sourcetype=="qso"]=tm.QSO
frame_fibermap['DESI_TARGET'][sourcetype=="sky"]=tm.SKY
frame_fibermap['DESI_TARGET'][sourcetype=="bgs"]=tm.BGS_ANY
if fibermap_columns is not None :
for k in fibermap_columns.keys() :
frame_fibermap[k] = fibermap_columns[k]
if targetid is None:
targetid = np.arange(nspec).astype(int)
# add TARGETID
frame_fibermap['TARGETID'] = targetid
# spectra fibermap has two extra fields : night and expid
# This would be cleaner if desispec would provide the spectra equivalent
# of desispec.io.empty_fibermap()
spectra_fibermap = desispec.io.empty_fibermap(nspec)
spectra_fibermap = desispec.io.util.add_columns(spectra_fibermap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(night), np.int32(expid), np.int32(tileid)],
)
for s in range(nspec):
for tp in frame_fibermap.dtype.fields:
spectra_fibermap[s][tp] = frame_fibermap[s][tp]
if ra is not None :
spectra_fibermap["TARGET_RA"] = ra
spectra_fibermap["FIBER_RA"] = ra
if dec is not None :
spectra_fibermap["TARGET_DEC"] = dec
spectra_fibermap["FIBER_DEC"] = dec
if obsconditions is None:
if program in ['dark', 'lrg', 'qso']:
obsconditions = desisim.simexp.reference_conditions['DARK']
elif program in ['elg', 'gray', 'grey']:
obsconditions = desisim.simexp.reference_conditions['GRAY']
elif program in ['mws', 'bgs', 'bright']:
obsconditions = desisim.simexp.reference_conditions['BRIGHT']
else:
raise ValueError('unknown program {}'.format(program))
elif isinstance(obsconditions, str):
try:
obsconditions = desisim.simexp.reference_conditions[obsconditions.upper()]
except KeyError:
raise ValueError('obsconditions {} not in {}'.format(
obsconditions.upper(),
list(desisim.simexp.reference_conditions.keys())))
try:
params = desimodel.io.load_desiparams()
wavemin = params['ccd']['b']['wavemin']
wavemax = params['ccd']['z']['wavemax']
except KeyError:
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
if wave[0] > wavemin:
log.warning('Minimum input wavelength {}>{}; padding with zeros'.format(
wave[0], wavemin))
dwave = wave[1] - wave[0]
npad = int((wave[0] - wavemin)/dwave + 1)
wavepad = np.arange(npad) * dwave
wavepad += wave[0] - dwave - wavepad[-1]
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wavepad, wave])
flux = np.hstack([fluxpad, flux])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wave[0] <= wavemin
if wave[-1] < wavemax:
log.warning('Maximum input wavelength {}<{}; padding with zeros'.format(
wave[-1], wavemax))
dwave = wave[-1] - wave[-2]
npad = int( (wavemax - wave[-1])/dwave + 1 )
wavepad = wave[-1] + dwave + np.arange(npad)*dwave
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wave, wavepad])
flux = np.hstack([flux, fluxpad])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wavemax <= wave[-1]
ii = (wavemin <= wave) & (wave <= wavemax)
flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm ** 2 )
wave = wave[ii]*u.Angstrom
flux = flux[:,ii]*flux_unit
sim = desisim.simexp.simulate_spectra(wave, flux, fibermap=frame_fibermap,
obsconditions=obsconditions, redshift=redshift, seed=seed,
psfconvolve=True)
random_state = np.random.RandomState(seed)
sim.generate_random_noise(random_state,use_poisson=use_poisson)
scale=1e17
specdata = None
resolution={}
for camera in sim.instrument.cameras:
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1])
skyscale = skyerr * random_state.normal(size=sim.num_fibers)
if fullsim :
for table in sim.camera_output :
band = table.meta['name'].strip()[0]
table_filename=spectra_filename.replace(".fits","-fullsim-{}.fits".format(band))
table.write(table_filename,format="fits",overwrite=True)
print("wrote",table_filename)
for table in sim.camera_output :
wave = table['wavelength'].astype(float)
flux = (table['observed_flux']+table['random_noise_electrons']*table['flux_calibration']).T.astype(float)
if np.any(skyscale):
flux += ((table['num_sky_electrons']*skyscale)*table['flux_calibration']).T.astype(float)
ivar = table['flux_inverse_variance'].T.astype(float)
band = table.meta['name'].strip()[0]
flux = flux * scale
ivar = ivar / scale**2
mask = np.zeros(flux.shape).astype(int)
spec = Spectra([band], {band : wave}, {band : flux}, {band : ivar},
resolution_data={band : resolution[band]},
mask={band : mask},
fibermap=spectra_fibermap,
meta=meta,
single=True)
if specdata is None :
specdata = spec
else :
specdata.update(spec)
desispec.io.write_spectra(spectra_filename, specdata)
log.info('Wrote '+spectra_filename)
# need to clear the simulation buffers that keeps growing otherwise
# because of a different number of fibers each time ...
desisim.specsim._simulators.clear()
desisim.specsim._simdefaults.clear()
def print_table(field='cdfs'):
swire_names, swire_coords, _ = pipeline.generate_swire_features(overwrite=False, field=field)
swire_labels = pipeline.generate_swire_labels(swire_names, swire_coords, overwrite=False, field=field)
(_, atlas_test_sets), (_, swire_test_sets) = pipeline.generate_data_sets(swire_coords, swire_labels, overwrite=False, field=field)
cids = list(pipeline.cross_identify_all(swire_names, swire_coords, swire_labels, swire_test_sets, swire_labels[:, 0], field=field))
atlas_to_swire = collections.defaultdict(dict) # ATLAS -> predictor -> SWIRE
swire_name_to_coord = {}
for name, coord in zip(swire_names, swire_coords):
swire_name_to_coord[name] = coord
atlas_to_swire_expert = {}
key_to_atlas = {}
atlas_to_ras = {}
atlas_to_decs = {}
id_to_atlas = {}
atlas_to_id = {}
atlas_to_zooniverse_id = {}
if field == 'cdfs':
table = astropy.io.ascii.read(pipeline.TABLE_PATH)
for row in table:
name = row['Component Name (Franzen)']
if not name:
continue
id_to_atlas[row['Component ID (Franzen)']] = name
atlas_to_id[name] = row['Component ID (Franzen)']
atlas_to_zooniverse_id[name] = row['Component Zooniverse ID (RGZ)']
key_to_atlas[row['Key']] = name
swire = row['Source SWIRE (Norris)']
atlas_to_swire_expert[name] = swire
atlas_to_ras[name] = row['Component RA (Franzen)']
atlas_to_decs[name] = row['Component DEC (Franzen)']
else:
swire_scoords = astropy.coordinates.SkyCoord(ra=swire_coords[:, 0],
dec=swire_coords[:, 1],
unit='deg')
with astropy.io.fits.open(pipeline.MIDDELBERG_TABLE4_PATH) as elais_components_fits:
elais_components = elais_components_fits[1].data
component_to_name = {}
for i, component in enumerate(elais_components):
name = component['ATELAIS']
id_to_atlas[component['CID']] = name
atlas_to_id[name] = component['CID']
atlas_to_zooniverse_id[name] = ''
key_to_atlas[i] = name
coord = astropy.coordinates.SkyCoord(
ra='{} {} {}'.format(component['RAh'], component['RAm'], component['RAs']),
dec='-{} {} {}'.format(component['DEd'], component['DEm'], component['DEs']),
unit=('hourangle', 'deg'))
coord = (coord.ra.deg, coord.dec.deg)
atlas_to_ras[name] = coord[0]
atlas_to_decs[name] = coord[1]
# Load SWIRE cross-identification from Table 5.
with open(pipeline.MIDDELBERG_TABLE5_PATH) as elais_file:
lines = [line.split('|') for line in elais_file]
for line in lines:
if 'ATELAISJ' not in line[0]:
continue
line_cids = line[1]
if 'C0' not in line_cids and 'C1' not in line_cids:
continue
line_cids = [cid.strip() for cid in line_cids.split(',')]
swire_coord_re = re.search(r'SWIRE4J(\d\d)(\d\d)(\d\d\.\d\d)(-\d\d)(\d\d)(\d\d\.\d)', line[2])
if not swire_coord_re:
continue
swire_coord_list = swire_coord_re.groups()
coord = astropy.coordinates.SkyCoord(
ra='{} {} {}'.format(*swire_coord_list[:3]),
dec='{} {} {}'.format(*swire_coord_list[3:]),
unit=('hourangle', 'deg'))
# Nearest SWIRE...
seps = coord.separation(swire_scoords)
nearest = numpy.argmin(seps)
dist = seps[nearest]
if dist.deg > 5 / 60 / 60:
continue
name = swire_names[nearest]
for cid in line_cids:
atlas_to_swire_expert[id_to_atlas[cid]] = name
atlas_to_rgz = {}
atlas_to_radio_consensus = {}
atlas_to_ir_consensus = {}
if field == 'cdfs':
for row in astropy.io.ascii.read(pipeline.RGZ_PATH):
name = id_to_atlas[row['atlas_id']]
atlas_to_radio_consensus[name] = row['consensus.radio_level']
atlas_to_ir_consensus[name] = row['consensus.ir_level']
atlas_to_rgz[name] = row['SWIRE.designation']
titlemap = {
'RGZ & Norris & compact': 'Compact',
'RGZ & Norris & resolved': 'Resolved',
'RGZ & Norris': 'All',
'RGZ & compact': 'Compact',
'RGZ & resolved': 'Resolved',
'RGZ': 'All',
}
known_predictors = set()
for cid in cids:
if cid.labeller == 'norris' and 'Norris' not in cid.dataset_name:
continue
if cid.classifier in {'Groundtruth', 'Random'}:
continue
if field == 'cdfs':
atlas_keys = atlas_test_sets[:, pipeline.SET_NAMES['RGZ & Norris'], cid.quadrant].nonzero()[0]
else:
atlas_keys = atlas_test_sets[:, 0, 0].nonzero()[0]
atlas_to_swire_predictor = dict(zip(cid.radio_names, cid.ir_names))
n_total = 0
n_correct = 0
n_skipped = 0
if 'Norris' in cid.dataset_name and cid.labeller == 'rgz':
labeller = 'RGZ N'
continue
elif cid.labeller == 'rgz':
labeller = 'RGZ'
else:
labeller = 'Norris'
predictor_name = '{}({} / {})'.format(
{'LogisticRegression': 'LR', 'CNN': 'CNN', 'RandomForestClassifier': 'RF'}[cid.classifier],
labeller, titlemap[cid.dataset_name])
known_predictors.add(predictor_name)
for i in atlas_keys:
name = key_to_atlas[i]
swire_predictor = atlas_to_swire_predictor.get(name, '')
atlas_to_swire[name][predictor_name] = swire_predictor
known_predictors = sorted(known_predictors)
atlases = sorted(atlas_to_swire)
ras = []
decs = []
expert_xids = []
expert_xid_ras = []
expert_xid_decs = []
rgzs = []
rgz_ras = []
rgz_decs = []
rcs = []
ircs = []
cids = []
zids = []
predictor_columns = collections.defaultdict(list)
predictor_ras = collections.defaultdict(list)
predictor_decs = collections.defaultdict(list)
for atlas in atlases:
for predictor in known_predictors:
predictor_columns[predictor].append(atlas_to_swire[atlas].get(predictor, ''))
predictor_ras[predictor].append(swire_name_to_coord.get(atlas_to_swire[atlas].get(predictor, ''), (None, None))[0])
predictor_decs[predictor].append(swire_name_to_coord.get(atlas_to_swire[atlas].get(predictor, ''), (None, None))[1])
ras.append(atlas_to_ras[atlas])
decs.append(atlas_to_decs[atlas])
rgzs.append(atlas_to_rgz.get(atlas, ''))
cids.append(atlas_to_id[atlas])
zids.append(atlas_to_zooniverse_id[atlas])
rgz_ras.append(swire_name_to_coord.get(atlas_to_rgz.get(atlas, ''), (None, None))[0])
rgz_decs.append(swire_name_to_coord.get(atlas_to_rgz.get(atlas, ''), (None, None))[1])
rcs.append(atlas_to_radio_consensus.get(atlas, 0.0))
ircs.append(atlas_to_ir_consensus.get(atlas, 0.0))
expert_xids.append(atlas_to_swire_expert.get(atlas, ''))
expert_xid_ras.append(swire_name_to_coord.get(atlas_to_swire_expert.get(atlas, ''), (None, None))[0])
expert_xid_decs.append(swire_name_to_coord.get(atlas_to_swire_expert.get(atlas, ''), (None, None))[1])
expert = 'Norris' if field == 'cdfs' else 'Middelberg'
table = astropy.table.Table(
data=[atlases, ras, decs,
cids, zids,
expert_xids, expert_xid_ras, expert_xid_decs,
rgzs, rgz_ras, rgz_decs,
rcs, ircs] + [k for p in known_predictors for k in (predictor_columns[p], predictor_ras[p], predictor_decs[p])],
names=['ATLAS', 'RA', 'Dec',
'CID', 'Zooniverse ID',
expert, expert + ' RA', expert + ' Dec',
'RGZ', 'RGZ RA', 'RGZ Dec',
'RGZ radio consensus', 'RGZ IR consensus'] + [k for p in known_predictors for k in (p, p + ' RA', p + ' Dec')])
table['RGZ radio consensus'].format = '{:.4f}'
table['RGZ IR consensus'].format = '{:.4f}'
table.write('/Users/alger/data/Crowdastro/predicted_cross_ids_table_21_03_18_{}.csv'.format(field), format='csv')
table.write('/Users/alger/data/Crowdastro/predicted_cross_ids_table_21_03_18_{}.tex'.format(field), format='latex')
def write(self, *args, **kwargs):
table = self[:]
table.write(*args, **kwargs)
def print_table(field='cdfs'):
titlemap = {
'RGZ & Norris & compact': 'Compact',
'RGZ & Norris & resolved': 'Resolved',
'RGZ & Norris': 'All',
'RGZ & compact': 'Compact',
'RGZ & resolved': 'Resolved',
'RGZ': 'All',
}
lr_predictions = itertools.chain(
pipeline.unserialise_predictions(
pipeline.WORKING_DIR + 'LogisticRegression_norris_{}_predictions'.format(field)),
pipeline.unserialise_predictions(
pipeline.WORKING_DIR + 'LogisticRegression_rgz_{}_predictions'.format(field)))
rf_predictions = itertools.chain(
pipeline.unserialise_predictions(
pipeline.WORKING_DIR + 'RandomForestClassifier_norris_{}_predictions'.format(field)),
pipeline.unserialise_predictions(
pipeline.WORKING_DIR + 'RandomForestClassifier_rgz_{}_predictions'.format(field)))
cnn_predictions = itertools.chain(
pipeline.unserialise_predictions(
pipeline.WORKING_DIR + 'CNN_norris_{}_predictions'.format(field)),
pipeline.unserialise_predictions(
pipeline.WORKING_DIR + 'CNN_rgz_{}_predictions'.format(field)))
swire_names, swire_coords, _ = pipeline.generate_swire_features(overwrite=False, field=field)
swire_labels = pipeline.generate_swire_labels(swire_names, swire_coords, overwrite=False, field=field)
_, (_, swire_test_sets) = pipeline.generate_data_sets(swire_coords, swire_labels, overwrite=False, field=field)
swire_names = numpy.array(swire_names)
swire_coords = numpy.array(swire_coords)
predictions_map = collections.defaultdict(dict) # SWIRE -> predictor -> probability
swire_coords_map = {}
swire_expert_map = {}
swire_rgz_map = {}
known_predictors = set()
for classifier, predictions_ in [['LR', lr_predictions], ['CNN', cnn_predictions], ['RF', rf_predictions]]:
for predictions in predictions_:
dataset_name = predictions.dataset_name
labeller = predictions.labeller
if labeller == 'rgz' and 'Norris' in dataset_name:
labeller = 'RGZ N'
continue
labeller = labeller.title() if labeller == 'norris' else labeller.upper()
predictor_name = '{}({} / {})'.format(classifier, labeller, titlemap[dataset_name])
if field == 'cdfs':
swire_names_ = swire_names[swire_test_sets[:, pipeline.SET_NAMES['RGZ'], predictions.quadrant]]
swire_coords_ = swire_coords[swire_test_sets[:, pipeline.SET_NAMES['RGZ'], predictions.quadrant]]
swire_labels_ = swire_labels[swire_test_sets[:, pipeline.SET_NAMES['RGZ'], predictions.quadrant]]
else:
swire_names_ = swire_names[swire_test_sets[:, 0, 0]]
swire_coords_ = swire_coords[swire_test_sets[:, 0, 0]]
swire_labels_ = swire_labels[swire_test_sets[:, 0, 0]]
assert predictions.probabilities.shape[0] == len(swire_names_), \
'expected {}, got {}'.format(predictions.probabilities.shape[0], len(swire_names_))
for name, coords, prediction, label in zip(swire_names_, swire_coords_, predictions.probabilities, swire_labels_):
predictions_map[name][predictor_name] = prediction
swire_coords_map[name] = coords
swire_expert_map[name] = label[0]
swire_rgz_map[name] = label[1]
known_predictors.add(predictor_name)
known_predictors = sorted(known_predictors)
swires = sorted(predictions_map)
ras = []
decs = []
is_expert_host = []
is_rgz_host = []
predictor_columns = collections.defaultdict(list)
for swire in swires:
for predictor in known_predictors:
predictor_columns[predictor].append(predictions_map[swire].get(predictor, ''))
ras.append(swire_coords_map[swire][0])
decs.append(swire_coords_map[swire][1])
is_expert_host.append(['no', 'yes'][swire_expert_map[swire]])
is_rgz_host.append(['no', 'yes'][swire_rgz_map[swire]])
table = astropy.table.Table(
data=[swires, ras, decs, is_expert_host, is_rgz_host] + [predictor_columns[p] for p in known_predictors],
names=['SWIRE', 'RA', 'Dec', 'Expert host', 'RGZ host'] + known_predictors)
table.write('/Users/alger/data/Crowdastro/predicted_swire_table_{}_21_03_18.csv'.format(field), format='csv')
for p in known_predictors:
table[p].format = '{:.4f}'
table.write('/Users/alger/data/Crowdastro/predicted_swire_table_{}_21_03_18.tex'.format(field), format='latex')
def extract(inputs, pixel_scale, filename, noisefile, outputfile, finalfits,
num_sections, height, width, x, y):
logger.info(f"Will source extract galaxies from noise file {noisefile}")
# run sextractor on noise image.
cmd = '/nfs/slac/g/ki/ki19/deuce/AEGIS/ismael/WLD/params/sextractor-2.25.0/src/sex {} -c {} -CATALOG_NAME {} ' \
'-PARAMETERS_NAME {} -FILTER_NAME {} -STARNNW_NAME {}'.format(
noisefile, inputs['config_file'], outputfile, inputs['param_file'], inputs['filter_file'],
inputs['starnnw_file'])
logger.info(f"With cmd: {cmd}")
subprocess.run(cmd, shell=True)
logger.success(f"Successfully source extracted {noisefile}!")
# read noise image to figure out image bounds.
fits_section = fitsio.FITS(noisefile)
stamp = fits_section[0].read()
img_width = stamp.shape[0] # pixels
img_height = stamp.shape[1]
assert img_width == width / num_sections and img_height == height / num_sections, \
"Something is wrong with the heights specified."
logger.info(
f"The image width and height from the stamp are {img_width} and {img_height} respectively."
)
# read results obtained (table obtained either from combining or from single.)
cat = descwl.output.Reader(filename).results
table = cat.table
detected, matched, indices, distance = cat.match_sextractor(outputfile)
logger.success(
f"Successfully matched catalogue with source extractor from sextract output: {outputfile}"
)
# convert to arcsecs and relative to this image's center (not to absolute, before w/ respect to corner.)
detected['X_IMAGE'] = (detected['X_IMAGE'] - 0.5 * img_width -
0.5) * pixel_scale
detected['Y_IMAGE'] = (detected['Y_IMAGE'] - 0.5 * img_height -
0.5) * pixel_scale
# adjust to absolute image center if necessary, both for measured centers and catalogue centers.
detected['X_IMAGE'] += x * (width * pixel_scale) # need to use arcsecs
detected['Y_IMAGE'] += y * (height * pixel_scale)
table['dx'] += x * (width * pixel_scale)
table['dy'] += y * (height * pixel_scale)
# also adjust the xmin,xmax, ymin, ymax bounding box edges, because why not?
# remember these are in pixels; xmin, xmax relative to left edge; ymin,ymax relative to right edge.
table['xmin'] += int(x * width + width / 2 - img_width / 2)
table['xmax'] += int(x * width + width / 2 - img_width / 2)
table['ymin'] += int(x * height + height / 2 - img_height / 2)
table['ymax'] += int(x * height + height / 2 - img_height / 2)
# convert second moments to arcsecs.
# Not adjust relative to center, because only used for sigma calculation.
detected['X2_IMAGE'] *= pixel_scale**2
detected['Y2_IMAGE'] *= pixel_scale**2
detected['XY_IMAGE'] *= pixel_scale**2
# calculate size from moments X2_IMAGE,Y2_IMAGE,XY_IMAGE -> remember in pixel**2 so have to convert to arcsecs.
sigmas = []
for x2, y2, xy in zip(detected['X2_IMAGE'], detected['Y2_IMAGE'],
detected['XY_IMAGE']):
second_moments = np.array([[x2, xy], [xy, y2]])
sigma = np.linalg.det(second_moments)**(+1. / 4) # should be a PLUS.
sigmas.append(sigma)
SIGMA = astropy.table.Column(name='SIGMA', data=sigmas)
detected.add_column(SIGMA)
# find the indices of the ambiguous blends.
logger.info("Finding indices/ids that are ambiguously blended")
ambg_blends = detected_ambiguous_blends(table, indices, detected)
logger.success("All indices have been found")
logger.info(
f"Adding column to original table and writing it to {finalfits}")
ambiguous_blend_column = []
for i, gal_row in enumerate(table):
if i in ambg_blends:
ambiguous_blend_column.append(True)
else:
ambiguous_blend_column.append(False)
column = astropy.table.Column(name='ambig_blend',
data=ambiguous_blend_column)
table.add_column(column)
logger.debug(
f"Number of galaxies in table from file {finalfits} is: {len(table)}")
table.write(finalfits)
def calculate_fiber_acceptance_fraction(focal_x,
focal_y,
wavelength,
source,
atmosphere,
instrument,
source_types=None,
source_fraction=None,
source_half_light_radius=None,
source_minor_major_axis_ratio=None,
source_position_angle=None,
oversampling=32,
saved_images_file=None,
saved_table_file=None):
"""Calculate the acceptance fraction for a single fiber.
The behavior of this function is customized by the instrument.fiberloss
configuration parameters. When instrument.fiberloss.method == 'table',
pre-tabulated values are returned using source.type as the key and
all other parameters to this function are ignored.
When instrument.fiberloss.method == 'galsim', fiberloss is calculated
on the fly using the GalSim package via :class:`GalsimFiberlossCalculator`
to model the PSF components and source profile and perform the convolutions.
To efficiently calculate fiberloss fractions for multiple sources with
GalSim, use :class:`GalsimFiberlossCalculator` directly instead of
repeatedly calling this method. See :mod:`specsim.quickfiberloss` for an
example of this approach.
Parameters
----------
focal_x : :class:`astropy.units.Quantity`
X coordinate of the fiber center in the focal plane with length units.
focal_y : :class:`astropy.units.Quantity`
Y coordinate of the fiber center in the focal plane with length units.
wavelength : :class:`astropy.units.Quantity`
Array of simulation wavelengths (with length units) where the fiber
acceptance fraction should be tabulated.
source : :class:`specsim.source.Source`
Source model to use for the calculation.
atmosphere : :class:`specsim.atmosphere.Atmosphere`
Atmosphere model to use for the calculation.
instrument : :class:`specsim.instrument.Instrument`
Instrument model to use for the calculation.
source_types : array or None
Array of source type names that identify which tabulated fiberloss
fraction should be used for each fiber with the ``table`` method.
Each name should already be defined as a key in the
``instruments.fiberloss.table.paths`` configuration.
Ignored for the ``galsim`` method.
source_fraction : array or None
Array of shape (num_fibers, 2). See
:meth:`GalsimFiberlossCalculator.create_source` for details.
Ignored for the ``table`` method.
source_half_light_radius : array or None
Array of shape (num_fibers, 2). See
:meth:`GalsimFiberlossCalculator.create_source` for details.
Ignored for the ``table`` method.
source_minor_major_axis_ratio : array or None
Array of shape (num_fibers, 2). See
:meth:`GalsimFiberlossCalculator.create_source` for details.
Ignored for the ``table`` method.
source_position_angle : array or None
Array of shape (num_fibers, 2). See
:meth:`GalsimFiberlossCalculator.create_source` for details.
Ignored for the ``table`` method.
oversampling : int
Oversampling factor to use for anti-aliasing the fiber aperture.
Ignored for the ``table`` method.
saved_images_file : str or None
See :meth:`GalsimFiberlossCalculator.calculate`.
Ignored for the ``table`` method.
saved_table_file : str or None
Write a table of calculated values to a file with this name. The
extension determines the file format, and .ecsv is recommended.
The saved file can then be used as a pre-tabulated input with
instrument.fiberloss.method = 'table'.
Returns
-------
numpy array
Array of fiber acceptance fractions (dimensionless) at each of the
input wavelengths.
"""
num_fibers = len(focal_x)
if len(focal_y) != num_fibers:
raise ValueError('Arrays focal_x and focal_y must have same length.')
# Use pre-tabulated fiberloss fractions when requested.
if instrument.fiberloss_method == 'table':
if source_types is None:
# Use same source type for all fibers.
return instrument.fiber_acceptance_dict[source.type_name][
np.newaxis, :]
elif len(source_types) != num_fibers:
raise ValueError('Unexpected shape for source_types.')
floss = np.empty((num_fibers, len(wavelength)))
for i, type_name in enumerate(source_types):
floss[i] = instrument.fiber_acceptance_dict[type_name]
return floss
# Otherwise, use GalSim or the fastfiberacceptance calibrated on galsim
# to calculate fiberloss fractions on the fly...
# Initialize the grid of wavelengths where the fiberloss will be
# calculated.
num_wlen = instrument.fiberloss_num_wlen
wlen_unit = wavelength.unit
wlen_grid = np.linspace(wavelength.value[0], wavelength.value[-1],
num_wlen) * wlen_unit
# Calculate the focal-plane optics at the fiber locations.
scale, blur, offset = instrument.get_focal_plane_optics(
focal_x, focal_y, wlen_grid)
# Calculate the atmospheric seeing at each wavelength.
seeing_fwhm = atmosphere.get_seeing_fwhm(wlen_grid).to(u.arcsec).value
# Replicate source parameters from the source config if they are not
# provided via args. If they are provided, check for the expected shape.
if source_fraction is None:
source_fraction = np.tile(
[source.disk_fraction, source.bulge_fraction], [num_fibers, 1])
elif source_fraction.shape != (num_fibers, 2):
raise ValueError('Unexpected shape for source_fraction.')
if source_half_light_radius is None:
source_half_light_radius = np.tile([
source.disk_shape.half_light_radius.to(u.arcsec).value,
source.bulge_shape.half_light_radius.to(u.arcsec).value
], [num_fibers, 1])
elif source_half_light_radius.shape != (num_fibers, 2):
raise ValueError('Unexpected shape for source_half_light_radius.')
if source_minor_major_axis_ratio is None:
source_minor_major_axis_ratio = np.tile([
source.disk_shape.minor_major_axis_ratio,
source.bulge_shape.minor_major_axis_ratio
], [num_fibers, 1])
elif source_minor_major_axis_ratio.shape != (num_fibers, 2):
raise ValueError('Unexpected shape for source_minor_major_axis_ratio.')
if source_position_angle is None:
source_position_angle = np.tile([
source.disk_shape.position_angle.to(u.deg).value,
source.bulge_shape.position_angle.to(u.deg).value
], [num_fibers, 1])
elif source_position_angle.shape != (num_fibers, 2):
raise ValueError('Unexpected shape for source_position_angle.')
fiberloss_grid = None
# choose here how to compute things
if instrument.fiberloss_method == 'fastsim':
scale_um_per_arcsec = scale.to(u.um / u.arcsec).value
blur_um = blur.to(u.um).value
sigma = np.zeros((offset.shape[0], offset.shape[1]))
disk_half_light_radius = np.zeros(sigma.shape)
bulge_half_light_radius = np.zeros(sigma.shape)
disk_frac = np.zeros(sigma.shape)
bulge_frac = np.zeros(sigma.shape)
for i in range(offset.shape[0]):
sigma[i] = np.sqrt((seeing_fwhm / 2.35482)**2 *
scale_um_per_arcsec[i, 0] *
scale_um_per_arcsec[i, 1] + blur_um[i]**2)
disk_half_light_radius[i] = source_half_light_radius[
i, 0] * np.ones(offset.shape[1])
bulge_half_light_radius[i] = source_half_light_radius[
i, 1] * np.ones(offset.shape[1])
disk_frac[i] = source_fraction[i, 0] * np.ones(offset.shape[1])
bulge_frac[i] = source_fraction[i, 1] * np.ones(offset.shape[1])
point_frac = 1 - disk_frac - bulge_frac
offset_um = offset.to(u.um).value
delta = np.sqrt(offset_um[:, :, 0]**2 + offset_um[:, :, 1]**2)
fiberloss_grid = np.zeros(sigma.shape)
if np.sum(point_frac) > 0:
fiberloss_grid += point_frac * instrument.fast_fiber_acceptance.value(
"POINT", sigma, delta)
if np.sum(disk_frac) > 0:
fiberloss_grid += disk_frac * instrument.fast_fiber_acceptance.value(
"DISK", sigma, delta, disk_half_light_radius)
if np.sum(bulge_frac) > 0:
fiberloss_grid += bulge_frac * instrument.fast_fiber_acceptance.value(
"BULGE", sigma, delta, bulge_half_light_radius)
else:
# Initialize a new calculator.
calc = GalsimFiberlossCalculator(
instrument.fiber_diameter.to(u.um).value,
wlen_grid.to(u.Angstrom).value, instrument.fiberloss_num_pixels,
oversampling, atmosphere.seeing_moffat_beta)
# Calculate fiberloss fractions. Note that the calculator expects arrays
# with implicit units.
fiberloss_grid = calc.calculate(seeing_fwhm,
scale.to(u.um / u.arcsec).value,
offset.to(u.um).value,
blur.to(u.um).value, source_fraction,
source_half_light_radius,
source_minor_major_axis_ratio,
source_position_angle,
saved_images_file)
# TODO: add support for saving table when num_fibers > 1.
if saved_table_file and num_fibers == 1:
meta = dict(
description='Fiberloss fraction for source "{0}"'.format(
source.name) +
' at focal (x,y) = ({0:.3f},{1:.3f})'.format(focal_x, focal_y))
table = astropy.table.Table(meta=meta)
table.add_column(
astropy.table.Column(name='Wavelength',
data=wlen_grid.value,
unit=wlen_grid.unit,
description='Observed wavelength'))
table.add_column(
astropy.table.Column(name='FiberAcceptance',
data=fiberloss_grid[0],
description='Fiber acceptance fraction'))
args = {}
if saved_table_file.endswith('.ecsv'):
args['format'] = 'ascii.ecsv'
table.write(saved_table_file, **args)
# Interpolate (linearly) to the simulation wavelength grid.
# Use scipy.interpolate instead of np.interp here to avoid looping
# over fibers.
interpolator = scipy.interpolate.interp1d(wlen_grid.value,
fiberloss_grid,
kind='linear',
axis=1,
copy=False,
assume_sorted=True)
# Both wavelength grids have the same units, by construction, so no
# conversion factor is required here.
return interpolator(wavelength.value)
column_names = ['Inner edge (pc)', 'Outer edge (pc)',
'GMC index', 'R', 'p',
'Truncation mass (M$_\odot$)',
'Largest cloud (M$_\odot$)',
'5th largest cloud (M$_\odot$)']
column_types = ['f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4']
table = Table(names=column_names, dtype=column_types)
for inneredge, outeredge in zip(inneredge, outeredge):
idx = np.where((t['RADIUS_PC'] >= inneredge) &
(t['RADIUS_PC'] < outeredge))
mass = t['MASS_GCORR'][idx].data
optresult = plf.plfit_adstat(mass / minmass)
pvec = optresult.x
# don't have to create an index for the xmin mass - defined in the fit_subset
table.add_row()
table[-1]['Inner edge (pc)'] = inneredge
table[-1]['Outer edge (pc)'] = outeredge
table[-1]['GMC index'] = pvec[0]
table[-1]['R'] = 0
table[-1]['p'] = 0
table[-1]['Truncation mass (M$_\odot$)'] = pvec[1] * minmass
table[-1]['Largest cloud (M$_\odot$)'] = mass.max()
table[-1]['5th largest cloud (M$_\odot$)'] =\
np.sort(t['MASS_GCORR'][idx])[-5]
print(table)
table.write('m83bininfo_ad.fits', overwrite=True)
def data(galaxyname, data, n_bins=1, r_nuc=0):
# Import the libraries.
from galaxies import Galaxy
from astropy.table import Table
from astropy.table import Column
import astropy
import powerlaw
import numpy as np
import astropy.table
import astropy.units as u
import matplotlib.pyplot as plt
import matplotlib as mpl
# Load its FITS file.
t = Table.read(data)
# Load the information about the galaxy.
gxy = Galaxy(galaxyname)
# Calculate the galaxy's properties.
distance = np.asarray(gxy.distance)
inclination = np.asarray(gxy.inclination)
rgal = gxy.radius(ra=(t['XPOS']), dec=(t['YPOS']))
rgal = rgal.to(u.kpc)
rpgal = np.asarray(rgal)
# Append these to the FITS table.
col_rgal = Column(name='RADIUS_KPC', data=(rgal))
t.add_column(col_rgal)
# Sort the masses according to galactocentric radius.
mass = t['MASS_EXTRAP'].data
i_sorted = np.argsort(rgal)
rgal_sorted = np.asarray(rgal[i_sorted])
mass_sorted = np.asarray(mass[i_sorted])
# Initiate a loop to calculate the bin boundaries and the indeces of these boundaries in the sorted list.
totmass = np.sum(mass) / n_bins
edge_f = 1.1 * np.max(rgal_sorted)
edges = np.zeros(n_bins - 1)
start = 0
mass_equiv = [0] #indeces for the sorted mass bins of equal mass
mass_area = [0] #indeces for the sorted mass bins of equal area
rgal_equiv = [
0, 2, 8**0.5, 12**0.5, 4, 20**0.5, 24**0.5, 28**0.5, 32**0.5, 6
]
r = 1 #equal-area radial index
e = 0 #edge index
f = 0 #loop flag to skip the mass_area loop
c = 0 #loop counter
for i in range(len(mass_sorted)):
#Find the indeces for bins of equal mass (equivalent to totmass)
if np.sum(mass_sorted[start:i]) > totmass:
edges[e] = 0.5 * (rgal_sorted[i] + rgal_sorted[i - 1])
start = i
mass_equiv = np.append(mass_equiv, i)
e = e + 1
#Find the indeces for bins of equal area (4pi kpc^2)
if rgal_sorted[i] > rgal_equiv[r] and not f:
if rgal_sorted[i] < rgal_equiv[3] and not f:
mass_area = np.append(mass_area, i)
r = r + 1
c = 0
if rgal_sorted[i] > rgal_equiv[3]:
f = 1
mass_area = np.append(mass_area, i)
mass_equiv = np.append(mass_equiv, i)
inneredge = np.concatenate(([0.000000], edges))
outeredge = np.concatenate((edges, [edge_f]))
# Create a template for a new table.
column_names = [
'Inner edge (kpc)', 'Outer edge (kpc)', 'GMC index', 'R', 'p',
'Truncation mass ($M_\mathrm{\odot}$)',
'Largest cloud ($M_\mathrm{\odot}$)',
'5th largest cloud ($M_\mathrm{\odot}$)'
]
column_types = ['f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4']
table = Table(names=column_names, dtype=column_types)
# Fill the table.
for inneredge, outeredge in zip(inneredge, outeredge):
idx = np.where((t['RADIUS_KPC'] >= inneredge)
& (t['RADIUS_KPC'] < outeredge))
mass = t['MASS_EXTRAP'][idx].data
fit = powerlaw.Fit(mass)
fit_subset = powerlaw.Fit(mass, xmin=3e5)
R, p = fit.distribution_compare('power_law', 'truncated_power_law')
table.add_row()
table[-1]['R'] = R
table[-1]['p'] = p
table[-1]['GMC index'] = -fit.alpha
table[-1]['Inner edge (kpc)'] = inneredge
table[-1]['Outer edge (kpc)'] = outeredge
table[-1]['Largest cloud ($M_\mathrm{\odot}$)'] = np.nanmax(mass)
table[-1][
'Truncation mass ($M_\mathrm{\odot}$)'] = 1 / fit.truncated_power_law.parameter2
table[-1]['5th largest cloud ($M_\mathrm{\odot}$)'] = np.sort(
t['MASS_EXTRAP'][idx])[-5]
# Write the data to a FITS file.
table.write('../Data/' + galaxyname + '_data.fits', overwrite=True)
# Plot the mass distribution trends for equal-mass bins.
t = Table.read('../Data/' + galaxyname + '_data.fits')
inneredge = t['Inner edge (kpc)'].data
outeredge = t['Outer edge (kpc)'].data
subplot_label = ('(a)', '(b)', '(c)', '(d)', '(e)', '(f)')
for i in range(len(mass_equiv) - 1):
binmass = mass_sorted[mass_equiv[i]:mass_equiv[i + 1]]
myfit = powerlaw.Fit(binmass)
R, p = myfit.distribution_compare('power_law', 'truncated_power_law')
fig = myfit.truncated_power_law.plot_ccdf(label='Truncated\nPower Law')
myfit.power_law.plot_ccdf(label='Power Law', ax=fig)
myfit.plot_ccdf(drawstyle='steps', label='Data', ax=fig)
# Format the plot.
plt.legend(loc=0)
#plt.title(galaxyname+'Equal-mass Mass Distribution, bin '+repr(i+1))
plt.ylim(ymin=10**-3)
plt.xlabel(r'$M_\mathrm{\odot}$', fontsize=20)
plt.ylabel('CCDF', fontsize=20)
mpl.rc('xtick', labelsize=16)
mpl.rc('ytick', labelsize=16)
plt.text(0.01,
0.5,
subplot_label[i],
ha='left',
va='center',
transform=fig.transAxes,
fontsize=16)
plt.text(
0.35,
0.01,
r'$M_{bin}\ =\ %e M_\mathrm{\odot}$' % (totmass) + '\n' +
r'$R_{gal}\ =\ %5.4f\ \mathrm{kpc}\ \mathrm{to}\ %5.4f\ \mathrm{kpc}$'
% (inneredge[i], outeredge[i]) + '\n' +
r'$\mathrm{R}\ =\ %5.4f,\ \mathrm{p}\ =\ %5.4f$' % (R, p) + '\n' +
r'$\alpha\ =\ %5.4f,\ M_\mathrm{0}\ =\ %5.4eM_\mathrm{\odot}$' %
(-myfit.alpha, 1 / myfit.truncated_power_law.parameter2),
ha='left',
va='bottom',
transform=fig.transAxes,
fontsize=16)
plt.savefig('../Data/' + galaxyname + '_power_law_equal_mass_' +
repr(i + 1) + '.png')
plt.close()
# Plot the mass distribution trend for equal-area bins.
for i in range(len(mass_area) - 1):
f = 0 #loop flag
if mass_area[i + 1] - mass_area[i] < 3:
f = 1
if not f:
binmass = mass_sorted[mass_area[i]:mass_area[i + 1]]
myfit = powerlaw.Fit(binmass)
R, p = myfit.distribution_compare('power_law',
'truncated_power_law')
fig = myfit.truncated_power_law.plot_ccdf(
label='Truncated\nPower Law')
myfit.power_law.plot_ccdf(label='Power Law', ax=fig)
myfit.plot_ccdf(drawstyle='steps', label='Data', ax=fig)
# Format the plot.
plt.legend(loc=0)
#plt.title(galaxyname+'Equal-area Mass Distribution, bin '+repr(i+1))
plt.ylim(ymin=10**-3)
plt.xlabel(r'$M_\mathrm{\odot}$', fontsize=20)
plt.ylabel('CCDF', fontsize=20)
mpl.rc('xtick', labelsize=16)
mpl.rc('ytick', labelsize=16)
plt.text(0.01,
0.5,
subplot_label[i],
ha='left',
va='center',
transform=fig.transAxes,
fontsize=16)
plt.text(
0.35,
0.01,
r'$R_{gal}\ =\ %5.4f\ \mathrm{kpc}\ \mathrm{to}\ %5.4f\ \mathrm{kpc}$'
% (rgal_equiv[i], rgal_equiv[i + 1]) + '\n' +
r'$\mathrm{R}\ =\ %5.4f,\ \mathrm{p}\ =\ %5.4f$' % (R, p) +
'\n' +
r'$\alpha\ =\ %5.4f,\ M_\mathrm{0}\ =\ %5.4eM_\mathrm{\odot}$'
% (-myfit.alpha, 1 / myfit.truncated_power_law.parameter2),
ha='left',
va='bottom',
transform=fig.transAxes,
fontsize=16)
plt.savefig('../Data/' + galaxyname + '_power_law_equal_area_' +
repr(i + 1) + '.png')
plt.close()
return [distance, inclination]
def table_info(table, label):
# Print basic info to console
print('\n*** {0} ***'.format(label))
print('Rows: {0}'.format(len(table)))
print('Columns: {0}'.format(table.colnames))
print('')
# Save to file for debugging
filename = 'crab_mwl_{0}.ipac'.format(label)
print('INFO: Writing {0}'.format(filename))
table.write(filename, format='ascii.ipac')
if __name__ == '__main__':
table = get_combined_table()
table_info(table, 'get_combined_table')
table = remove_flare_data(table)
table_info(table, 'remove_flare_data')
table = combine_columns(table)
table_info(table, 'combine_columns')
table = clean_up(table)
table_info(table, 'clean_up')
filename = 'crab_mwl.fits'
print('INFO: Writing {0}'.format(filename))
table.write(filename, overwrite=True)
'Inner edge (pc)', 'Outer edge (pc)', 'GMC index', 'R', 'p',
'Truncation mass (M$_\odot$)', 'Largest cloud (M$_\odot$)',
'5th largest cloud (M$_\odot$)'
]
column_types = ['f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4']
table = Table(names=column_names, dtype=column_types)
for inneredge, outeredge in zip(inneredge, outeredge):
idx = np.where((t['RADIUS_PC'] >= inneredge)
& (t['RADIUS_PC'] < outeredge))
mass = t['MASS_EXTRAP'][idx].data
#don't have to create an index for the xmin mass - defined in the fit_subset
fit = powerlaw.Fit(mass)
fit_subset = powerlaw.Fit(mass, xmin=3e5)
R, p = fit_subset.distribution_compare('power_law', 'truncated_power_law')
table.add_row()
table[-1]['Inner edge (pc)'] = inneredge
table[-1]['Outer edge (pc)'] = outeredge
table[-1]['GMC index'] = -fit_subset.alpha
table[-1]['R'] = R
table[-1]['p'] = p
table[-1][
'Truncation mass (M$_\odot$)'] = 1 / fit_subset.truncated_power_law.parameter2
table[-1]['Largest cloud (M$_\odot$)'] = mass.max()
table[-1]['5th largest cloud (M$_\odot$)'] = np.sort(
t['MASS_EXTRAP'][idx])[-5]
print(table)
#print(-fit.alpha, -fit_subset.alpha, R, p, 1/fit_subset.truncated_power_law.parameter2)
table.write('m83bininfo.fits', overwrite=True)
expec = np.array(expec)
calc = np.array(calc)
calc_err = np.array(calc_err)
y_calc = np.array(y_calc)
y_expec = np.array(y_expec)
expec_err = np.array(expec_err)
print(names)
print(expec)
print(calc)
t = Table()
t['NAMES'] = Column(names, description='MaNGA PlateIFU')
t['EXPECTED'] = Column(expec, description='gradient expected')
t['CALCULATED'] = Column(calc, description='gradient calculate')
t['CALCULATED_ERR'] = Column(calc_err,
description='error on the calculated gradient')
t['EXPECTED_ERR'] = Column(expec_err,
description='error on the expected gradient')
t['Y_INT_CALC'] = Column(y_calc,
description='y intercept of the calculated gradient')
t['Y_INT_MASS'] = Column(y_expec,
description='y intercept with the mass from Adam')
t['Y_INT_EXPEC'] = Column(y_expec,
description='y intercept of the expected gradient')
t['MASS'] = Column(y_expec, description='just the mass of the galaxy')
t['MASS_GRAD'] = Column(
y_expec,
description='gradient from the mass from the polynomial from Adam')
t.write('/home/celeste/Documents/astro_research/summer_2018/slopesv4.fits')
def t_table1_radec_xref_jhk_irac(write=False):
"""
Generates Table 1.
Current columns:
ONCvar ID : int ## note: this comes from our internal UKvar_ID column.
RA : float, degrees
DEC : float, degrees
X-ref : string # Using SIMBAD names for now.
Data quality : int (0, 1, 2) corresponding to subjective/auto/strict
Periodic : int (0, 1)
Median J, H, K, with error bars : six floats
IRAC colors from Megeath, and errors : eight floats
Class from Megeath : string {'P', 'D', 'ND', or 'na'}
Parameters
----------
write : bool, optional (default False)
Write to disk? Either way, this function returns an ATpy table.
Returns
-------
table : atpy.Table
Table 1.
"""
table = astropy.table.Table()
table.table_name = "Table 1"
columns_data_and_formats = [
('ONCvar ID', ukvar_spread.UKvar_ID, '%i'),
('R.A. (deg)', np.degrees(ukvar_spread.RA), '%.6f'),
('Decl. (deg)', np.degrees(ukvar_spread.DEC), '%.6f'),
('SIMBAD Cross-reference', ukvar_spread.SIMBAD_name, '%s'),
('Data quality flag', ukvar_spread.autovar + ukvar_spread.strict, '%i'),
('Periodic flag', ukvar_spread.periodic, '%i'),
('Median J mag', ukvar_spread.j_median, '%.3f'),
('Median J mag error', ukvar_spread.j_err_median, '%.3f'),
('Median H mag', ukvar_spread.h_median, '%.3f'),
('Median H mag error', ukvar_spread.h_err_median, '%.3f'),
('Median K mag', ukvar_spread.k_median, '%.3f'),
('Median K mag error', ukvar_spread.k_err_median, '%.3f'),
('Spitzer [3.6] mag', megeath2012_full_by_ukvar['3.6'], '%.3f'),
('Spitzer [3.6] mag error', megeath2012_full_by_ukvar['e_3.6'], '%.3f'),
('Spitzer [4.5] mag', megeath2012_full_by_ukvar['4.5'], '%.3f'),
('Spitzer [4.5] mag error', megeath2012_full_by_ukvar['e_4.5'], '%.3f'),
('Spitzer [5.8] mag', megeath2012_full_by_ukvar['5.8'], '%.3f'),
('Spitzer [5.8] mag error', megeath2012_full_by_ukvar['e_5.8'], '%.3f'),
('Spitzer [8.0] mag', megeath2012_full_by_ukvar['8'], '%.3f'),
('Spitzer [8.0] mag error', megeath2012_full_by_ukvar['e_8'], '%.3f'),
('Class (from Megeath et al. 2012)', make_megeath_class_column(), '%s') ]
column_to_format = {}
for column, data, format in columns_data_and_formats:
table[column] = data
column_to_format[column] = format
if write:
table.write(output_directory+'AJ_table_2.txt', format='ascii.basic', delimiter="&", formats=column_to_format)
return table
#plot fit for all 3 ('both')
sp.plotter(figure=1)
sp.specfit(fittype='formaldehyde_mm_radex', guesses=[95, 14.5, 4.5, 67, 4.0], limits=[(50,250), (11,17), (3,5.5), (65,70), (0.5,10)], limited=[(True, True)]*5, fixed=[False, False, False, False, False])
#only change center parameter from example
#sp.plotter.savefig('H2CO_all_radexfit.pdf')
table.add_row()
table[-1]['x']=v
table[-1]['y']=w
table[-1]['temp']=sp.specfit.modelpars[0]
table[-1]['column']=sp.specfit.modelpars[1]
table[-1]['density']=sp.specfit.modelpars[2]
table[-1]['center']=sp.specfit.modelpars[3]
table[-1]['width']=sp.specfit.modelpars[4]
table[-1]['temp errors']=sp.specfit.modelerrs[0]
table[-1]['column errors']=sp.specfit.modelerrs[1]
table[-1]['density errors']=sp.specfit.modelerrs[2]
table[-1]['center errors']=sp.specfit.modelerrs[3]
table[-1]['width errors']=sp.specfit.modelerrs[4]
table.write('grs1915H2COparameters.fits', overwrite=True)
plt.show()
t=Table.read('grs1915H2COparameters.fits')
plt.scatter(t['x'], t['y'], c=t['temp'], marker='o', cmap='hot', edgecolor='none')
plt.colorbar(label='Temperature(K)')
plt.savefig('grs1915H2COtempmap.pdf')
def t_table2_variability_periods_periodics_bymegeathclass(write=False):
"""
Generates Table 2, which comes in three pieces.
Note that we're only using periodic stars here!
TODO: add in flags for whether a given star has valid
J, H, or K data, respectively
Current columns:
N_j, N_h, N_k : three integers
Delta-J, H, K: three floats
Delta-J-H, J-K, H-K: three floats
Color slopes (JH, HK, JHK): three floats
Stetson : one float
Stetson_choice : one string
Period : one float
Data quality : int (0, 1, 2) corresponding to subjective/auto/strict
Parameters
----------
write : bool, optional (default False)
Write to disk? Either way, this function returns an ATpy table.
"""
table = astropy.table.Table()
table.table_name = "Table 2"
periodics = ukvar_spread.where(ukvar_spread.periodic != 0)
periodic_periods = ukvar_periods[ukvar_spread.periodic != 0]
megeath_class_by_periodics = make_megeath_class_column()[ukvar_spread.periodic != 0]
# Do some stuff where we blank out color slopes that are no good
jhk_slope_column = periodics.jhk_slope
jhk_slope_error = periodics.jhk_slope_err
jhk_slope_column[~np.in1d(periodics.SOURCEID,
jhk_slope_reference.SOURCEID)] = np.nan
jhk_slope_error[~np.in1d(periodics.SOURCEID,
jhk_slope_reference.SOURCEID)] = np.nan
jjh_slope_column = periodics.jjh_slope
jjh_slope_error = periodics.jjh_slope_err
jjh_slope_column[~np.in1d(periodics.SOURCEID,
jh_slope_reference.SOURCEID)] = np.nan
jjh_slope_error[~np.in1d(periodics.SOURCEID,
jh_slope_reference.SOURCEID)] = np.nan
khk_slope_column = periodics.khk_slope
khk_slope_error = periodics.khk_slope_err
khk_slope_column[~np.in1d(periodics.SOURCEID,
hk_slope_reference.SOURCEID)] = np.nan
khk_slope_error[~np.in1d(periodics.SOURCEID,
hk_slope_reference.SOURCEID)] = np.nan
columns_data_and_formats = [
('ONCvar ID', periodics.UKvar_ID, '%i'),
('N_J', periodics.N_j, '%i'),
('N_H', periodics.N_h, '%i'),
('N_K', periodics.N_k, '%i'),
('J mag range (robust)', periodics.j_ranger, '%.3f'),
('H mag range (robust)', periodics.h_ranger, '%.3f'),
('K mag range (robust)', periodics.k_ranger, '%.3f'),
('J-H range (robust)', periodics.jmh_ranger, '%.3f'),
('H-K range (robust)', periodics.hmk_ranger, '%.3f'),
('(J-H), (H-K) color slope', jhk_slope_column, '%.3f'),
('jhk_slope_error', jhk_slope_error, '%.3f'),
('J, (J-H) color slope', jjh_slope_column, '%.3f'),
('jjh_slope_error', jjh_slope_error, '%.3f'),
('K, (H-K) color slope', khk_slope_column, '%.3f'),
('khk_slope_error', khk_slope_error, '%.3f'),
('Stetson Variability Index', periodics.Stetson, '%.3f'),
('Bands used to compute Stetson', periodics.Stetson_choice, '%s'),
('Best-fit period', periodic_periods, '%.4f'),
('Data quality flag', periodics.autovar + periodics.strict, '%i'),
('Class', megeath_class_by_periodics, '%s')
]
column_to_format = {}
for column, data, format in columns_data_and_formats:
table[column] = data
column_to_format[column] = format
if write:
table.write(output_directory+"AJ_table_4.txt", format='ascii.basic', delimiter="&", formats=column_to_format)
return table
"GMC index",
"R",
"p",
"Truncation mass (M$_\odot$)",
"Largest cloud (M$_\odot$)",
"5th largest cloud (M$_\odot$)",
]
column_types = ["f4", "f4", "f4", "f4", "f4", "f4", "f4", "f4"]
table = Table(names=column_names, dtype=column_types)
for inneredge, outeredge in zip(inneredge, outeredge):
idx = np.where((t["RADIUS_PC"] >= inneredge) & (t["RADIUS_PC"] < outeredge))
mass = t["MASS_EXTRAP"][idx].data
# don't have to create an index for the xmin mass - defined in the fit_subset
fit = powerlaw.Fit(mass)
fit_subset = powerlaw.Fit(mass, xmin=3e5)
R, p = fit_subset.distribution_compare("power_law", "truncated_power_law")
table.add_row()
table[-1]["Inner edge (pc)"] = inneredge
table[-1]["Outer edge (pc)"] = outeredge
table[-1]["GMC index"] = -fit_subset.alpha
table[-1]["R"] = R
table[-1]["p"] = p
table[-1]["Truncation mass (M$_\odot$)"] = 1 / fit_subset.truncated_power_law.parameter2
table[-1]["Largest cloud (M$_\odot$)"] = mass.max()
table[-1]["5th largest cloud (M$_\odot$)"] = np.sort(t["MASS_EXTRAP"][idx])[-5]
print(table)
# print(-fit.alpha, -fit_subset.alpha, R, p, 1/fit_subset.truncated_power_law.parameter2)
table.write("m83bininfo.fits", overwrite=True)
def t_table3_variability_nonperiodics_bymegeathclass(write=False):
"""
Generates Table 3, which also comes in three pieces.
Only non-periodic stars; has many of the same columns as Table 2.
Current columns:
N_j, N_h, N_k : three integers
Delta-J, H, K: three floats
Delta-J-H, J-K, H-K: three floats
Color slopes (JH, HK, JHK): three floats
Stetson : one float
Stetson_choice : one string
Data quality : int (0, 1, 2) corresponding to subjective/auto/strict
Parameters
----------
write : bool, optional (default False)
Write to disk? Either way, this function returns an ATpy table.
"""
table = astropy.table.Table()
table.table_name = "Table 3"
nonperiodics = ukvar_spread.where(ukvar_spread.periodic == 0)
megeath_class_by_nonperiodics = make_megeath_class_column()[ukvar_spread.periodic == 0]
# Do some stuff where we blank out color slopes that are no good
jhk_slope_column = nonperiodics.jhk_slope
jhk_slope_error = nonperiodics.jhk_slope_err
jhk_slope_column[~np.in1d(nonperiodics.SOURCEID, jhk_slope_reference.SOURCEID)] = np.nan
jhk_slope_error[~np.in1d(nonperiodics.SOURCEID, jhk_slope_reference.SOURCEID)] = np.nan
jjh_slope_column = nonperiodics.jjh_slope
jjh_slope_error = nonperiodics.jjh_slope_err
jjh_slope_column[~np.in1d(nonperiodics.SOURCEID, jh_slope_reference.SOURCEID)] = np.nan
jjh_slope_error[~np.in1d(nonperiodics.SOURCEID, jh_slope_reference.SOURCEID)] = np.nan
khk_slope_column = nonperiodics.khk_slope
khk_slope_error = nonperiodics.khk_slope_err
khk_slope_column[~np.in1d(nonperiodics.SOURCEID, hk_slope_reference.SOURCEID)] = np.nan
khk_slope_error[~np.in1d(nonperiodics.SOURCEID, hk_slope_reference.SOURCEID)] = np.nan
columns_data_and_formats = [
('ONCvar ID', nonperiodics.UKvar_ID, '%i'),
('N_J', nonperiodics.N_j, '%i'),
('N_H', nonperiodics.N_h, '%i'),
('N_K', nonperiodics.N_k, '%i'),
('J mag range (robust)', nonperiodics.j_ranger, '%.3f'),
('H mag range (robust)', nonperiodics.h_ranger, '%.3f'),
('K mag range (robust)', nonperiodics.k_ranger, '%.3f'),
('J-H range (robust)', nonperiodics.jmh_ranger, '%.3f'),
('H-K range (robust)', nonperiodics.hmk_ranger, '%.3f'),
('(J-H), (H-K) color slope', jhk_slope_column, '%.3f'),
('jhk_slope_error', jhk_slope_error, '%.3f'),
('J, (J-H) color slope', jjh_slope_column, '%.3f'),
('jjh_slope_error', jjh_slope_error, '%.3f'),
('K, (H-K) color slope', khk_slope_column, '%.3f'),
('khk_slope_error', khk_slope_error, '%.3f'),
('Stetson Variability Index', nonperiodics.Stetson, '%.3f'),
('Bands used to compute Stetson', nonperiodics.Stetson_choice, '%s'),
('Data quality flag', nonperiodics.autovar + nonperiodics.strict, '%i'),
('Class', megeath_class_by_nonperiodics, '%s')
]
# Now split it into three pieces and compute medians!
column_to_format = {}
for column, data, format in columns_data_and_formats:
table[column] = data
column_to_format[column] = format
if write:
table.write(output_directory+"AJ_table_5.txt", format='ascii.basic', delimiter="&", formats=column_to_format)
return table
