def matches_sortCSV(gaia_catalogue,
search_radius=0.000048481,
db_path='../BDNYCdb_practice/bdnycdev_copy.db',
save_all=False,
save_needsreview=False):
"""Sort Gaia data into separate dataframes row by row, one for 'matches', one for 'new objects', and one for objects with more than one match that need review. Returns dataframes for matches and new objects.
Required Parameters:
gaia_catalogue: pandas dataframe of data. Requires RA and DEC for matching against the database
Optional Parameters:
search_radius: __float__ Search radius for cross matching RA/DEC against database.
Default is 0.000048481 radians (10 arcseconds).
db_path: __string__ Local path to database.
Default is '../BDNYCdb_practice/bdnycdev_copy.db'.
save_all: __boolean__ Saves all dataframes as CSV files. Default is False
save_needsreview: __boolean__ Saves only the needs_review dataframe as a CSV file.
Default is False.
"""
db = astrodb.Database('../BDNYCdb_practice/bdnycdev_copy.db')
# matches will store gaia data for objects in BDNYC database
matches = pd.DataFrame(columns=np.insert(
gaia_catalogue.columns.values, 0, 'source_id', axis=0))
# new_objects will store gaia data for objects that do not exist in BDNYC database
new_objects = pd.DataFrame(columns=gaia_catalogue.columns.values)
# needs_review will store gaia data for objects that have too many matched in the database and need further review
needs_review = pd.DataFrame(columns=gaia_catalogue.columns.values)
# ===============================================
# sort each row of gaia data into matches/new_objects using celestial coordinates: right ascension (ra/RA) and declination (dec/DEC)
# ===============================================
for i in range(len(gaia_catalogue)):
results = db.search(
(gaia_catalogue['RA'][i], gaia_catalogue['DEC'][i]),
'sources',
radius=0.000048481,
fetch=True)
if len(results) == 1:
matches = matches.append(gaia_catalogue.loc[[i]])
matches['source_id'].loc[i] = results['id'][0]
elif len(results) > 1:
# if there is MORE THAN ONE result, just print a note
needs_review = needs_review.append(gaia_catalogue.loc[[i]])
else:
new_objects = new_objects.append(gaia_catalogue.loc[[i]])
if save_all == True:
matches.to_csv('matches.csv')
new_objects.to_csv('new_objects.csv')
needs_review.to_csv('needs_review.csv')
print('matches, new_objects, and needs_review saved as CSV files.')
if save_needsreview == True:
needs_review.to_csv('needs_review.csv')
return matches, new_objects
def genTheList(x, y):
dbpath = '/Users/jamescrook/Desktop/plotterApp/bdnycdev_20160723_kc.db'
db = astrodb.Database(dbpath)
x_loc = getLocale(x)[0]
x_id = getLocale(x)[1]
y_loc = getLocale(y)[0]
y_id = getLocale(y)[1]
xDica = db.query('select {}, {} from {}'.format(x_id, x, x_loc),
fmt='dict')
yDica = db.query('select {}, {} from {}'.format(y_id, y, y_loc),
fmt='dict')
xDic = {}
yDic = {}
xList = []
yList = []
for a in xDica:
xDic[a[x_id]] = a[x]
for a in yDica:
yDic[a[y_id]] = a[y]
for v in xDic:
try:
if (yDic[v] is not None) and (xDic[v] is not None):
xList.append(xDic[v])
yList.append(yDic[v])
except:
pass
return xList, yList
def make_model_db(model_grid_name, model_atmosphere_db, model_grid=None, grid_data='spec',
param_lims=[('teff', 400, 1600, 50), ('logg', 3.5, 5.5, 0.5)], fill_holes=True, bands=[],
rebin_models=True, use_pandas=False):
"""
Given a **model_grid_name**, returns the grid from the model_atmospheres.db as a Pandas DataFrame
Parameters
----------
model_grid_name: str
The name of the model grid table in the model_atmospheres.db SQL file, e.g. 'bt_settl_2013'
model_atmosphere_db: str
The path to model_atmospheres.db
model_grid: instance
Default is None. This is the str or variable name given to a pickle file path of the model grid if available.
grid_data: 'spec' or 'phot'
Returns a grid of spectra or synthetic photometry
param_lims: list of tuples (optional)
A list of tuples with the parameter name, lower limit, upper limit, and increment for each parameter to be
constrained, e.g. [('teff',400,800,100),('logg',4,5,0.5)]
fill_holes: bool
Default is True. Fills the holes in the model grid to give full coverage over your parameters
bands: no one knows!
rebin_models: array or bool
The wavelength array to which all model spectra should be rebinned OR True if random rebinning is desired
may need to update u.rebin_spec if not working.
use_pandas: bool
Default is False. Uses a pandas dataframe as output.
Returns
-------
models: Pandas DataFrame
The resulting model grid as a Pandas DataFrame
"""
# If not using model grid form a pickle file, load the model_atmospheres database and pull all the data from
# the specified table
if model_grid == None:
ma_db = astrodb.Database(model_atmosphere_db)
if param_lims:
limit_text = ' AND '.join(
[l[0] + ' IN (' + ','.join(map(str, np.arange(l[1], l[2] + l[3], l[3]))) + ')' for l in param_lims])
model_grid = ma_db.dict("SELECT * FROM {} WHERE {}".format(model_grid_name, limit_text)).fetchall()
else:
model_grid = ma_db.dict("SELECT * FROM {}".format(model_grid_name)).fetchall()
# Load the model atmospheres into a data frame and define the parameters
models = pd.DataFrame(model_grid)
params = [p for p in models.columns.values.tolist() if p in ['teff', 'logg', 'f_sed', 'k_zz']]
# Get the upper bound, lower bound, and increment of the parameters
plims = {p[0]: p[1:] for p in param_lims} if param_lims else {}
for p in params:
if p not in plims:
plims[p] = (min(models.loc[:, p]), max(models.loc[:, p]),
max(np.diff(np.unique(np.asarray(models.loc[:, p]))),
key=list(np.diff(np.unique(np.asarray(models.loc[:, p])))).count))
# Choose template wavelength array to rebin all other spectra
W = rebin_models if isinstance(rebin_models, (list, np.ndarray)) else models['wavelength'][0]
# Rebin model spectra or calculate synthetic magnitudes
if grid_data == 'phot':
import syn_phot as s
model_phot = []
for w, f in zip(list(models['wavelength']), list(models['flux'])):
model_phot.append(
s.all_mags([w * q.um, f * q.erg / q.s / q.cm ** 2 / q.AA], bands=bands, Flam=False, to_flux=True,
photon=False, to_list=True))
pF, pW = map(list, zip(
*[map(list, zip(*[[i.value if hasattr(i, 'unit') else i for i in j][::2] for j in k])) for k in
model_phot]))
models['wavelength'], models['flux'] = pd.Series([np.array(i) for i in pW]), pd.Series(
[np.array(i) for i in pF])
else:
models['flux'] = pd.Series(
[u.rebin_spec([w * q.um, f * q.erg / q.s / q.cm ** 2 / q.AA], W * q.um)[1].value for w, f in
zip(list(models['wavelength']), list(models['flux']))])
models['wavelength'] = pd.Series([W] * len(models['flux']))
if fill_holes:
# Get the coordinates in parameter space of each existing grid point
coords = models.loc[:, params].values
# Get the coordinates in parameter space of each desired grid point
template = np.asarray(list(itertools.product(*[np.arange(l[0], l[1] + l[2], l[2]) for p, l in plims.items()])))
# Find the holes in the grid based on the defined grid resolution without expanding the grid borders
def find_holes(coords, template=''):
# Make a grid of all the parameters
coords = np.asanyarray(coords)
uniq, labels = zip(*[np.unique(c, return_inverse=True) for c in coords.T])
grid = np.zeros(map(len, uniq), bool)
# if template!='':
# temp = np.asanyarray(template)
# uniqT, labelsT = zip(*[np.unique(c, return_inverse=True) for c in temp.T])
# gridT = np.zeros(map(len, uniqT), bool)
grid[labels] = True
candidates = np.zeros_like(grid)
# Test if there are neighboring models for interpolation
for dim in range(grid.ndim):
grid0 = np.rollaxis(grid, dim)
inside = np.logical_or.accumulate(grid0, axis=0) & np.logical_or.accumulate(grid0[::-1], axis=0)[::-1]
candidates |= np.rollaxis(inside, 0, dim + 1)
holes = candidates & ~grid
hole_labels = np.where(holes)
return np.column_stack([u[h] for u, h in zip(uniq, hole_labels)])
grid_holes = find_holes(coords, template=template)
# Interpolate the grid to fill in the holes
for h in grid_holes:
print 'Filling grid hole at {}'.format(h)
new_spectrum = pd_interp_models(params, h, models, smoothing=False)
new_row = {k: v for k, v in zip(params, h)}
new_row.update({'wavelength': new_spectrum[0], 'flux': new_spectrum[1], 'comments': 'interpolated'})
new_row.update(
{'wavelength': new_spectrum[0], 'flux': new_spectrum[1], 'comments': 'interpolated', 'metallicity': 0,
'id': None})
models = models.append(new_row, ignore_index=True)
# Sort the DataFrame by teff and logg?
models.sort(list(reversed(params)), inplace=True)
# Turn Pandas DataFrame into a dictionary of arrays if not using Pandas
if not use_pandas:
M = {k: models[k].values for k in models.columns.values}
M['flux'] = q.erg / q.AA / q.cm ** 2 / q.s * np.asarray(M['flux'])
M['wavelength'] = q.um * M['wavelength'][0]
return M
else:
return models
## Example script for synth_fit
import logging
import cPickle
import astropy.units as u
from astrodbkit import astrodb
import synth_fit
import synth_fit.bdfit
logging.basicConfig(level=logging.INFO)
# load the database - replace with appropriate path
db = astrodb.Database(
'/Users/eileengonzales/Dropbox/BDNYC/BDNYCdb_copy/BDNYCdevdb/bdnycdev.db')
#db = BDdb.get_db('/home/stephanie/Dropbox/BDNYC_new.db')
#db = BDdb.get_db('/home/stephanie/Dropbox/BDNYCdb/BDNYC.db')
object_name = '0036+1821'
query_spectrum = db.query(
"SELECT spec.wavelength_units, spec.flux_units, "
"spec.spectrum FROM "
"spectra AS spec JOIN sources as s ON spec.source_id=s.id "
"WHERE s.shortname='0036+1821' AND spec.telescope_id=7 AND "
"spec.instrument_id=6 AND mode_id=1",
fetch='one')
# ================ This is a hot mess. This example can't be used as is since the format of the database is different
# The stuff below this line will NOT work properly ==========================
from astropy.coordinates import SkyCoord
import astropy.constants as ac
from scipy.ndimage.interpolation import zoom
import pandas as pd
import numpy as np
TABLE_CLASSES = 'display no-wrap hover table'
app_onc.vars = dict()
app_onc.vars['query'] = ''
app_onc.vars['search'] = ''
app_onc.vars['specid'] = ''
app_onc.vars['source_id'] = ''
db_file = 'bdnyc_database.db'
db = astrodb.Database(db_file)
pd.set_option('max_colwidth', -1)
# Redirect to the main page
@app_onc.route('/')
@app_onc.route('/index')
# Page with a text box to take the SQL query
@app_onc.route('/index', methods=['GET', 'POST'])
def onc_query():
defquery = 'SELECT * FROM sources'
if app_onc.vars['query'] == '':
app_onc.vars['query'] = defquery
# Get list of the catalogs
source_count, = db.list("SELECT Count(*) FROM sources").fetchone()
##################################################
# CODE STARTS HERE
##################################################
# ===============================================
# import files needed
# ===============================================
# import gaia csv
gaia_catalogue = pd.read_csv('GUCDScat.csv')
# import bdnyc database
db = astrodb.Database('BDNYCdb_practice/bdnycdev_copy.db')
# load matches and new_objects csvs
matches = pd.read_csv('matches.csv', index_col=0)
new_objects = pd.read_csv('new_objects.csv', index_col=0)
# ===============================================
# sort data into matches and not matches and save as new csv files
# ===============================================
# matches, new_objects = matches_sortCSV(gaia_catalogue, db, save=True)
# ===============================================
##### this tests T dwarfs which have IR spectral types. remember to change sptype regime in montecarlotest_2 to 'IR'
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from astrodbkit import astrodb
db = astrodb.Database('/Users/cammyfbuzard/Dropbox/BDNYCdb/BDNYCdev.db')
import montecarlotest_2 as mc
sources = pd.DataFrame(db.list("select source_id from spectral_types where regime='IR'").fetchall())
sources = sources.drop_duplicates(0)
### No unresolved binaries
comp = []
for i in sources[0]:
try:
comp.append(db.list("SELECT components FROM sources WHERE id={}".format(i)).fetchone()[0])
except TypeError:
comp.append(1)
for index,item in enumerate(comp):
if item == None:
comp[index] = 1
else:
comp[index] = None
sources[1] = comp
sources = sources.dropna()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from astrodbkit import astrodb
db = astrodb.Database('/Users/cammyfbuzard/Code/Python/BDNYC/BDNYCdev.db')
import montecarlotest_2 as mc
sources = pd.DataFrame(
db.list(
"select source_id from spectral_types where gravity is null and regime='OPT'"
).fetchall())
sources = sources.drop_duplicates(0)
### No unresolved binaries
comp = []
for i in sources[0]:
try:
comp.append(
db.list("SELECT components FROM sources WHERE id={}".format(
i)).fetchone()[0])
except TypeError:
comp.append(1)
for index, item in enumerate(comp):
if item == None:
comp[index] = 1
else:
comp[index] = None
sources[1] = comp
# wl_bin_cl = speck2[0].value
# fl_bin_cl = speck2[1].value
#
# # Normalize SED of 1256 to peak flux
# norm_1256 = f/max(f)
#
# plt.plot(wl_bin_nc, fl_bin_nc)
# plt.plot(wl_bin_cl, fl_bin_cl)
# plt.plot(w, norm_1256)
# ---------------------------------------------------------------------------------------------------------------------
# This is for the normalized by across a region method. I chose to normalized everything near the Y band peak for J1256,
# which was over the 0.98-0.988 micron region. This needs to be updated in the Chi-Squared when running on other objects
# ----------------------------------------------------------------------------------------------------------------------
# Pull out models where normalized to max flux.
ma_db = astrodb.Database(model_atmosphere_db)
ms_norm = ma_db.query(
"select wavelength, flux from marley_saumon where logg=5.5 and teff=2400 and f_sed=2 and k_zz=4",
fmt='dict')
bt_norm = ma_db.query(
"select wavelength, flux from bt_settl_2013 where logg=5.0 and teff=2500",
fmt='dict')
# Re-Read in SED of 1256-0224 so that there is not units and can manipulate to plot.
SED_path = '../Atmospheres_paper/Data/Gaia1256-0224 (L3.5sd) SED.txt' # Move up to Modules and into folder
w, f, e = np.loadtxt(SED_path, delimiter=" ", unpack=True)
# pull out the wavelength and flux
flux_ms = ms_norm[0]['flux']
wave_ms = ms_norm[0]['wavelength']
flux_bt = bt_norm[0]['flux']
