def __init__(self, coord, radius, **kwargs):
surveycoord.SurveyCoord.__init__(self, coord, radius, **kwargs)
self.survey = 'WISE'
self.bands = WISE_bands
self.service = TAPService('https://irsa.ipac.caltech.edu/TAP')
self.query = None
self.database = "allwise_p3as_psd"
def conesearch(self, ra, dec, radius, **kwargs):
## ToDo: rewrite to use Gaia Archive, not tap service. Need to ditch distance param and just use parallax
# input parameters in degrees:
ra_ = ra.to(u.degree); dec_= dec.to(u.degree); rad_=radius.to(u.degree)
maxrec = kwargs.get('maxrec', 20000)
columnlist = self._get_col_list() + ', gd.r_est'
dbsource = '\n\t'.join(['\nFROM gaiadr2_complements.geometric_distance gd',
'INNER JOIN gaiadr2.gaia_source gs using (source_id) '])
constraints = '\n\t'.join(['\nWHERE ',
'CONTAINS(POINT(\'\', gs.ra, gs.dec), ',
'\tCIRCLE(\'\', {ra}, {dec}, {rad})) = 1 '.format(ra=ra_.value, dec=dec_.value, rad=rad_.value)])
if self.source_constraints is not None:
constraints = constraints + ' AND '+ self.source_constraints
self.tap_query_string = 'SELECT \n\t\t'+ columnlist + dbsource + constraints
tap_service = TAPService(self.tap_service_url)
tap_results = tap_service.search(self.tap_query_string, maxrec=maxrec)
self.objs = tap_results.to_table().to_pandas()
self.objs.set_index('source_id', inplace=True)
def connect(self, login, password):
'''
Authenticate the service through CEFCA portal.
Parameters
----------
login : str
User login, usually an email.
password : str
User password.
'''
self.auth = CEFCA_authenticate(login, password)
self.service = TAPService(self.serviceUrl, self.auth)
def from_source_idlist(self, source_idlist, source_idcol=None, filters=False):
#xml-ify the source_idlist to a file
if isinstance(source_idlist, Table):
#guess which column contains the source ids
if source_idcol is None:
if 'source_id' in source_idlist.colnames:
sidcol = 'source_id'
elif 'source' in source_idlist.colnames:
sidcol = 'source'
else:
raise ValueError('no column to use as source_id')
else:
if source_idcol not in source_idlist.colnames:
raise ValueError(f'invalid column specified as source id column: {source_idcol}')
sidcol = source_idcol
tbl = source_idlist
elif isinstance(source_idlist, np.ndarray) or isinstance(source_idlist, list):
sidcol = 'source_id'
tbl = Table({sidcol:source_idlist})
else:
raise ValueError(f'invalid source_idlist type: {type(source_idlist)}')
xml_path = 'source_idlist.xml'
tbl.write(xml_path, table_id='source_idlist', format='votable', overwrite=True)
#build the query:
col_list = self._get_col_list() + ', gd.r_est'
dbsource = ''.join([' FROM tap_upload.source_idlist sidl',
f' LEFT JOIN gaiadr2.gaia_source gs ON gs.source_id = sidl.{sidcol}',
f' LEFT JOIN gaiadr2_complements.geometric_distance gd on gs.source_id = gd.source_id' ])
query_str = f'SELECT sidl.{sidcol} as "source", '+col_list+dbsource
if filters:
query_str = query_str + ' WHERE '+ self.source_constraints
self.tap_query_string = query_str
#fetch the data
#job = Gaia.launch_job_async(query=query_str, upload_resource=xml_path,upload_table_name='source_idlist')
#self.objs = job.get_results().to_pandas()
#fetch data via tap query
tap_service = TAPService(self.tap_service_url)
tap_results = tap_service.search(self.tap_query_string, maxrec=len(tbl),uploads={'source_idlist':tbl})
self.objs = tap_results.to_table().to_pandas()
self.objs.set_index('source', inplace=True)
def __init__(self):
self.service = TAPService(nancay_tap)
log.info(f'TAP service {nancay_tap} accessed.')
self.meta_names = [
'target_name', 'obs_creator_did', 's_ra', 's_dec', 't_min',
't_max', 'em_min', 'em_max'
]
self._conditions = {
'time': '',
'freq': '',
'pos': '',
}
self.time_range = [None, None]
self.freq_range = [None, None]
self.fov_radius = 180 * u.deg
self.fov_center = None
class WISE_Survey(surveycoord.SurveyCoord):
"""
Class to handle queries on the WISE survey
Child of DL_Survey which uses datalab to access NOAO
Args:
coord (SkyCoord): Coordiante for surveying around
radius (Angle): Search radius around the coordinate
"""
def __init__(self, coord, radius, **kwargs):
surveycoord.SurveyCoord.__init__(self, coord, radius, **kwargs)
self.survey = 'WISE'
self.bands = WISE_bands
self.service = TAPService('https://irsa.ipac.caltech.edu/TAP')
self.query = None
self.database = "allwise_p3as_psd"
def get_catalog(self,
query=None,
query_fields=_DEFAULT_query_fields,
print_query=False,
system='AB'):
"""
Grab a catalog of sources around the input coordinate to the search radius
Args:
query: Not used
query_fields (list, optional): Over-ride list of items to query
print_query (bool): Print the SQL query generated
system (str): Magnitude system ['AB', 'Vega']
Returns:
astropy.table.Table: Catalog of sources returned. Includes WISE
photometry for matched sources.
Magnitudes are in AB by default
"""
# Main WISE query
if query is None:
self._gen_cat_query(query_fields)
if print_query:
print(self.query)
main_cat = self.service.run_async(self.query).to_table()
main_cat.meta['radius'] = self.radius
main_cat.meta['survey'] = self.survey
if len(main_cat) == 0:
main_cat = catalog_utils.clean_cat(main_cat,
photom['WISE'],
fill_mask=-999.)
return main_cat
main_cat = catalog_utils.clean_cat(main_cat,
photom['WISE'],
fill_mask=-999.)
# Convert to AB mag
if system == 'AB':
fnu0 = {
'WISE_W1': 309.54,
'WISE_W2': 171.787,
'WISE_W3': 31.674,
'WISE_W4': 8.363
}
for item in ['1', '2', '3', '4']:
filt = 'WISE_W' + item
main_cat[filt] -= 2.5 * np.log10(fnu0[filt] / 3631.)
elif system == 'Vega':
pass
# Finish
self.catalog = main_cat
self.validate_catalog()
return self.catalog.copy()
def get_cutout(self, imsize, filter, timeout=120):
"""
Download an image from IRSA
Args:
imsize(Quantity): Size of the cutout in angular units.
filter(str): One of "W1", "W2", "W3" or "W4"
timeout(float): Number of seconds to wait to hear a response from
the IRSA SIA server.
Returns:
imghdu(fits.HDU): Fits HDU with image
"""
assert filter.upper() in self.bands, "Invalid filter name " + filter
# First get a table with the image coadd_id
meta_url = "https://irsa.ipac.caltech.edu/ibe/search/wise/allsky/4band_p3am_cdd?POS={:f},{:f}".format(
self.coord.ra.value, self.coord.dec.value)
img_metatab = Table.read(utils.data.download_file(meta_url,
cache=True,
show_progress=False,
timeout=timeout),
format="ascii")
coadd_id = img_metatab['coadd_id'][img_metatab['band'] == int(
filter[1])][0]
# Now generate the image url
img_url = "https://irsa.ipac.caltech.edu/ibe/data/wise/allsky/4band_p3am_cdd/{:s}/{:s}/{:s}/{:s}-{:s}-int-3.fits".format(
coadd_id[:2], coadd_id[:4], coadd_id, coadd_id, filter.lower())
img_url += "?center={:f},{:f}&size={:f}pix".format(
self.coord.ra.value, self.coord.dec.value,
imsize.to("arcsec").value / 2.75)
self.cutout = io.fits.open(
utils.data.download_file(img_url,
cache=True,
show_progress=False,
timeout=timeout))[0]
self.cutout_size = imsize
return self.cutout.copy()
def _gen_cat_query(self, query_fields=_DEFAULT_query_fields):
"""
Generate ADQL query for catalog search
self.query is modified in place
Args:
query_fields (list): Override the default list for the SQL query
"""
query_field_str = ""
for field in query_fields:
query_field_str += " {:s},".format(field)
# Remove last comma
query_field_str = query_field_str[:-1]
self.query = """SELECT{:s}
FROM {:s}
WHERE CONTAINS(POINT('ICRS',ra, dec), CIRCLE('ICRS',{:f},{:f},{:f}))=1""".format(
query_field_str, self.database, self.coord.ra.value,
self.coord.dec.value,
self.radius.to(units.deg).value)
return self.query
import numpy as np
import pandas as pd
import pickle
import astropy.units as u
import astropy.coordinates as coord
from astropy.coordinates.sky_coordinate import SkyCoord
from astropy.units import Quantity
from pyvo.dal import TAPService
import matplotlib.pyplot as plt
from matplotlib.pyplot import cm
tap_service_url = "http://gaia.ari.uni-heidelberg.de/tap"
tap_service = TAPService(tap_service_url)
#default query columns:
column_list = ['source_id', 'ra','dec','parallax','pmra','pmdec','radial_velocity',
'phot_g_mean_mag','phot_bp_mean_mag', 'phot_rp_mean_mag','r_est']
class fieldstars():
def __init__(self, name:str):
self.name=name
self.coords = None
self.tap_query_string = None
class TAPQueueManager(object):
'''
Submit, run, review and fetch results of TAP jobs.
This is a little overcomplicated because, for some reason, the job list
returned by JPAS TAPService is always empty.
Here we keep the job list in a local file.
Parameters
----------
service_url : str
TAP service to connect.
Example: https://archive.cefca.es/catalogues/vo/tap/minijpas-idr201910
tables_dir : str, optional
Directory to save tables if no path is specified when downloading.
Default: current directory.
'''
def __init__(self, service_url, tables_dir='.'):
self.serviceUrl = service_url
self.tablesDir = tables_dir
self.auth = None
self.service = None
self.jobs = {}
def connect(self, login, password):
'''
Authenticate the service through CEFCA portal.
Parameters
----------
login : str
User login, usually an email.
password : str
User password.
'''
self.auth = CEFCA_authenticate(login, password)
self.service = TAPService(self.serviceUrl, self.auth)
def loadJobList(self, filename):
'''
Recover job information from a list kept in an ascii table in `filename`.
The lines must contain:
table_name job_id PHASE
where job_id is an integer, and PHASE is a (informative only) string.
Parameters
----------
filename : str
Ascii file containing the submitted jobs.
'''
with open(filename, 'r') as f:
for l in f.readlines():
w = l.split()
if len(w) != 3:
continue
tablename, job_id, phase = w
log.debug('Recovering table %s (job id %s, phase %s)' %
(tablename, job_id, phase))
self._recoverJob(tablename, job_id)
@suppress_spec_warnings
def saveJobList(self, filename):
'''
Save current submitted jobs information.
NOTE: this will overwrite the file.
Paramters
---------
filename : str
Ascii file containing the submitted jobs.
'''
with open(filename, 'w') as f:
f.writelines('%s %s %s\n' % (t, j.job_id, j.phase)
for t, j in self.jobs.items())
def _query(self, tablename):
return 'select * from minijpas.%s' % tablename
def _filePath(self, tablename):
return path.join(self.tablesDir, '%s.fits' % tablename)
@suppress_spec_warnings
def requestTable(self, tablename, force=False, maxrec=1000000):
'''
Parameters
----------
tablename : str
Name of the table, must be present in the database.
force : bool, optional
If the table is already in the queue, the default
behaviour is to use that job. if `force=True`, submit another job.
maxrec : int, optional
Maximum number of records to return.
Default: 1000000
'''
if tablename in self.jobs and not force:
log.debug(
'Table %s already submitted. Use force=True to resubmit.' %
tablename)
return
query = self._query(tablename)
j = self.service.submit_job(query, maxrec=maxrec)
log.debug('Created job %s for table %s.' % (j.job_id, tablename))
self.jobs[tablename] = j
return j
@suppress_spec_warnings
def runNextJob(self):
'''
Run next pending job.
'''
for tab, j in self.jobs.items():
if j.phase == 'PENDING':
j.run()
log.info('Started job %s (%s).' % (j.job_id, tab))
return j
return None
@suppress_spec_warnings
def _listFiltered(self, phases):
return [t for t, j in self.jobs.items() if j.phase in phases]
def listComplete(self):
'''
List complete jobs, which have results ready for download.
Returns
-------
complete_jobs : list
List containing the table names of the complete jobs.
'''
return self._listFiltered(['COMPLETED'])
def listRunning(self):
'''
List jobs currently running.
Returns
-------
complete_jobs : list
List containing the table names of the running jobs.
'''
return self._listFiltered(['RUN', 'EXECUTING'])
def listPending(self):
'''
List jobs currently in queue.
Returns
-------
complete_jobs : list
List containing the table names of the queued jobs.
'''
return self._listFiltered(['PENDING'])
def _downloaded(self, tablename):
filename = self._filePath(tablename)
return path.exists(filename)
def removeDownload(self, tablename):
filename = self._filePath(tablename)
if path.exists(filename):
log.debug('Deleting file %s.' % tablename)
def listDownloadPending(self):
'''
List complete jobs, not yet downloaded to the default path.
Returns
-------
available_jobs : list
List containing the table names of the downloadable jobs.
'''
return [
tab for tab in self.listComplete() if not self._downloaded(tab)
]
@suppress_spec_warnings
def download(self, tablename, filename=None, overwrite=False):
'''
Download results of job. The job must be already completed.
The table will be saved as an `astropy.table.Table`, in FITS format.
Parameters
---------
tablename : str
Name of the table to download.
filename : str, optional
Path to the save the table.
Default: `'table_dir/tablename.fits'`, where
`table_dir` may be set in the constructor(falls back to `'.'`).
overwrite : bool, optional
Overwrite the file if it already exists.
Will skip the download otherwise.
Default: `False`
'''
if filename is None:
filename = self._filePath(tablename)
if path.exists(filename) and not overwrite:
log.debug('File %s already exists, skipping download.' % filename)
return
j = self.jobs[tablename]
if j.phase != 'COMPLETED':
raise Exception(
'Job %s (table %s) is in phase %s. Download unavailable.' %
(j.job_id, tablename, j.phase))
log.debug('Fetching job %s results...' % j.job_id)
res = j.fetch_result()
j.raise_if_error()
t = res.to_table()
log.debug('Fixing table structure.')
fix_names(t)
convert_dtype(t)
del t.meta['description']
log.info('Saving table to %s.' % filename)
t.write(filename, overwrite=overwrite)
def downloadPending(self, overwrite=False):
'''
Download all completed jobs to their default paths.
Parameters
----------
overwrite : bool, optional
Overwrite the files if they already exist.
Will skip the download otherwise.
Default: `False`
'''
for tab in self.listDownloadPending():
log.info('Downloading table %s.' % tab)
self.download(tab, overwrite=overwrite)
@suppress_spec_warnings
def _recoverJob(self, tablename, job_id):
url = '%s/async/%s' % (self.serviceUrl, job_id)
job = AsyncTAPJob(url, self.auth)
self.jobs[tablename] = job
from pyvo.dal import TAPService
import astropy
from astropy.table import Table, vstack
import sys
tap_service = TAPService("http://dc.g-vo.org/tap")
tap_results = tap_service.search("SELECT TOP 10 * FROM ivoa.obscore")
print(tap_results)
class ObsDatabase(object):
""" Class to access NenuFAR BST TAP service.
"""
def __init__(self):
self.service = TAPService(nancay_tap)
log.info(f'TAP service {nancay_tap} accessed.')
self.meta_names = [
'target_name', 'obs_creator_did', 's_ra', 's_dec', 't_min',
't_max', 'em_min', 'em_max'
]
self._conditions = {
'time': '',
'freq': '',
'pos': '',
}
self.time_range = [None, None]
self.freq_range = [None, None]
self.fov_radius = 180 * u.deg
self.fov_center = None
# --------------------------------------------------------- #
# --------------------- Getter/Setter --------------------- #
@property
def meta_names(self):
""" Column names (observation properties) to return using
:meth:`~nenupy.observation.tapdatabase.ObsDatabase.search`.
:setter: `list` of column names.
:getter: Properties to query.
:type: `str`
:seealso:
`Database description
<http://vogate.obs-nancay.fr/__system__/dc_tables/show/tableinfo/nenufar.bst>`_
"""
return ', '.join(self._meta_names)
@meta_names.setter
def meta_names(self, m):
if not isinstance(m, list):
raise TypeError('meta_names must be a list')
unknown_mask = ~np.isin(m, colnames)
if unknown_mask.any():
unknown = np.array(m)[unknown_mask]
raise ValueError('Unknown meta_names: {}, available: {}'.format(
unknown, colnames))
self._meta_names = m
return
@property
def time_range(self):
""" Time range selection for the ADQL query using
:meth:`~nenupy.observation.tapdatabase.ObsDatabase.search`.
Default is ``[None, None]`` which means that no
condition based on observation time will be applied.
:setter: Length-2 list of ``[start, stop]``. ``start``
and ``stop`` may be passed as :class:`~astropy.time.Time`
instances or as ISO/ISOT `str`.
:getter: ``[start, stop]`` list.
:type: `list`
"""
return self._time_range
@time_range.setter
def time_range(self, t):
if t == [None, None]:
self._conditions['time'] = ''
self._time_range = t
return
if not isinstance(t, list):
raise TypeError('time_range must be a list')
if not len(t) == 2:
raise ValueError('time_range must be of length 2')
if not all([isinstance(ti, Time) for ti in t]):
t = [Time(ti) for ti in t]
self._conditions['time'] = f'(t_min >= {t[0].mjd} '\
f'AND t_max <= {t[1].mjd})'
log.info(f'time_range set to {t}.')
self._time_range = t
return
@property
def freq_range(self):
""" Frequency range selection for the ADQL query using
:meth:`~nenupy.observation.tapdatabase.ObsDatabase.search`.
Default is ``[None, None]`` which means that no
condition based on observation frequencies will be applied.
:setter: Length-2 list of ``[fmin, fmax]``. ``fmin``
and ``fmax`` may be passed as :class:`~astropy.units.Quantity`
instances or as `float` (assumed to be expressed
in MHz).
:getter: ``[fmin, fmax]`` list.
:type: `list`
"""
return self._freq_range
@freq_range.setter
def freq_range(self, f):
if f == [None, None]:
self._conditions['freq'] = ''
self._freq_range = f
return
if not isinstance(f, list):
raise TypeError('freq_range must be a list')
if not len(f) == 2:
raise ValueError('freq_range must be of length 2')
if not all([isinstance(fi, u.Quantity) for fi in f]):
f = [fi * u.MHz for fi in f]
lmax = wavelength(f[0]).to(u.m).value
lmin = wavelength(f[1]).to(u.m).value
self._conditions['freq'] = f'(em_min >= {lmin} AND '\
f'em_max <= {lmax})'
log.info(f'freq_range set to {f}.')
self._freq_range = f
return
@property
def fov_radius(self):
""" Radius of the query, in combination with the query
center :attr:`~nenupy.observation.tapdatabase.ObsDatabase.fov_center`.
:setter: Radius (in degrees if no unit is provided). Default is ``180 deg``.
:getter: Radius in degrees.
:type: `float` or :class:`~astropy.units.Quantity`
.. warning::
Must be set **before**
:attr:`~nenupy.observation.tapdatabase.ObsDatabase.fov_center`.
"""
return self._fov_radius
@fov_radius.setter
def fov_radius(self, r):
if r is None:
r = 180 * u.deg
if not isinstance(r, u.Quantity):
r *= u.deg
if not r.isscalar:
raise ValueError('FOV radius must be a scalar.')
self._fov_radius = r
return
@property
def fov_center(self):
""" Center of the field of view queried, in comination
with the radius :attr:`~nenupy.observation.tapdatabase.ObsDatabase.fov_radius`.
:setter: Center of the field of view.
:getter: Center of the field of view.
:type: :class:`~astropy.coordinates.SkyCoord`
.. warning::
Must be set **after**
:attr:`~nenupy.observation.tapdatabase.ObsDatabase.fov_radius`.
"""
return self._fov_center
@fov_center.setter
def fov_center(self, f):
if f is None:
self._conditions['pos'] = ''
self._fov_center = f
return
if not isinstance(f, SkyCoord):
raise TypeError('fov_center must be a SkyCoord instance.')
if not f.isscalar:
raise ValueError('fov_center must be scalar.')
radius = self.fov_radius.to(u.deg).value
self._conditions['pos'] = f'1 = CONTAINS'\
f"(POINT('ICRS', s_ra, s_dec), CIRCLE('ICRS', "\
f'{f.ra.deg}, {f.dec.deg}, {radius}))'
log.info(f'fov_center set to {f}, radius={radius} deg.')
self._fov_center = f
return
@property
def conditions(self):
""" Conditions summary of the query.
:getter: Query conditions.
:type: `str`
"""
conds = []
for key in self._conditions.keys():
if self._conditions[key] != '':
conds.append(self._conditions[key])
conditions = ' AND '.join(conds)
return conditions
@property
def query(self):
""" Full query, combining returned parameteres
:attr:`~nenupy.observation.tapdatabase.ObsDatabase.meta_names`
and the conditions
:attr:`~nenupy.observation.tapdatabase.ObsDatabase.conditions`.
:getter: Query.
:type: `str`
"""
q = f'SELECT {self.meta_names} from nenufar.bst '\
f'WHERE ({self.conditions})'
return q
# --------------------------------------------------------- #
# ------------------------ Methods ------------------------ #
def search(self):
""" Run the TAP :attr:`~nenupy.observation.tapdatabase.ObsDatabase.query`
on the `NenuFAR BST service <http://vogate.obs-nancay.fr/tap>`_.
:returns:
NenuFAR observation properties resulting from the
ADQL query.
:rtype: :class:`~astropy.table.Table`
"""
if self.conditions == '':
raise ValueError('Empty query, fill out some attributes first.')
log.info(f'TAP service columns to return: {self._meta_names}.')
log.debug(f"Querying: '{self.query}'")
result = self.service.search(self.query)
return result.to_table()
def reset(self):
""" Reset query parameters to default values.
"""
self.meta_names = [
'target_name', 'obs_creator_did', 's_ra', 's_dec', 't_min',
't_max', 'em_min', 'em_max'
]
self.time_range = [None, None]
self.freq_range = [None, None]
self.fov_radius = 180 * u.deg
self.fov_center = None
log.info('Query parameters reset to default values.')
return