def download(dp_ids,
min_disk_space=float(default.get_value('min_disk_space'))):
r"""Given a filename in the ADP format, the code download the file from the
`ESO archive <http://archive.eso.org>`_
Args:
dp_ids (any): list data product ID (or single product ID) to be downloaded
min_disk_space (float): the file will be downloaded only if there is this amount of space (in Gb) free on the
disk
Returns:
None
"""
# Check for disk space
checks.check_disk_space(min_disk_space=min_disk_space)
# Cleaning list
dp_ids_list = cleaning_lists.from_element_to_list(
cleaning_lists.from_bytes_to_string(dp_ids), element_type=str)
for dp_id in dp_ids_list:
# Given a dp_id of a public file, the link to download it is constructed as follows:
download_url = 'http://archive.eso.org/datalink/links?ID=ivo://eso.org/ID?{}&eso_download=file'.format(
dp_id)
msgs.work('Retrieving file {}.fits'.format(dp_id))
urllib.request.urlretrieve(download_url, filename=dp_id + '.fits')
msgs.info('File {}.fits downloaded'.format(dp_id))
def _is_column_list_in_catalogues(columns, collections=None, tables=None):
r"""Check if a given list of columns is present in the ESO archive
It is possible to test for given collection (or table) by setting the appropriate values as input
Args:
columns (any): list of string containing the column_name (or the single `str`) to be tested
collections (any): list of `str` containing the names of the collections (or a single `str`) from which the
columns will be extracted
tables (any): list of `str` (or a single `str`) containing the names of the tables from which the columns
will be extracted
Returns:
list: same of `columns` but columns not present at in the collections/tables are removed
"""
assert columns is None or isinstance(
columns,
(str, list)), r'`columns` must be `None` or a `str` or a `list`'
columns_list = cleaning_lists.from_element_to_list(columns,
element_type=str)
if columns is not None:
# test if it is a valid column
clean_columns = []
for column in columns_list:
if _is_column_in_catalogues(column,
collections=collections,
tables=tables):
clean_columns.append(column)
else:
clean_columns = None
return clean_columns
def query_from_radec(positions,
radius=None,
instruments=None,
data_types=None,
open_link=False,
show_link=False):
r"""Query the ESO `ASP service <http://archive.eso.org/scienceportal/home>`_ given a position
The `positions` value (or list) needs to be given as an `astropy.coordinates.SkyCoord` object. For further detail
see here: `astropy coordinates <https://docs.astropy.org/en/stable/coordinates/>`_
Args:
positions (astropy.coordinates.SkyCoord): coordinates (or list of coordinates) of the sky you want to query
radius (float): search radius in arcseconds
instruments (list): list of `str` (or single `str`) containing the instruments used to limit the search
data_types (list): list of `str` (or single `str`) containing the data types used to limit the search
open_link (bool): open a link to the ASP page
show_link (bool): show the link on the terminal
Returns:
None
"""
# Check input position
positions_list = cleaning_lists.from_element_to_list(
positions, element_type=coordinates.SkyCoord)
# Working on instruments
instruments_list = cleaning_lists.from_element_to_list(instruments,
element_type=str)
# Working on data_types
data_types_list = cleaning_lists.from_element_to_list(data_types,
element_type=str)
# Run one query per position
for iii, position in enumerate(positions_list):
position.transform_to(ICRS)
url = '{0}{1}{2}{3}{4}'.format(
asp_queries.base_url(),
asp_queries.condition_position(position.ra.degree,
position.dec.degree,
radius,
connector=None),
asp_queries.condition_instruments(instruments_list, connector='&'),
asp_queries.condition_data_types(data_types_list, connector='&'),
asp_queries.sort_by('-obs_date', connector='&'))
asp_queries.run_query(url, show_link=show_link, open_link=open_link)
return
def query_from_polygons(polygons=None,
instruments=None,
data_types=None,
open_link=False,
show_link=False):
r"""Query the ESO `ASP service <http://archive.eso.org/scienceportal/home>`_ given a polygon
The `polygons` value (or list) needs to be given as a string defining the location in the sky of the polygon
with RA, Dec, separated by commas and with the first RA, Dec pair that matches the last one (to close the
polygon)
Args:
polygons (list): list of `str` (or single `str`) containing the coordinates of the polygon in the sky you want
to query
instruments (list): list of `str` (or single `str`) containing the instruments used to limit the search
data_types (list): list of `str` (or single `str`) containing the data types used to limit the search
open_link (bool): open a link to the ASP page
show_link (bool): show the link on the terminal
Returns:
None
"""
# Check input position
polygons_list = cleaning_lists.from_element_to_list(polygons,
element_type=str)
# Working on instruments
instruments_list = cleaning_lists.from_element_to_list(instruments,
element_type=str)
# Working on data_types
data_types_list = cleaning_lists.from_element_to_list(data_types,
element_type=str)
# Run one query per position
for iii, polygon in enumerate(polygons_list):
url = '{0}{1}{2}{3}{4}'.format(
asp_queries.base_url(),
asp_queries.condition_polygon(polygon, connector=None),
asp_queries.condition_instruments(instruments_list, connector='&'),
asp_queries.condition_data_types(data_types_list, connector='&'),
asp_queries.sort_by('-obs_date', connector='&'))
asp_queries.run_query(url, show_link=show_link, open_link=open_link)
return
def main(args):
from astropy import coordinates
from astropy import units as u
from ESOAsg import archive_observations
from ESOAsg import msgs
from ESOAsg.ancillary import cleaning_lists
ra_list = cleaning_lists.from_element_to_list(args.ra_degree,
element_type=float)
dec_list = cleaning_lists.from_element_to_list(args.dec_degree,
element_type=float)
if args.radius is None:
radius = None
else:
radius = cleaning_lists.from_element_to_list(args.radius,
element_type=float)[0]
instruments = cleaning_lists.from_element_to_list(args.instruments,
element_type=str)
data_types = cleaning_lists.from_element_to_list(args.data_types,
element_type=str)
msgs.start()
for ra, dec in zip(ra_list, dec_list):
position = coordinates.SkyCoord(ra=ra * u.degree,
dec=dec * u.degree,
frame='fk5')
msgs.newline()
msgs.info('Query for ESO archival data around the position: {}'.format(
position.to_string('hmsdms')))
result_from_query = archive_observations.query_from_radec(
position,
radius=radius,
instruments=instruments,
data_types=data_types,
verbose=False)
if len(result_from_query['dp_id']) > 0:
archive_observations.download(result_from_query['dp_id'])
msgs.end()
def _is_table_list_at_eso(tables):
r"""Check if a given list of table_names is present in the ESO archive
Args:
tables (any): `list` of table_names (or a single `str`) to be tested
Returns:
list: same of `tables` but tables not present at ESO are removed
"""
assert tables is None or isinstance(
tables, (str, list)), r'`tables` must be `None` or a `str` or a `list`'
tables_list = cleaning_lists.from_element_to_list(tables, element_type=str)
if tables is not None:
clean_tables = []
for table in tables_list:
if _is_table_at_eso(table):
clean_tables.append(table)
else:
clean_tables = None
return clean_tables
def _is_collection_list_at_eso(collections):
r"""Check if a given list of collections is present in the ESO archive
Args:
collections (any): str or list of collections to be tested
Returns:
list: same of `collections` but collection not present at ESO are removed
"""
assert collections is None or isinstance(collections, (str, list)), r'`collections` must be `None`, ' \
r'or a `str` or a `list`'
collections_list = cleaning_lists.from_element_to_list(collections,
element_type=str)
if collections_list is not None:
clean_collections = []
for collection in collections_list:
if _is_collection_at_eso(collection):
clean_collections.append(collection)
else:
clean_collections = None
return clean_collections
def _is_column_list_in_obscore(columns):
r"""Check if a given list of columns is present in `ivoa.ObsCore`
Args:
columns (any): list of string containing the column_name (or the single `str`) to be tested
Returns:
list: same of `columns` but columns not present at in the collections/tables are removed
"""
assert columns is None or isinstance(
columns,
(str, list)), r'`columns` must be `None` or a `str` or a `list`'
columns_list = cleaning_lists.from_element_to_list(columns,
element_type=str)
if columns is not None:
# test if it is a valid column
clean_columns = []
for column in columns_list:
if _is_column_in_obscore(column):
clean_columns.append(column)
else:
clean_columns = None
return clean_columns
def main(args):
from ESOAsg.ancillary import astro
from ESOAsg.ancillary import cleaning_lists
from ESOAsg.ancillary import polygons
from ESOAsg import archive_science_portal
from ESOAsg import archive_observations
from ESOAsg import msgs
# Cleaning input lists
input_fits_files = cleaning_lists.make_list_of_fits_files(args.input_fits)
instruments = cleaning_lists.from_element_to_list(args.instruments,
element_type=str)
data_types = cleaning_lists.from_element_to_list(args.data_types,
element_type=str)
# Cleaning input values
confidence_level = cleaning_lists.from_element_to_list(
args.confidence_level, element_type=float)[0]
maxrec = cleaning_lists.from_element_to_list(args.maxrec,
element_type=int)[0]
max_vertices = cleaning_lists.from_element_to_list(args.max_vertices,
element_type=int)[0]
# Cleaning bool
show_figure = args.show_sky
# Show link
show_asp = args.asp_link
# Download data
download_data = args.download_data
msgs.start()
for fits_file in input_fits_files:
msgs.info(' ')
msgs.info('Working on file: {}'.format(fits_file))
msgs.info(' ')
contours = astro.contours_from_gw_bayestar(
fits_file, credible_level=confidence_level)
astro.show_contours_from_gw_bayestar(
fits_file,
contours=contours,
cmap='afmhot',
contours_color='white',
show_figure=show_figure,
matplotlib_backend=STARTING_MATPLOTLIB_BACKEND)
polygons_list = polygons.contours_to_polygons(
contours, max_vertices=max_vertices)
if show_asp:
msgs.info('Opening links to ASP')
archive_science_portal.query_from_polygons(polygons=polygons_list,
open_link=True)
results_from_query = archive_observations.query_from_polygons(
polygons=polygons_list,
maxrec=maxrec,
instruments=instruments,
data_types=data_types,
verbose=False)
msgs.info(' ')
if download_data:
if results_from_query is None:
msgs.warning('No data retrieved')
elif isinstance(results_from_query, list):
for idx_poly, result_from_query in enumerate(
results_from_query):
msgs.info(
'Downloading data for polygon N.{}'.format(idx_poly +
1))
archive_observations.download(result_from_query['dp_id'])
else:
msgs.info('Downloading data')
archive_observations.download(results_from_query['dp_id'])
msgs.info(' ')
msgs.end()
def query_from_radec(positions=None,
radius=None,
instruments=None,
data_types=None,
columns=None,
verbose=False,
maxrec=None):
r"""Query the ESO archive for data at a given position in RA and Dec
The `positions` value (or list) needs to be given as an
`astropy.coordinates.SkyCoord <https://docs.astropy.org/en/stable/coordinates/>`_ object.
The output is in an (list of) `astropy.table` with columns defined in: `core.tap_queries.COLUMNS_FROM_OBSCORE`
It is possible to change the columns to query by setting the value of `columns`
.. note::
In case you are querying radius=`None` is set, the query will performed with:
`INTERSECT(POINT('',RA,Dec), s_region)`
instead of:
`INTERSECT(s_region,CIRCLE('',RA,Dec,radius/3600.))`.
See here for further examples: `tap obs examples <http://archive.eso.org/tap_obs/examples>`_
Args:
positions (astropy.coordinates.SkyCoord): coordinates (or list of coordinates) of the sky you want to query
radius (float, optional): search radius in arcseconds
instruments (list): list of `str` (or single `str`) containing the instruments used to limit the search
data_types (list): list of `str` (or single `str`) containing the data types used to limit the search
columns (list): list of `str` (or single `str`) containing the columns to be queried
verbose (bool): if set to `True` additional info will be displayed
maxrec (int, optional): define the maximum number of entries that a single query can return. If it is `None` the
value is set by the limit of the service.
Returns:
any: results from the queries
"""
# Check inputs:
# Working on positions
positions_list = cleaning_lists.from_element_to_list(
positions, element_type=coordinates.SkyCoord)
# Working on radius
if radius is not None:
if isinstance(radius, int):
radius = float(radius)
else:
assert isinstance(radius, float), r'Input radius is not a number'
# Working on instruments
instruments_list = cleaning_lists.from_element_to_list(instruments,
element_type=str)
# Working on data_types
data_types_list = cleaning_lists.from_element_to_list(data_types,
element_type=str)
# Working on columns
columns_list = _is_column_list_in_obscore(columns)
if verbose:
how_many_positions = len(positions_list)
if how_many_positions > 1:
msgs.work(
'Exploring ESO archive around {} locations in the sky'.format(
how_many_positions))
else:
msgs.work(
'Exploring ESO archive around the input location in the sky')
# Running over all positions
results_from_query = []
for idx, position in enumerate(positions_list):
position.transform_to(ICRS)
ra, dec = np.float_(position.ra.degree), np.float_(position.dec.degree)
msgs.work(
'Running query {} to the ESO archive (out of {} total)'.format(
idx + 1, len(positions_list)))
# Define query
query = "{0}{1}{2}{3}".format(
tap_queries.create_query_obscore_base(columns_list),
tap_queries.condition_intersects_ra_dec(ra, dec, radius=radius),
tap_queries.condition_instruments_like(instruments_list),
tap_queries.condition_data_types_like(data_types_list))
# instantiate ESOCatalogues
query_for_observations = query_observations.ESOObservations(
query=query, type_of_query='sync', maxrec=maxrec)
# running query and append results to the list
if verbose:
query_for_observations.print_query()
# Obtaining query results
query_for_observations.run_query(to_string=True)
result_from_query = query_for_observations.get_result_from_query()
if len(result_from_query) < 1:
msgs.warning('No data has been retrieved')
else:
msgs.info('A total of {} entries has been retrieved'.format(
len(result_from_query)))
if verbose:
msgs.info('For the following instrument:')
for inst_name in np.unique(
result_from_query['instrument_name'].data):
msgs.info(' - {}'.format(inst_name))
results_from_query.append(result_from_query)
# Returning results
return _return_results_from_query(results_from_query)
def query_from_polygons(polygons,
instruments=None,
data_types=None,
verbose=False,
columns=None,
maxrec=None):
r"""Query the ESO archive for data at a area in the sky defined by a polygon
The `polygons` value (or list) needs to be given as a string defining the location in the sky of the polygon
with RA, Dec, separated by commas and with the first RA, Dec pair that matches the last one (to close the
polygon)
The output is in an (list of) `astropy.table` with columns defined in: `core.tap_queries.COLUMNS_FROM_OBSCORE`
It is possible to change the columns to query by setting the value of `columns`
Args:
polygons (list): ist of `str` (or single `str`) containing the coordinates of the polygon in the sky you want
to query
instruments (list): list of `str` (or single `str`) containing the instruments used to limit the search
data_types (list): list of `str` (or single `str`) containing the data types used to limit the search
columns (list): list of `str` (or single `str`) containing the columns to be queried
verbose (bool): if set to `True` additional info will be displayed
maxrec (int, optional): define the maximum number of entries that a single query can return. If it is `None` the
value is set by the limit of the service.
Returns:
any: results from the queries
"""
# Check inputs:
# Working on polygons
polygons_list = cleaning_lists.from_element_to_list(polygons,
element_type=str)
# Working on instruments
instruments_list = cleaning_lists.from_element_to_list(instruments,
element_type=str)
# Working on data_types
data_types_list = cleaning_lists.from_element_to_list(data_types,
element_type=str)
# Working on columns
columns_list = _is_column_list_in_obscore(columns)
if verbose:
how_many_polygons = len(polygons_list)
if how_many_polygons > 1:
msgs.work(
'Exploring ESO archive around {} locations in the sky'.format(
how_many_polygons))
else:
msgs.work(
'Exploring ESO archive around the input location in the sky')
# Running over all positions
results_from_query = []
for idx, polygon in enumerate(polygons_list):
msgs.work(
'Running query {} to the ESO archive (out of {} total)'.format(
idx + 1, len(polygons_list)))
# Define query
query = "{0}{1}{2}{3}".format(
tap_queries.create_query_obscore_base(columns_list),
tap_queries.condition_intersects_polygon(polygon),
tap_queries.condition_instruments_like(instruments_list),
tap_queries.condition_data_types_like(data_types_list))
# instantiate ESOCatalogues
query_for_observations = query_observations.ESOObservations(
query=query, type_of_query='sync', maxrec=maxrec)
# running query and append results to the list
if verbose:
query_for_observations.print_query()
# Obtaining query results
query_for_observations.run_query(to_string=True)
result_from_query = query_for_observations.get_result_from_query()
if len(result_from_query) < 1:
msgs.warning('No data has been retrieved')
else:
msgs.info(
'A total of {} entries has been retrieved (with maxrec={})'.
format(len(result_from_query), maxrec))
msgs.info('For the following instrument:')
for inst_name in np.unique(
result_from_query['instrument_name'].data):
msgs.info(' - {}'.format(inst_name))
results_from_query.append(result_from_query)
# Returning results
return _return_results_from_query(results_from_query)