def check_disk_space(min_disk_space=float( default.get_value('min_disk_space'))) -> bool: r"""Check that there is enough space on the location where the code is running Given a disk space limit in GB, the macro returns `True` if the disk where the code is running has more free GB than the given limit. .. warning:: The current implementation checks the disk where the code is running (i.e., from the directory: `./`). This may cause some troubles with shared disks. Args: min_disk_space (float): Size of free space on disk required Returns: bool: `True` if there is enough space on disk """ total, used, free = shutil.disk_usage("./") total = total / (1024.**3.) used = used / (1024.**3.) free = free / (1024.**3.) msgs.info('Your disk has:') msgs.info('Total: {0:.2f} GB, Used: {0:.2f} GB, Free: {0:.2f} GB'.format( total, used, free)) if free > min_disk_space: enough_space = True else: enough_space = False msgs.warning('Not enough space on disk') return enough_space
def _run_query(query, verbose=False, remove_bytes=True, maxrec=default.get_value('maxrec')): r"""Run tap query and return result as a table Args: query (`str`): Query to be run verbose (`bool`): if set to `True` additional info will be displayed remove_bytes ('bool') if set to True, it converts all bytes entries to standard strings maxrec (`int`): Define the maximum number of entries that a single query can return Returns: result_from_query (`astropy.Table`): Result from the query to the TAP service """ # Load tap service tapobs = _define_tap_service(verbose=False) if verbose: msgs.info('The query is: \n {} \n'.format(str(query))) # Obtaining query results and convert it to an astropy table result_from_query = tapobs.search(query=query, maxrec=maxrec).to_table() # removing bytes code from some columns: if remove_bytes: for column_name in result_from_query.colnames: result_from_query[ column_name].data.data[:] = checks.from_bytes_to_string( result_from_query[column_name].data.data) return result_from_query
def run_query(tap_service, query, type_of_query, maxrec=default.get_value('maxrec')): r"""Run query to TAP service and return result as an `astropy.Table` If the job requires to much time to run, the code will move to an asynchronous query. Args: tap_service (pyvo.dal.tap.TAPService): TAP service that will be used for the query query (str): query to be run type_of_query (str): type of query to be run maxrec (int): define the maximum number of entries that a single query can return. Default is set by default.get_value('maxrec') Returns: astropy.table: result from the query to the TAP service """ if type_of_query not in TAP_QUERY_TYPES: msgs.error('{} not a valid entry for the type of TAP query. Possibilities are: {}'.format(type_of_query, TAP_QUERY_TYPES)) # Obtaining query results and convert it to an astropy table if query is not None: if type_of_query == 'sync': result_from_query = run_query_sync(tap_service, query, maxrec=maxrec) else: result_from_query = run_query_async(tap_service, query, maxrec=maxrec) else: msgs.warning('Empty query provided') result_from_query = None return result_from_query
def download(dp_ids, min_disk_space=float(default.get_value('min_disk_space'))): r"""Given a filename in the ADP format, the code download the file from the `ESO archive <http://archive.eso.org>`_ Args: dp_ids (any): list data product ID (or single product ID) to be downloaded min_disk_space (float): the file will be downloaded only if there is this amount of space (in Gb) free on the disk Returns: None """ # Check for disk space checks.check_disk_space(min_disk_space=min_disk_space) # Cleaning list dp_ids_list = cleaning_lists.from_element_to_list( cleaning_lists.from_bytes_to_string(dp_ids), element_type=str) for dp_id in dp_ids_list: # Given a dp_id of a public file, the link to download it is constructed as follows: download_url = 'http://archive.eso.org/datalink/links?ID=ivo://eso.org/ID?{}&eso_download=file'.format( dp_id) msgs.work('Retrieving file {}.fits'.format(dp_id)) urllib.request.urlretrieve(download_url, filename=dp_id + '.fits') msgs.info('File {}.fits downloaded'.format(dp_id))
def base_url(): r"""Return the url for ASP Returns: str: url """ return str(default.get_value('eso_asp_url'))
def _define_tap_service(verbose=False): r"""Load tap service from defaults The TAP service for raw. reduced, and ambient data is defined in `ESOAsg\default.txt` as `eso_tap_obs` Args: verbose (`bool`): if set to `True` additional info will be displayed Returns: tapcat (`pyvo.dal.tap.TAPService`) TAP service that will be used for the queries """ if verbose: msgs.info('Querying the ESO TAP service at:') msgs.info('{}'.format(str(default.get_value('eso_tap_obs')))) tapobs = dal.tap.TAPService(default.get_value('eso_tap_obs')) return tapobs
def _define_tap_service(verbose=False): r"""Load tap service from defaults The TAP service for scientific catalogues generated by ESO observing teams is defined in `ESOAsg\default.txt` as `eso_tap_cat` Args: verbose (`bool`): if set to `True` additional info will be displayed Returns: tapcat (`pyvo.dal.tap.TAPService`) TAP service that will be used for the queries. For more details on this see the `pyvo documents<https://pyvo.readthedocs.io/en/latest/>`_ """ if verbose: msgs.info('Querying the ESO TAP service at:') msgs.info('{}'.format(str(default.get_value('eso_tap_cat')))) tapcat = dal.tap.TAPService(default.get_value('eso_tap_cat')) return tapcat
def __init__(self, query=None, result_from_query=None, type_of_query='sync', maxrec=default.get_value('maxrec')): # assign tap_service super().__init__( tap_service=tap_queries.define_tap_service('eso_tap_cat'), query=query, result_from_query=result_from_query, type_of_query=type_of_query, maxrec=maxrec)
def download(dp_id, min_disk_space=np.float32(default.get_value('min_disk_space'))): r"""Given a filename in the ADP format, the code download the file from the `ESO archive <http://archive.eso.org>`_ ..note:: if dp_id is not a `numpy.str`, a WARNING message will be raised and the content of `dp_id` will be converted into a string. Args: dp_id (`numpy.str`): Data product ID to be downloaded. min_disk_space (`numpy.float`): The file will be downloaded only if there is this amount of space (in Gb) free on the disk. By default is set by the `default.txt` file. Returns: This downloads a fits ADP file with the same name of the input. """ # Check for disk space checks.check_disk_space(min_disk_space=min_disk_space) for file_name in dp_id: # if the file name is in byte, this decode it. if not isinstance(file_name, str): msgs.warning('The content of dp_id is not in a string format.') msgs.warning('The code is trying to fix this.') if isinstance(file_name, bytes): file_name = np.str(file_name.decode("utf-8")) msgs.warning('Converted to {}.'.format(type(file_name))) else: msgs.error( 'Unable to understand the format of the dp_id entry: {}'. format(type(file_name))) # Given a dp_id of a public file, the link to download it is constructed as follows: download_url = 'http://archive.eso.org/datalink/links?ID=ivo://eso.org/ID?{}&eso_download=file'.format( str(file_name)) msgs.work( 'Downloading file {}. This may take some time.'.format(file_name + '.fits')) urllib.request.urlretrieve(download_url, filename=file_name + '.fits') msgs.info('File {} downloaded.'.format(file_name + '.fits'))
def run_query_sync(tap_service, query, maxrec=default.get_value('maxrec')): r"""Run a synchronous query to TAP service and return result as an `astropy.Table` If the synchronous query fails, the code automatically tries to run the same query asynchronously Args: tap_service (pyvo.dal.tap.TAPService): TAP service that will be used for the query query (str): query to be run maxrec (int): define the maximum number of entries that a single query can return. Default is set by default.get_value('maxrec') Returns: astropy.table: result from the query to the TAP service """ try: result_from_query = tap_service.search(query=query, maxrec=maxrec).to_table() except (ValueError, DALQueryError, DALFormatError): msgs.warning('The query timed out. Trying `async` instead') result_from_query = run_query_async(tap_service=tap_service, query=query, maxrec=maxrec) return result_from_query
def define_tap_service(which_tap_service): r"""Load a Table Access Protocol (TAP) service from defaults Currently the supported `TAP services <http://archive.eso.org/programmatic/#TAP>`_ are: * `eso_tap_cat`: TAP service for scientific catalogues generated by ESO observing teams * `eso_tap_obs`: TAP service for raw, reduced, and ambient data See `pyvo docs <https://pyvo.readthedocs.io/en/latest/api/pyvo.dal.TAPService.html#>`_ for further details Args: which_tap_service (str): Select the `TAP services <http://archive.eso.org/programmatic/#TAP>`_ to be queried Returns: pyvo.dal.tap.TAPService: TAP service used for the queries """ if which_tap_service not in TAP_SERVICES: msgs.error('{} not a valid entry for TAP services. Possibilities are: {}'.format(which_tap_service, TAP_SERVICES)) tap_service = dal.tap.TAPService(default.get_value(which_tap_service)) return tap_service
def _run_query(query, verbose=False, maxrec=default.get_value('maxrec')): r"""Run tap query and return result as a table Args: query (`str`): Query to be run verbose (`bool`): if set to `True` additional info will be displayed maxrec (`int`, `None`): Define the maximum number of entries that a single query can return Returns: result_from_query (`astropy.Table`): Result from the query to the TAP service """ # Load tap service tapcat = _define_tap_service(verbose=False) if verbose: msgs.info('The query is:') msgs.info('{}'.format(str(query))) # Obtaining query results and convert it to an astropy table result_from_query = tapcat.search(query=query, maxrec=maxrec).to_table() return result_from_query
def run_query_async(tap_service, query, maxrec=default.get_value('maxrec')): r"""Run an asynchronous query to TAP service and return result as an `astropy.Table` Args: tap_service (pyvo.dal.tap.TAPService): TAP service that will be used for the query query (str): query to be run maxrec (int): define the maximum number of entries that a single query can return. Default is set by default.get_value('maxrec') Returns: astropy.table: result from the query to the TAP service """ tap_job = tap_service.submit_job(query=query, maxrec=maxrec) tap_job.run() # Wait for Executing tap_job.wait(phases=["EXECUTING", "ERROR", "ABORTED"], timeout=10.) msgs.info('The query to the tap_service is in the status: {}'.format(str(tap_job.phase))) # Wait for Completed tap_job.wait(phases=["COMPLETED", "ERROR", "ABORTED"], timeout=10.) msgs.info('The query to the tap_service is in the status: {}'.format(str(tap_job.phase))) # Fetch the results tap_job.raise_if_error() return tap_job.fetch_result().to_table()
def parser(options=None): parser = argparse.ArgumentParser( description=r"""Collect data from the ESO archive given a `bayestar` """ + r"""file associated to a GW superevent. """ + """\n""" + """\n""" + r"""In summary: the code reads in a probability map in HEALPix format """ + r"""(`input_fits`), finds the `confidence_level`% confidence """ + r"""contours, and query the ESO archive for data within them. """ + r"""Data can be showed with links to the ESO """ + r"""`Archive Science Portal <http://archive.eso.org/scienceportal/home>`_ """ + r"""and can be automatically downloaded on the disk. """ + """\n""" + """\n""" + r"""It is possible to retrieve data for a specific instrument or data type by """ + r"""setting `instruments` and/or `data_types`. """ + r"""Note that `maxrec` set a limit on the number of files """ + r"""that could be retrieved from the archive. """ + """\n""" + """\n""" + r"""A step-by-step description of the procedure is available in this """ + r"""`jupyter notebook <https://github.com/EmAstro/ESOAsg/blob/master/ESOAsg/docs/notebooks/HOWTO_getDataFromGWContours.ipynb>`_.""" + """\n""" + """\n""" + r"""This uses ESOAsg version {:s}""".format(__version__), formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES) parser.add_argument( "input_fits", nargs="+", type=str, help="Input probability map out of the GW event pipeline") parser.add_argument( "-cl", "--confidence_level", nargs="+", type=float, default=50.0, help="Confidence level at which extract the contours. Default is 50.0") parser.add_argument( "-mv", "--max_vertices", nargs="+", type=int, default=30, help="Max number of vertices to be considered in the conversion from" + "contours to polygons. Default it 30") parser.add_argument("-s", "--show_sky", action="store_true", default=False, help="Show the contours on a sky-map") parser.add_argument("-a", "--asp_link", action="store_true", default=False, help="Open ASP web-pages of the selected contours") parser.add_argument( "-dd", "--download_data", action="store_true", default=False, help="Download all data collected within the considered contours") parser.add_argument( "-i", "--instruments", nargs="+", type=str, default=None, help="Limit the search to the selected ESO instruments") parser.add_argument("-dt", "--data_types", nargs="+", type=str, default=None, help="Limit the search to the selected data type") parser.add_argument( "-m", "--maxrec", nargs="+", type=int, default=int(default.get_value('maxrec')), help="Maximum number of files you would like to retrieve") parser.add_argument("-v", "--version", action="version", version=__version__) return parser.parse_args()
def query_from_radec(table_names, position, radius=None, maxrec=default.get_value('maxrec')): r"""Query the ESO tap_cat service (link defined in `ESOAsg\default.txt`) for a specific a specific location.
def query_TAP_from_polygons(polygons=None, merge=False, instrument=None, maxrec=default.get_value('maxrec'), verbose=False): tapobs = dal.tap.TAPService(default.get_value('eso_tap_obs')) msgs.info('Querying the ESO TAP service at:') msgs.info('{}'.format(str(default.get_value('eso_tap_obs')))) results_from_query = [] if polygons is not None: polygon_union = '' for iii, polygon in enumerate(polygons): polygon_union += """intersects(s_region, POLYGON('', """ + polygon + """)) = 1 OR """ if not merge: query = """SELECT target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, dataproduct_type, instrument_name, abmaglim, proposal_id FROM ivoa.ObsCore WHERE intersects(s_region, POLYGON('', """ + polygon + """)) = 1 """ if instrument is not None: instrument_selection = str( """ AND instrument_name='{}'""". format(str(instrument))) query = '\n'.join([query, instrument_selection]) if verbose: msgs.info('The query is:') msgs.info('{}'.format(str(query))) # Obtaining query results result_from_query = tapobs.search(query=query, maxrec=maxrec) if len(result_from_query) < 1: msgs.warning('No data has been retrieved') else: msgs.info( 'A total of {} entries has been retrieved for polygon N.{}' .format(len(result_from_query), iii)) msgs.info('For the following instrument:') for inst_name in np.unique( result_from_query['instrument_name'].data): msgs.info(' - {}'.format(inst_name.decode("utf-8"))) if verbose: result_from_query.to_table().pprint(max_width=-1) results_from_query.append(result_from_query) polygon_union = polygon_union[:-4] if merge: query = """SELECT target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, dataproduct_type, instrument_name, abmaglim, proposal_id FROM ivoa.ObsCore WHERE (""" + polygon_union + """)""" if instrument is not None: instrument_selection = str( """ AND instrument_name='{}'""".format( str(instrument))) query = '\n'.join([query, instrument_selection]) if verbose: msgs.info('The query is:') msgs.info('{}'.format(str(query))) # Obtaining query results result_from_query = tapobs.search(query=query, maxrec=maxrec) if len(result_from_query) < 1: msgs.warning('No data has been retrieved') else: msgs.info( 'A total of {} entries has been retrieved for polygon N.{}' .format(len(result_from_query), iii)) msgs.info('For the following instrument:') for inst_name in np.unique( result_from_query['instrument_name'].data): msgs.info(' - {}'.format(inst_name.decode("utf-8"))) if verbose: result_from_query.to_table().pprint(max_width=-1) results_from_query.append(result_from_query) return results_from_query '''
def get_header_from_archive(file_id, text_file=None): # written by Ema. 04.03.2020 r"""Given a file ID the macro download the corresponding header. Args: file_id (`str`): ESO file ID for which the header will be downloaded text_file (`str`): text file where the header will be downloaded. If `None` it will it will be set to the same string `file_id` but with a `.hdr` extension. """ # checks for connection to ESO archive archive_url = default.get_value('eso_archive_url') if not checks.connection_to_website(archive_url, timeout=1): msgs.error('Cannot connect to the ESO archive website:\n {}'.format( archive_url)) # checks for file id assert isinstance(file_id, list) or isinstance( file_id, (str, np.str)), 'file_id needs to be a str or a list' if isinstance(file_id, str): list_of_files = [file_id] else: list_of_files = file_id list_of_files = [ files if not files.endswith('.fits') else files.replace('.fits', '') for files in list_of_files ] # checks for text_file assert isinstance(text_file, list) or isinstance(text_file, (str, np.str)) or \ isinstance(text_file, (type(None), bytes)), 'text_file needs to be a str or a list' if isinstance(text_file, str): if len(list_of_files) == 1: list_of_outputs = [text_file] else: list_of_outputs = [output + text_file for output in list_of_files] elif isinstance(text_file, list): if len(list_of_files) == len(text_file): list_of_outputs = text_file else: list_of_outputs = [files + '.hdr' for files in list_of_files] else: list_of_outputs = [files + '.hdr' for files in list_of_files] # Downloading headers for file_name, file_out in zip(list_of_files, list_of_outputs): if os.path.isfile(file_out): msgs.warning('Overwriting existing text file: {}'.format(file_out)) os.remove(file_out) url_for_header = archive_url + 'hdr?DpId=' + file_name response_url = requests.get(url_for_header, allow_redirects=True) # Removing html from text header_txt = response_url.text.split('<pre>')[1].split('</pre>')[0] if not header_txt.startswith('No info found for'): file_header = open(file_out, 'w') for line in header_txt.splitlines(): file_header.write(line + '\n') file_header.close() msgs.info('Header successfully saved in: {}'.format(file_out)) else: msgs.warning( '{} is not present in the ESO archive'.format(file_name)) return
def query_from_radec(positions, radius=None, instruments=None, data_types=None, verbose=False, maxrec=default.get_value('maxrec')): r"""Query the ESO TAP service given a position in RA and Dec. The `positions` value (or list) needs to be given as an `astropy.coordinates.SkyCoord` object. Args: positions (`astropy.coordinates.SkyCoord`): Coordinates (or list of coordinates) of the sky you want to query in the format of an `astropy.coordinates.SkyCoord` object. For further detail see here: `astropy coordinates <https://docs.astropy.org/en/stable/coordinates/>`_ radius (`float`): Search radius you want to query in arcseconds. Note that in case `None` is given, the query will be performed with the `INTERSECT(POINT('',RA,Dec), s_region)` clause instead of the `INTERSECT(s_region,CIRCLE('',RA,Dec,radius/3600.))` one. See here for further examples: `tap obs examples <http://archive.eso.org/tap_obs/examples>`_ instruments (`list`): Limit the search to the selected list of instruments (e.g., `XSHOOTER`) data_types (`list`): Limit the search to the selected types of data (e.g., `spectrum`) verbose (`bool`): if set to `True` additional info will be displayed maxrec (`int`): Define the maximum number of file that a single query can return from the ESO archive. The default values is set in the `default.txt` file. Returns: results_from_query (`list`): Results from the query in a list with the same length of the input position. Currently it contains: target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, em_min, dataproduct_type, instrument_name, abmaglim, proposal_id, obs_collection """ # Check inputs: # Working on positions if isinstance(positions, list): positions_list = positions else: positions_list = [positions] for position in positions_list: assert isinstance( position, coordinates.SkyCoord), r'Input positions not a SkyCoord object' # Working on radius if radius is not None: if isinstance(radius, int): radius = float(radius) else: assert isinstance(radius, float), r'Input radius is not a number' # Working on instruments if instruments is not None: if isinstance(instruments, list): instruments_list = instruments else: instruments_list = [instruments] for instrument in instruments_list: assert isinstance( instrument, str), r'Input instrument: {} not valid'.format(instrument) # Working on data_types if data_types is not None: if isinstance(data_types, list): data_types_list = data_types else: data_types_list = [data_types] for data_type in data_types_list: assert isinstance( data_type, str), r'Input data type: {} not valid'.format(data_type) # Running over all positions if verbose: how_many_positions = len(positions_list) if how_many_positions > 1: msgs.work( 'Exploring ESO archive around {} locations in the sky'.format( how_many_positions)) else: msgs.work( 'Exploring ESO archive around the input location in the sky') results_from_query = [] for position, idx in zip(positions_list, range(len(positions_list))): position.transform_to(ICRS) ra, dec = np.float_(position.ra.degree), np.float_(position.dec.degree) msgs.work( 'Running query {} to the ESO archive (out of {} total)'.format( idx + 1, len(positions_list))) # Define query # base query: query = _query_obscore_base() # selection of the location: query = query + _query_obscore_intersect_ra_dec(ra, dec, radius=radius) # selection of the instrument(s) if instruments is not None: query = query + _query_obscore_select_instruments(instruments_list) # selection of the data_type(s) if data_types is not None: query = query + _query_obscore_select_data_types(data_types_list) # running query and append results to the list result_from_query = _run_query(query, verbose=verbose, remove_bytes=True, maxrec=maxrec) if len(result_from_query) < 1: msgs.warning('No data has been retrieved') else: msgs.info('A total of {} entries has been retrieved'.format( len(result_from_query))) if verbose: msgs.info('For the following instrument:') for inst_name in np.unique( result_from_query['instrument_name'].data): msgs.info(' - {}'.format(inst_name)) results_from_query.append(result_from_query) return results_from_query
def parser(options=None): parser = argparse.ArgumentParser( description= r"""Collect data from the ESO archive given RA and Dec in degrees.""" + """\n""" + """\n""" + r"""This uses ESOAsg version {:s}""".format(__version__), formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES) parser.add_argument("-ra", "--ra_degree", nargs="+", type=float, default=None, help=r"RA of the target in degree [J2000]") parser.add_argument("-dec", "--dec_degree", nargs="+", type=float, default=None, help=r"Dec of the target in degree [J2000]") parser.add_argument("-r", "--radius", nargs="+", type=float, default=None, help=r"Search cone radius in arcsec") parser.add_argument( "-i", "--instruments", nargs="+", type=str, default=None, help="Limit the search to the selected ESO instruments") parser.add_argument("-dt", "--data_types", nargs="+", type=str, default=None, help="Limit the search to the selected data type") parser.add_argument( "-m", "--maxrec", nargs="+", type=int, default=default.get_value('maxrec'), help="Maximum number of files you would like to retrieve") parser.add_argument("-v", "--version", action="version", version=__version__) # allows for negative values in dec for i in sys.argv[1:]: if i.startswith('--dec_') or i.startswith('-dec'): sys.argv[sys.argv.index(i) + 1] = str( float(sys.argv[sys.argv.index(i) + 1])) if options is None: args = parser.parse_args() else: args = parser.parse_args(options) return args
def query_catalogue(table_name, which_columns=None, maxrec=default.get_value('maxrec')): r"""Query the ESO tap_cat service (link defined in `ESOAsg\default.txt`) for a specific catalogue. Args: table_name (`str`): Table to be queried. To check the full list of catalogues run `all_catalogues()` which_columns (`list`): List of the columns that you want to download. The full list of the columns in a table can be found running `columns_in_catalogue(table_name)` maxrec (`int`): Define the maximum number of entries that a single query can return. If set to `None` the value is set by the limit of the service. The default values is set in`ESOAsg\default.txt` as `max_rec` Returns: catalogue """ # Check for the table if not _is_table_at_eso(table_name): msgs.error('{} is not a valid table'.format(table_name)) # Check for columns and select the one present in the table if which_columns is not None: test_columns = _are_columns_in_table(which_columns, table_name) good_columns = [ which_column for which_column, test_column in zip(which_columns, test_columns) if test_column ] bad_columns = [ which_column for which_column, test_column in zip(which_columns, test_columns) if not test_column ] if len(bad_columns) > 0: msgs.warning( 'The following columns will be excluded from the query: {}'. format(bad_columns)) if len(good_columns) == 0: good_columns = ['*'] msgs.warning( 'No valid column selected. All the columns will be queried') else: good_columns = ['*'] # Create string of all columns for the query good_columns_string = str(' ') for good_column in good_columns: good_columns_string = good_columns_string + str(good_column) + ', ' good_columns_string = good_columns_string.strip()[:-1] # query query = ''' SELECT {} FROM {} '''.format(good_columns_string, table_name) # Obtaining query results result_from_query = _run_query(query, maxrec=maxrec, verbose=True) return result_from_query