def get_hdul(fits_name, mode='readonly', checksum=True): # Written by Ema 05.03.2020 r"""Wrapper for astropy `fits.open`. It checks if the file exists and in case returns its HDUList. Args: fits_name (str): fits file name mode (str): Open mode for the file. Possibilities are: `readonly’, `update`, `append`, `denywrite`, or `ostream` checksum (bool): If True, verifies that both `DATASUM` and `CHECKSUM` card values (when present in the HDU header) match the header and data of all HDU’s in the file. Updates to a file that already has a checksum will preserve and update the existing checksums unless this argument is given a value of `remove`, in which case the `CHECKSUM` and `DATASUM` values are not checked, and are removed when saving changes to the file Returns: hdul: list-like collection of HDU objects """ if not checks.fits_file_is_valid(fits_name): msgs.error('Fits file not valid') return None else: hdul = fits.open(fits_name, mode=mode, checksum=checksum) msgs.info('The fits file {} contains {} HDUs'.format( fits_name, len(hdul))) msgs.info('Summary:') hdul.info() return hdul
def check_disk_space(min_disk_space=float( default.get_value('min_disk_space'))) -> bool: r"""Check that there is enough space on the location where the code is running Given a disk space limit in GB, the macro returns `True` if the disk where the code is running has more free GB than the given limit. .. warning:: The current implementation checks the disk where the code is running (i.e., from the directory: `./`). This may cause some troubles with shared disks. Args: min_disk_space (float): Size of free space on disk required Returns: bool: `True` if there is enough space on disk """ total, used, free = shutil.disk_usage("./") total = total / (1024.**3.) used = used / (1024.**3.) free = free / (1024.**3.) msgs.info('Your disk has:') msgs.info('Total: {0:.2f} GB, Used: {0:.2f} GB, Free: {0:.2f} GB'.format( total, used, free)) if free > min_disk_space: enough_space = True else: enough_space = False msgs.warning('Not enough space on disk') return enough_space
def _run_query(query, verbose=False, remove_bytes=True, maxrec=default.get_value('maxrec')): r"""Run tap query and return result as a table Args: query (`str`): Query to be run verbose (`bool`): if set to `True` additional info will be displayed remove_bytes ('bool') if set to True, it converts all bytes entries to standard strings maxrec (`int`): Define the maximum number of entries that a single query can return Returns: result_from_query (`astropy.Table`): Result from the query to the TAP service """ # Load tap service tapobs = _define_tap_service(verbose=False) if verbose: msgs.info('The query is: \n {} \n'.format(str(query))) # Obtaining query results and convert it to an astropy table result_from_query = tapobs.search(query=query, maxrec=maxrec).to_table() # removing bytes code from some columns: if remove_bytes: for column_name in result_from_query.colnames: result_from_query[ column_name].data.data[:] = checks.from_bytes_to_string( result_from_query[column_name].data.data) return result_from_query
def download(dp_ids, min_disk_space=float(default.get_value('min_disk_space'))): r"""Given a filename in the ADP format, the code download the file from the `ESO archive <http://archive.eso.org>`_ Args: dp_ids (any): list data product ID (or single product ID) to be downloaded min_disk_space (float): the file will be downloaded only if there is this amount of space (in Gb) free on the disk Returns: None """ # Check for disk space checks.check_disk_space(min_disk_space=min_disk_space) # Cleaning list dp_ids_list = cleaning_lists.from_element_to_list( cleaning_lists.from_bytes_to_string(dp_ids), element_type=str) for dp_id in dp_ids_list: # Given a dp_id of a public file, the link to download it is constructed as follows: download_url = 'http://archive.eso.org/datalink/links?ID=ivo://eso.org/ID?{}&eso_download=file'.format( dp_id) msgs.work('Retrieving file {}.fits'.format(dp_id)) urllib.request.urlretrieve(download_url, filename=dp_id + '.fits') msgs.info('File {}.fits downloaded'.format(dp_id))
def query_ASP_from_polygons(polygons=None, open_link=False, show_link=False): if polygons is not None: for iii, polygon in enumerate(polygons): url = 'http://archive.eso.org/scienceportal/home?' + 'poly=' + polygon + '&sort=-obs_date' if show_link: msgs.info('ASP link to region N.{} is:\n {}\n'.format( np.str(iii + 1), url)) if open_link: webbrowser.open(url)
def query_ASP_from_radec(positions, radius=None, open_link=False, show_link=False): r"""Query the ESO ASP service given a position in RA and Dec. The `positions` value (or list) needs to be given as an `astropy.coordinates.SkyCoord` object. Args: positions (`astropy.coordinates.SkyCoord`): Coordinates (or list of coordinates) of the sky you want to query in the format of an `astropy.coordinates.SkyCoord` object. For further detail see here: `astropy coordinates <https://docs.astropy.org/en/stable/coordinates/>`_ radius (`float`): Search radius you want to query in arcseconds. Note that in case `None` is given, the query will be performed with the `CONTAINS(POINT('',RA,Dec), s_region)` clause instead of the `CONTAINS(s_region,CIRCLE('',RA,Dec,radius/3600.))` one. See here for further examples: `tap obs examples <http://archive.eso.org/tap_obs/examples>`_ open_link (`bool`): open a link to the ASP page show_link (`bool`): show the link on the terminal """ # Check inputs: # Working on positions if isinstance(positions, list): positions_list = positions else: positions_list = [positions] for position in positions_list: assert isinstance( position, coordinates.SkyCoord), r'Input positions not a SkyCoord object' # Working on radius if radius is not None: if isinstance(radius, int): radius = float(radius) else: assert isinstance(radius, float), r'Input radius is not a number' for iii, position in enumerate(positions_list): position.transform_to(ICRS) radec_string = np.str(np.float_(position.ra.degree)) + ',' + np.str( np.float_(position.dec.degree)) radius_string = np.str(radius / 3600.) url = 'http://archive.eso.org/scienceportal/home?' + 'pos=' + radec_string + '&r=' + radius_string + \ '&sort=-obs_date' if show_link: msgs.info('ASP link to region N.{} is:\n {}\n'.format( np.str(iii + 1), url)) if open_link: webbrowser.open(url)
def contours_from_gw_bayestar(file_name, credible_level=50.): r"""Given a bayestar.fits.gz HEALPix maps of a GW superevent it extracts contours at the given `credible_level` This is a wrapper around `ligo_skymap_contour`. The different contours are then counterclockwise oriented to be compatible with TAP and ASP queries. Args: file_name (str): Name of HEALPix maps credible_level (float): Probability level at which contours are returned Returns: list: [RA,Dec] list defining the contours location. These are counter-clockwise oriented as seen on the sky from inside the sphere. The length of contours represent the number of not connected regions identified. """ # Some checks assert isinstance(file_name, (str, np.str)), '{} is not a valid string'.format(file_name) if not checks.fits_file_is_valid(file_name): msgs.error('{} not a valid fits file'.format(file_name)) assert isinstance(credible_level, (int, float, np.int, np.float)), '`credible_level` is not a float or an int' # Create temporary file where to store output from ligo_skymap_contour contour_tmp_file = '.' + file_name + '.tmp.json' if path.isfile(contour_tmp_file): remove(contour_tmp_file) # Use ligo_skymap_contour to compute contours ligo_skymap_contour(args=[file_name, '--output', contour_tmp_file, '--contour', str(credible_level)]) # Parse the content of the resulting json file into a dict with open(contour_tmp_file, 'r') as json_file: json_data = json_file.read() contours_dict = json.loads(json_data) json_file.close() # cleaning up remove(contour_tmp_file) # Parse the resulting json to obtain a list of coordinates defining the vertices of the contours of each peak # contours is a list, each element of which contains the vertices of one contour encircling the desired significance contours = contours_dict['features'][0]['geometry']['coordinates'] # Make sure that the orientation of the polygons on the sky is the correct one for the TAP queries, # i.e. counter-clockwise as seen on the sky from inside the sphere. for iii, contour in enumerate(contours): contours[iii] = _ensure_orientation(contour) # Quick summary: if len(contours) > 0: msgs.info('Extracted the contours for {} regions at {} credible level'.format(len(contours), credible_level)) else: msgs.info('No contours extracted at {} credible level'.format(credible_level)) contours = None return contours
def which_service(tap_service): r"""Print a summary description of the TAP service used Args: tap_service (pyvo.dal.tap.TAPService): TAP service used for the queries Returns: None """ msgs.info('The TAP service used is:') tap_service.describe() return
def fits_file_is_valid(fits_file, verify_fits=False, overwrite=False) -> bool: # Written by Ema 05.03.2020 r"""Check if a file exists and has a valid extension The option `verify_fits` checks the header of the fits file using `astropy.io.fits.verify` Args: fits_file (str): fits file you would like to check verify_fits (bool): if set to `True`, it will verify that the fits file is complaint to the FITS standard. overwrite (bool): if `True`, overwrite the input fits file with the header corrections from `verify_fits` Returns: bool: `True` if exists `False` and warning raised if not. """ is_fits = True # Checks if it is a string assert isinstance(fits_file, str), 'input `fits_file` needs to be a string' # Check for ending # ToDo # to be updated to: PERMITTED_FITS_ENDINGS if not fits_file.endswith('.fits') and not fits_file.endswith( '.fits.fz') and not fits_file.endswith('.fits.gz'): msgs.warning( 'File: {} does not end with `fits` or `fits.fz` or `fits.gz`'. format(fits_file)) is_fits = False # Check for existence if not os.path.exists(fits_file): msgs.warning('File: {} does not exists'.format(fits_file)) is_fits = False # Check for compliance with FITS standard if verify_fits: if overwrite: hdul = fits.open(fits_file, mode='update', checksum=False) if not check_checksums(hdul): is_fits = False hdul.flush(output_verify='fix+warn', verbose=True) hdul.writeto(fits_file, checksum=True, overwrite=True) msgs.info('File checked and rewritten') else: hdul = fits.open(fits_file, mode='readonly', checksum=True) if not check_checksums(hdul): is_fits = False hdul.verify('fix+warn') hdul.close() else: if overwrite: msgs.error('The option overwrite works only if verify_fits = True') return is_fits
def prodcatg(self, prodcatg_type): self.__prodcatg = eso_prodcatg.ProdCatg(prodcatg_type=prodcatg_type) if self.is_primary is True: _prodcatg_value = self.get('PRODCATG', default=None) if _prodcatg_value is None: msgs.info( 'Added PRODCATG = {} to the header'.format(prodcatg_type)) self.set('PRODCATG', prodcatg_type) elif _prodcatg_value != prodcatg_type: msgs.warning('Updating value fo PRODCATG from {} to {}'.format( _prodcatg_value, prodcatg_type)) self.set('PRODCATG', prodcatg_type) elif _prodcatg_value == prodcatg_type: msgs.info('PRODCATG = {}'.format(_prodcatg_value)) else: msgs.error('Cannot set the value of PRODCATG')
def run_query(query_url, show_link=True, open_link=True): r"""Run the ASP query Args: query_url (str): url of the ASP query open_link (bool): open a link to the ASP page show_link (bool): show the link on the terminal Returns: None """ if show_link: msgs.info('The ASP link is:\n {}\n'.format(query_url)) if open_link: webbrowser.open(query_url) return
def print_query(query): r"""Print the query on the terminal In case the `query` is empty, a warning is raised Args: query (str): String containing the query Returns: None """ if query is None: msgs.warning('The query is empty') else: msgs.info('The query is:') msgs.info('{}'.format(query)) return
def _define_tap_service(verbose=False): r"""Load tap service from defaults The TAP service for raw. reduced, and ambient data is defined in `ESOAsg\default.txt` as `eso_tap_obs` Args: verbose (`bool`): if set to `True` additional info will be displayed Returns: tapcat (`pyvo.dal.tap.TAPService`) TAP service that will be used for the queries """ if verbose: msgs.info('Querying the ESO TAP service at:') msgs.info('{}'.format(str(default.get_value('eso_tap_obs')))) tapobs = dal.tap.TAPService(default.get_value('eso_tap_obs')) return tapobs
def download(dp_id, min_disk_space=np.float32(default.get_value('min_disk_space'))): r"""Given a filename in the ADP format, the code download the file from the `ESO archive <http://archive.eso.org>`_ ..note:: if dp_id is not a `numpy.str`, a WARNING message will be raised and the content of `dp_id` will be converted into a string. Args: dp_id (`numpy.str`): Data product ID to be downloaded. min_disk_space (`numpy.float`): The file will be downloaded only if there is this amount of space (in Gb) free on the disk. By default is set by the `default.txt` file. Returns: This downloads a fits ADP file with the same name of the input. """ # Check for disk space checks.check_disk_space(min_disk_space=min_disk_space) for file_name in dp_id: # if the file name is in byte, this decode it. if not isinstance(file_name, str): msgs.warning('The content of dp_id is not in a string format.') msgs.warning('The code is trying to fix this.') if isinstance(file_name, bytes): file_name = np.str(file_name.decode("utf-8")) msgs.warning('Converted to {}.'.format(type(file_name))) else: msgs.error( 'Unable to understand the format of the dp_id entry: {}'. format(type(file_name))) # Given a dp_id of a public file, the link to download it is constructed as follows: download_url = 'http://archive.eso.org/datalink/links?ID=ivo://eso.org/ID?{}&eso_download=file'.format( str(file_name)) msgs.work( 'Downloading file {}. This may take some time.'.format(file_name + '.fits')) urllib.request.urlretrieve(download_url, filename=file_name + '.fits') msgs.info('File {} downloaded.'.format(file_name + '.fits'))
def download_gw_bayestar(superevent_name, file_name='bayestar.fits.gz'): r"""Download the bayestar.fits.gz of a GW superevent This is simply checking the existence of: `https://gracedb.ligo.org/superevents/<superevent_name>/files/` and downloading the file: `https://gracedb.ligo.org/apiweb/superevents/<superevent_name>/iles/bayestar.fits.gz` The downloaded file will be renamed as `superevent_name`+`file_name` Args: superevent_name (str): name of the superevent: e.g., 'S191205ah' file_name (str): name of the file to download. Default is `bayestar.fits.gz` Returns: bool: it returns `True` if data are retrieved and `False` otherwise. """ # Some checks assert isinstance(superevent_name, str), '{} is not a valid string'.format(superevent_name) assert isinstance(file_name, str), '{} is not a valid string'.format(file_name) # Checks if superevent exists gw_files_url = 'https://gracedb.ligo.org/superevents/' + superevent_name + '/files/' checks.connection_to_website(gw_files_url) # download the and save the superevent file gw_bayestar_url = 'https://gracedb.ligo.org/apiweb/superevents/' + superevent_name + '/files/' + file_name r = requests.get(gw_bayestar_url, allow_redirects=True) if r.status_code == 404: msgs.warning('Failed to access to: {}'.format(gw_bayestar_url)) return False open(superevent_name + '_bayestar.fits.gz', 'wb').write(r.content) # check that the file actually arrived on disk if path.isfile(superevent_name + '_bayestar.fits.gz'): _test_header = fitsfiles.header_from_fits_file(superevent_name + '_' + file_name) if len(_test_header) > 0: msgs.info('File {}_{} successfully downloaded'.format(superevent_name, file_name)) else: msgs.warning('Not a valid fits file in: {}'.format(gw_bayestar_url)) return False else: msgs.warning('Failed to download: {}'.format(gw_bayestar_url)) return False return True
def _define_tap_service(verbose=False): r"""Load tap service from defaults The TAP service for scientific catalogues generated by ESO observing teams is defined in `ESOAsg\default.txt` as `eso_tap_cat` Args: verbose (`bool`): if set to `True` additional info will be displayed Returns: tapcat (`pyvo.dal.tap.TAPService`) TAP service that will be used for the queries. For more details on this see the `pyvo documents<https://pyvo.readthedocs.io/en/latest/>`_ """ if verbose: msgs.info('Querying the ESO TAP service at:') msgs.info('{}'.format(str(default.get_value('eso_tap_cat')))) tapcat = dal.tap.TAPService(default.get_value('eso_tap_cat')) return tapcat
def main(args): from astropy import coordinates from astropy import units as u from ESOAsg import archive_observations from ESOAsg import msgs from ESOAsg.ancillary import cleaning_lists ra_list = cleaning_lists.from_element_to_list(args.ra_degree, element_type=float) dec_list = cleaning_lists.from_element_to_list(args.dec_degree, element_type=float) if args.radius is None: radius = None else: radius = cleaning_lists.from_element_to_list(args.radius, element_type=float)[0] instruments = cleaning_lists.from_element_to_list(args.instruments, element_type=str) data_types = cleaning_lists.from_element_to_list(args.data_types, element_type=str) msgs.start() for ra, dec in zip(ra_list, dec_list): position = coordinates.SkyCoord(ra=ra * u.degree, dec=dec * u.degree, frame='fk5') msgs.newline() msgs.info('Query for ESO archival data around the position: {}'.format( position.to_string('hmsdms'))) result_from_query = archive_observations.query_from_radec( position, radius=radius, instruments=instruments, data_types=data_types, verbose=False) if len(result_from_query['dp_id']) > 0: archive_observations.download(result_from_query['dp_id']) msgs.end()
def _run_query(query, verbose=False, maxrec=default.get_value('maxrec')): r"""Run tap query and return result as a table Args: query (`str`): Query to be run verbose (`bool`): if set to `True` additional info will be displayed maxrec (`int`, `None`): Define the maximum number of entries that a single query can return Returns: result_from_query (`astropy.Table`): Result from the query to the TAP service """ # Load tap service tapcat = _define_tap_service(verbose=False) if verbose: msgs.info('The query is:') msgs.info('{}'.format(str(query))) # Obtaining query results and convert it to an astropy table result_from_query = tapcat.search(query=query, maxrec=maxrec).to_table() return result_from_query
def run_query_async(tap_service, query, maxrec=default.get_value('maxrec')): r"""Run an asynchronous query to TAP service and return result as an `astropy.Table` Args: tap_service (pyvo.dal.tap.TAPService): TAP service that will be used for the query query (str): query to be run maxrec (int): define the maximum number of entries that a single query can return. Default is set by default.get_value('maxrec') Returns: astropy.table: result from the query to the TAP service """ tap_job = tap_service.submit_job(query=query, maxrec=maxrec) tap_job.run() # Wait for Executing tap_job.wait(phases=["EXECUTING", "ERROR", "ABORTED"], timeout=10.) msgs.info('The query to the tap_service is in the status: {}'.format(str(tap_job.phase))) # Wait for Completed tap_job.wait(phases=["COMPLETED", "ERROR", "ABORTED"], timeout=10.) msgs.info('The query to the tap_service is in the status: {}'.format(str(tap_job.phase))) # Fetch the results tap_job.raise_if_error() return tap_job.fetch_result().to_table()
def query_TAP_from_polygons(polygons=None, merge=False, instrument=None, maxrec=default.get_value('maxrec'), verbose=False): tapobs = dal.tap.TAPService(default.get_value('eso_tap_obs')) msgs.info('Querying the ESO TAP service at:') msgs.info('{}'.format(str(default.get_value('eso_tap_obs')))) results_from_query = [] if polygons is not None: polygon_union = '' for iii, polygon in enumerate(polygons): polygon_union += """intersects(s_region, POLYGON('', """ + polygon + """)) = 1 OR """ if not merge: query = """SELECT target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, dataproduct_type, instrument_name, abmaglim, proposal_id FROM ivoa.ObsCore WHERE intersects(s_region, POLYGON('', """ + polygon + """)) = 1 """ if instrument is not None: instrument_selection = str( """ AND instrument_name='{}'""". format(str(instrument))) query = '\n'.join([query, instrument_selection]) if verbose: msgs.info('The query is:') msgs.info('{}'.format(str(query))) # Obtaining query results result_from_query = tapobs.search(query=query, maxrec=maxrec) if len(result_from_query) < 1: msgs.warning('No data has been retrieved') else: msgs.info( 'A total of {} entries has been retrieved for polygon N.{}' .format(len(result_from_query), iii)) msgs.info('For the following instrument:') for inst_name in np.unique( result_from_query['instrument_name'].data): msgs.info(' - {}'.format(inst_name.decode("utf-8"))) if verbose: result_from_query.to_table().pprint(max_width=-1) results_from_query.append(result_from_query) polygon_union = polygon_union[:-4] if merge: query = """SELECT target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, dataproduct_type, instrument_name, abmaglim, proposal_id FROM ivoa.ObsCore WHERE (""" + polygon_union + """)""" if instrument is not None: instrument_selection = str( """ AND instrument_name='{}'""".format( str(instrument))) query = '\n'.join([query, instrument_selection]) if verbose: msgs.info('The query is:') msgs.info('{}'.format(str(query))) # Obtaining query results result_from_query = tapobs.search(query=query, maxrec=maxrec) if len(result_from_query) < 1: msgs.warning('No data has been retrieved') else: msgs.info( 'A total of {} entries has been retrieved for polygon N.{}' .format(len(result_from_query), iii)) msgs.info('For the following instrument:') for inst_name in np.unique( result_from_query['instrument_name'].data): msgs.info(' - {}'.format(inst_name.decode("utf-8"))) if verbose: result_from_query.to_table().pprint(max_width=-1) results_from_query.append(result_from_query) return results_from_query '''
def get_header_from_archive(file_id, text_file=None): # written by Ema. 04.03.2020 r"""Given a file ID the macro download the corresponding header. Args: file_id (`str`): ESO file ID for which the header will be downloaded text_file (`str`): text file where the header will be downloaded. If `None` it will it will be set to the same string `file_id` but with a `.hdr` extension. """ # checks for connection to ESO archive archive_url = default.get_value('eso_archive_url') if not checks.connection_to_website(archive_url, timeout=1): msgs.error('Cannot connect to the ESO archive website:\n {}'.format( archive_url)) # checks for file id assert isinstance(file_id, list) or isinstance( file_id, (str, np.str)), 'file_id needs to be a str or a list' if isinstance(file_id, str): list_of_files = [file_id] else: list_of_files = file_id list_of_files = [ files if not files.endswith('.fits') else files.replace('.fits', '') for files in list_of_files ] # checks for text_file assert isinstance(text_file, list) or isinstance(text_file, (str, np.str)) or \ isinstance(text_file, (type(None), bytes)), 'text_file needs to be a str or a list' if isinstance(text_file, str): if len(list_of_files) == 1: list_of_outputs = [text_file] else: list_of_outputs = [output + text_file for output in list_of_files] elif isinstance(text_file, list): if len(list_of_files) == len(text_file): list_of_outputs = text_file else: list_of_outputs = [files + '.hdr' for files in list_of_files] else: list_of_outputs = [files + '.hdr' for files in list_of_files] # Downloading headers for file_name, file_out in zip(list_of_files, list_of_outputs): if os.path.isfile(file_out): msgs.warning('Overwriting existing text file: {}'.format(file_out)) os.remove(file_out) url_for_header = archive_url + 'hdr?DpId=' + file_name response_url = requests.get(url_for_header, allow_redirects=True) # Removing html from text header_txt = response_url.text.split('<pre>')[1].split('</pre>')[0] if not header_txt.startswith('No info found for'): file_header = open(file_out, 'w') for line in header_txt.splitlines(): file_header.write(line + '\n') file_header.close() msgs.info('Header successfully saved in: {}'.format(file_out)) else: msgs.warning( '{} is not present in the ESO archive'.format(file_name)) return
if __name__ == '__main__': args = parse_arguments() # getting fits names input_fits = np.str(args.input_fits[0]) if args.output_fits is None: output_fits = input_fits.replace('.fits', '_fixed.fits') else: output_fits = np.str(args.output_fits[0]) msgs.start() hdul_original = fits.open(input_fits) msgs.work('Updating checksum and datasum') hdul[0].add_datasum() hdul[1].add_datasum() hdul[0].add_checksum(override_datasum=True) hdul[1].add_checksum(override_datasum=True) msgs.work('Flushing changes.') hdul.flush() hdul.close() msgs.newline() msgs.info('File {} produced.'.format(output_fits)) msgs.end()
def main(args): import numpy as np import os import shutil from astropy.coordinates import SkyCoord from astropy.coordinates import name_resolve from astropy import units as u from astropy.io import fits from ESOAsg.ancillary import cleaning_lists from ESOAsg.core import fitsfiles from ESOAsg import msgs # Cleaning input lists input_fits_files = cleaning_lists.make_list_of_fits_files(args.input_fits) # Make whitelight images if args.whitelight: make_whitelight_image = True else: make_whitelight_image = False # Creating output list if args.suffix is None: overwrite = False msgs.warning('The file will overwrite the input files') else: overwrite = True suffix_string = cleaning_lists.make_string(args.suffix) output_fits_files = cleaning_lists.make_list_of_fits_files_and_append_suffix( input_fits_files, suffix=suffix_string) if make_whitelight_image: output_whitelight_files = cleaning_lists.make_list_of_fits_files_and_append_suffix( input_fits_files, suffix=suffix_string + '_WL') else: output_whitelight_files = [None] * len(input_fits_files) ''' # reference if args.referenc is not None: reference = str(args.referenc[0]) else: reference = str(' ') # fluxcal if args.fluxcal == 'ABSOLUTE': fluxcal = 'ABSOLUTE' elif args.fluxcal == 'UNCALIBRATED': fluxcal = 'UNCALIBRATED' else: msgs.error('Possible values for fluxcal are: `ABSOLUTE` or `UNCALIBRATED`') # abmaglim if args.abmaglim is not None: abmaglim = args.abmaglim assert isinstance(abmaglim, (int, np.float_)), 'ABMAGLIM must be a float' if abmaglim < 0: msgs.error('ABMAGLIM must be positive') else: abmaglim = np.float_(-1.) ''' msgs.start() for fits_in, fits_out, image_out in zip(input_fits_files, output_fits_files, output_whitelight_files): if os.path.exists(fits_out): shutil.copy(fits_out, fits_out.replace('.fit', '_old.fit')) msgs.warning('{} already exists. Backup created.'.format(fits_out)) if image_out is not None: if os.path.exists(image_out): shutil.copy(image_out, image_out.replace('.fit', '_old.fit')) msgs.warning( '{} already exists. Backup created.'.format(image_out)) full_hdul = fitsfiles.get_hdul(fits_in) try: instrument = full_hdul[0].header['HIERARCH ESO SEQ ARM'] except KeyError: msgs.error( 'Failed to read the keyword HIERARCH ESO SEQ ARM from the primary header' ) finally: if instrument in SUPPORTED_INSTRUMENT: msgs.info( 'The input file is from SPHERE/{}'.format(instrument)) else: msgs.warning( 'Instrument SPHERE/{} not supported'.format(instrument)) # ToDo # These needs to be transformed in objects if instrument.startswith('IFS'): msgs.work('Fixing header for SPHERE/{} file {}'.format( instrument, fits_in)) # Create a copy of the file where there is a primary HDU and data are in the 'DATA" HDU msgs.work('Reshaping cube into PrimaryHEADER and Data Header') fitsfiles.new_fits_like(fits_in, [0], fits_out, overwrite=overwrite, fix_header=True) hdul = fitsfiles.get_hdul(fits_out, 'update', checksum=True) hdr0 = hdul[0].header hdr1 = hdul[1].header # Check for HISTORY # Primary Header if 'HISTORY' in hdr0.keys(): history_cards_hdr0 = [ history_card_hdr0 for history_card_hdr0 in hdr0 if history_card_hdr0.startswith('HISTORY') ] history_values_hdr0 = [ hdr0[history_card_hdr0] for history_card_hdr0 in hdr0 if history_card_hdr0.startswith('HISTORY') ] for history_card_hdr0, history_value_hdr0 in zip( history_cards_hdr0, history_values_hdr0): msgs.work('Cleaning cards: {} = {}'.format( history_card_hdr0, history_value_hdr0)) del hdr0['HISTORY'][:] # Data Header if 'HISTORY' in hdr1.keys(): history_values_hdr1 = hdr1['HISTORY'][:] for history_number in range(0, len(history_values_hdr1)): clean_history = cleaning_lists.remove_non_ascii( history_values_hdr1[history_number]) if len(clean_history) > 0: hdr1['HISTORY'][history_number] = str(clean_history) else: hdr1['HISTORY'][history_number] = str(' ') # Update cards for headers: # Updating values with different CARD in the header cards_input = [ 'CRPIX4', 'CRVAL4', 'CTYPE4', 'CUNIT4', 'CD4_4', 'CD1_4', 'CD2_4', 'CD4_1', 'CD4_2' ] cards_output = [ 'CRPIX3', 'CRVAL3', 'CTYPE3', 'CUNIT3', 'CD3_3', 'CD1_3', 'CD2_3', 'CD3_1', 'CD3_2' ] fitsfiles.transfer_header_cards(hdr1, hdr1, cards_input, output_cards=cards_output, delete_card=True) # Remove not used values cards_to_be_removed_hdr1 = ['CD4_3', 'CD3_4'] for card_to_be_removed_hdr1 in cards_to_be_removed_hdr1: hdr1.remove(card_to_be_removed_hdr1, ignore_missing=True) # Transfer cards from HDU1 to the PrimaryHDU not_to_be_transfer = [ hdr1_card for hdr1_card in hdr1 if hdr1_card.startswith('COMMENT') or hdr1_card.startswith( 'EXTNAME') or hdr1_card.startswith('BITPIX') or hdr1_card.startswith('NAXIS') or hdr1_card.startswith('CRPIX') or hdr1_card.startswith('CRVAL') or hdr1_card.startswith( 'CDELT') or hdr1_card.startswith('CTYPE') or hdr1_card.startswith('CD1_') or hdr1_card.startswith('CD2_') or hdr1_card.startswith('CD3_') or hdr1_card.startswith( 'CUNIT') or hdr1_card.startswith('CSYER') or hdr1_card. startswith('HDUCLAS') or hdr1_card.startswith('XTENSION') or hdr1_card.startswith('PCOUNT') or hdr1_card.startswith( 'GCOUNT') or hdr1_card.startswith('HDUDOC') or hdr1_card. startswith('HDUVER') or hdr1_card.startswith('HISTORY') ] cards_to_be_transfer = [] for hdr1_card in hdr1: if hdr1_card not in not_to_be_transfer: cards_to_be_transfer.append(hdr1_card) fitsfiles.transfer_header_cards(hdr1, hdr0, cards_to_be_transfer, with_comment=True, delete_card=True) # Try to guess coordinates if 'CRVAL1' not in hdr1.keys(): msgs.warning('CRVAL position keywords not preset') if 'OBJECT' in hdr0.keys(): try: object_coordinate = SkyCoord.from_name( str(hdr0['OBJECT']).strip()) ra_obj, dec_obj = object_coordinate.ra.degree, object_coordinate.dec.degree if 'RA' in hdr0.keys() and 'DEC' in hdr0.keys(): pointing_coordinate = SkyCoord(float(hdr0['RA']), float(hdr0['DEC']), unit='deg') msgs.work( 'Testing from separation from pointing position' ) separation = object_coordinate.separation( pointing_coordinate).arcsec if separation < 120.: msgs.info('Object - Pointing separation is {}'. format(separation)) msgs.info( 'Updating CRVAL1 = {}'.format(ra_obj)) msgs.info( 'Updating CRVAL2 = {}'.format(dec_obj)) hdr1['CRVAL1'] = ra_obj hdr1['CRVAL2'] = dec_obj msgs.work('Updating CUNIT') hdr1['CUNIT1'] = 'deg' hdr1['CUNIT2'] = 'deg' msgs.work('Updating CTYPE') hdr1['CTYPE1'] = 'RA---TAN' hdr1['CTYPE2'] = 'DEC--TAN' msgs.work('Updating CRPIX') hdr1['CRPIX1'] = float( hdul[1].data.shape[2]) / 2. hdr1['CRPIX2'] = float( hdul[1].data.shape[1]) / 2. msgs.info('Updating CD1 and CD2') hdr1['CD1_1'] = 2.06E-06 hdr1['CD2_2'] = 2.06E-06 hdr1['CD1_2'] = 0. hdr1['CD2_1'] = 0. msgs.work('Updating RA, DEC') hdr0['RA'] = ra_obj hdr0.comments[ 'RA'] = object_coordinate.ra.to_string( u.hour) hdr0['DEC'] = dec_obj hdr0.comments[ 'DEC'] = object_coordinate.dec.to_string( u.degree, alwayssign=True) else: msgs.warning( 'Object - Pointing separation is {}'. format(separation)) msgs.warning( 'This is suspicious, CRVAL not updated') except name_resolve.NameResolveError: msgs.warning('Object {} not recognized'.format( str(hdr0['OBJECT']).strip())) msgs.warning('CRVAL not updated') # Updating file prodcatg msgs.work('Updating PRODCATG to SCIENCE.CUBE.IFS') hdr0['PRODCATG'] = str('SCIENCE.CUBE.IFS') # Some more updates msgs.work('Setting NAXIS = 0 in primary header') hdr0['NAXIS'] = 0 if 'OBSTECH' not in hdr0.keys(): msgs.warning('OBSTECH missing') if 'ESO PRO TECH' in hdr0.keys(): msgs.info('Deriving OBSTECH from HIERARCH ESO PRO TECH') msgs.work('Updating OBSTECH to {}'.format( str(hdr0['HIERARCH ESO PRO TECH']))) hdr0['OBSTECH'] = str(hdr0['HIERARCH ESO PRO TECH']) if 'EXPTIME' not in hdr0.keys(): msgs.warning('EXPTIME missing') if 'ESO DET SEQ1 REALDIT' in hdr0.keys( ) and 'ESO DET NDIT' in hdr0.keys(): msgs.info('Deriving EXPTIME and TEXPTIME as REALDIT * DIT') hdr0['EXPTIME'] = hdr0[ 'HIERARCH ESO DET SEQ1 REALDIT'] * hdr0[ 'HIERARCH ESO DET NDIT'] hdr0['TEXPTIME'] = hdr0[ 'HIERARCH ESO DET SEQ1 REALDIT'] * hdr0[ 'HIERARCH ESO DET NDIT'] msgs.work('Updating EXPTIME to {}'.format( str(hdr0['EXPTIME']))) msgs.work('Updating TEXPTIME to {}'.format( str(hdr0['TEXPTIME']))) if 'WAVELMIN' not in hdr0.keys(): msgs.warning('WAVELMIN missing') z_pixel = np.arange(int(hdul[1].data.shape[0])) z_wave = float( hdr1['CRVAL3']) + (z_pixel * float(hdr1['CD3_3'])) if str(hdr1['CUNIT3']).strip().upper() == 'MICRONS': msgs.info('Deriving WAVELMIN and WAVELMAX from CRVAL1') z_wave = z_wave * 1000. # convert to nanometers hdr0['WAVELMIN'] = np.nanmin(z_wave) hdr0['WAVELMAX'] = np.nanmax(z_wave) msgs.work('Updating WAVELMIN to {}'.format( str(hdr0['WAVELMIN']))) msgs.work('Updating WAVELMAX to {}'.format( str(hdr0['WAVELMAX']))) else: msgs.warning( 'Unknown units {}. WAVELMIN and WAVELMAX not calculated' .format(str(hdr1['CUNIT3']))) if 'SPEC_RES' not in hdr0.keys(): msgs.warning('SPEC_RES missing') if 'WAVELMAX' in hdr0.keys(): msgs.info('Deriving SPEC_RES from WAVELMAX') if (float(hdr0['WAVELMAX']) > 1300.) and (float( hdr0['WAVELMAX']) < 1400.): hdr0['SPEC_RES'] = 50. msgs.work('Updating SPEC_RES to {}'.format( str(hdr0['SPEC_RES']))) elif (float(hdr0['WAVELMAX']) > 1600.) and (float( hdr0['WAVELMAX']) < 1700.): hdr0['SPEC_RES'] = 30. msgs.work('Updating SPEC_RES to {}'.format( str(hdr0['SPEC_RES']))) else: msgs.warning('WAVELMAX = {} is not in the expected ' / + 'range of possible values'.format( str(hdr0['WAVELMAX']))) if 'PROGID' not in hdr0.keys(): msgs.warning('PROG_ID missing') if 'ESO OBS PROG ID' in hdr0.keys(): msgs.info('Deriving PROG_ID from HIERARCH ESO OBS PROG ID') msgs.work('Updating PROG_ID to {}'.format( str(hdr0['HIERARCH ESO OBS PROG ID']))) hdr0['PROG_ID'] = str(hdr0['HIERARCH ESO OBS PROG ID']) if 'MJD-END' not in hdr0.keys(): msgs.warning('MJD-END missing') if 'TEXPTIME' in hdr0.keys(): msgs.info('Deriving MJD-END from MJD-OBS and TEXPTIME') texptime_sec = float(hdr0['TEXPTIME']) texptime_day = texptime_sec / (60. * 60. * 24.) mjdend = float(hdr0['MJD-OBS']) + texptime_day fitsfiles.add_header_card(hdr0, 'MJD-END', mjdend, 'End of observation') msgs.work('MJD-OBS = {} and TEXPTIME = {} days'.format( str(hdr0['MJD-OBS']), str(texptime_day))) msgs.work('Updating MJD-END to {}'.format( str(hdr0['MJD-END']))) # Remove not used values cards_to_be_removed_hdr0 = ['ERRDATA', 'QUALDATA', 'SCIDATA'] for card_to_be_removed_hdr0 in cards_to_be_removed_hdr0: hdr0.remove(card_to_be_removed_hdr0, ignore_missing=True) cards_to_be_removed_hdr1 = ['HDUCLASS3'] for card_to_be_removed_hdr1 in cards_to_be_removed_hdr1: hdr1.remove(card_to_be_removed_hdr1, ignore_missing=True) # Updating the FITS file definition comment line hdr0.add_comment( " FITS (Flexible Image Transport System) format is defined in 'Astronomy" + " and " "Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H", after='EXTEND') if 'COMMENT' in hdr1.keys(): comment_values_hdr1 = hdr1['COMMENT'][:] for index, comment_value_hdr1 in enumerate( comment_values_hdr1): msgs.work('Removing COMMENT card : {}'.format( comment_value_hdr1)) hdr1.remove('COMMENT', ignore_missing=True, remove_all=True) # Creating white light image keyword: if make_whitelight_image: fitsfiles.add_header_card( hdr0, 'ASSON1', image_out.split('/')[-1], 'ANCILLARY.IMAGE.WHITELIGHT filename') msgs.work('Updating ASSON1 to {}'.format(hdr0['ASSON1'])) # Actually creating the white-light image if make_whitelight_image: msgs.info('Making white light image') image_hdu = fits.PrimaryHDU() image_hdul = fits.HDUList([image_hdu]) if str(hdr1['CUNIT3']).strip().upper() == 'MICRONS': to_ang = 10000. else: msgs.error('Spectral unit: {} not recognized'.format( hdr1['CUNIT3'])) delta_wave_bin = hdr1['CD3_3'] image_hdul.append( fits.ImageHDU( to_ang * delta_wave_bin * np.nansum(hdul[1].data, axis=0, dtype=np.float_))) image_hdr0 = image_hdul[0].header image_hdr1 = image_hdul[1].header card_for_image0 = [ 'WAVELMIN', 'WAVELMAX', 'OBJECT', 'TELESCOP', 'INSTRUME', 'RADECSYS', 'RA', 'DEC', 'EQUINOX' ] fitsfiles.transfer_header_cards(hdr0, image_hdr0, card_for_image0, with_comment=True, delete_card=False) image_hdr0['PRODCATG'] = str('ANCILLARY.IMAGE.WHITELIGHT') card_for_image1 = [ 'CRPIX1', 'CRPIX2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2', 'NAXIS1', 'NAXIS2', 'EXTNAME', 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2', 'CTYPE1', 'CTYPE2' ] fitsfiles.transfer_header_cards(hdr1, image_hdr1, card_for_image1, with_comment=True, delete_card=False) # Update checksum and datasum msgs.work('Updating checksum and datasum') hdul[0].add_checksum(override_datasum=False) hdul[1].add_datasum() hdul[1].add_checksum(override_datasum=True) hdul.flush(output_verify='fix') hdul.close() msgs.info('File {} produced.'.format(fits_out)) if make_whitelight_image: image_hdul[0].add_datasum() image_hdul[1].add_datasum() image_hdul[0].add_checksum(override_datasum=True) image_hdul[1].add_checksum(override_datasum=True) image_hdul.writeto(image_out, overwrite=True, output_verify='fix') msgs.info('Image {} produced.'.format(image_out)) elif instrument.startswith('IRDIS'): msgs.work('Fixing header for SPHERE/{} file {}'.format( instrument, fits_in)) hdr = fitsfiles.header_from_fits_file(fits_in) if 'ESO DPR TECH' in hdr.keys(): if str(hdr['ESO DPR TECH']).strip( ) == 'IMAGE,DUAL,CORONOGRAPHY': msgs.work('Working with {} as observing technique'.format( str(hdr['ESO DPR TECH']).strip())) elif 'DUAL' in str(hdr['ESO DPR TECH']).strip(): msgs.error('{} needs to be tested'.format( str(hdr['ESO DPR TECH']).strip())) else: msgs.error('Only DUAL imaging currently implemented') else: msgs.error('Cannot recognize the observing technique') # defining the two fits_out files: fits_out_index = [0, 1] fits_out_files = [] for index in fits_out_index: fits_out_file = fits_out.replace('.fit', '_' + str(index) + '.fit') if os.path.exists(fits_out_file): shutil.copy(fits_out_file, fits_out_file.replace('.fit', '_old.fit')) msgs.warning( '{} already exists. Backup created.'.format(fits_out)) fitsfiles.new_fits_like(fits_in, [0], fits_out_file, overwrite=overwrite, fix_header=True, empty_primary_hdu=False) fits_out_files.append(fits_out_file) for index, fits_out_file in zip(fits_out_index, fits_out_files): hdul = fitsfiles.get_hdul(fits_out_file, 'update', checksum=True) hdr0 = hdul[0].header hdul[0].data = hdul[0].data[index, :, :] # Check for HISTORY # Primary Header if 'HISTORY' in hdr0.keys(): history_cards_hdr0 = [ history_card_hdr0 for history_card_hdr0 in hdr0 if history_card_hdr0.startswith('HISTORY') ] history_values_hdr0 = [ hdr0[history_card_hdr0] for history_card_hdr0 in hdr0 if history_card_hdr0.startswith('HISTORY') ] for history_card_hdr0, history_value_hdr0 in zip( history_cards_hdr0, history_values_hdr0): msgs.work('Cleaning cards: {} = {}'.format( history_card_hdr0, history_value_hdr0)) del hdr0['HISTORY'][:] # Try to guess coordinates if 'CRVAL1' not in hdr0.keys(): msgs.warning('CRVAL position keywords not preset') if 'OBJECT' in hdr0.keys(): try: object_coordinate = SkyCoord.from_name( str(hdr0['OBJECT']).strip()) ra_obj, dec_obj = object_coordinate.ra.degree, object_coordinate.dec.degree if 'RA' in hdr0.keys() and 'DEC' in hdr0.keys(): pointing_coordinate = SkyCoord( float(hdr0['RA']), float(hdr0['DEC']), unit='deg') msgs.work( 'Testing from separation from pointing position' ) separation = object_coordinate.separation( pointing_coordinate).arcsec if separation < 120.: msgs.info( 'Object - Pointing separation is {}'. format(separation)) msgs.info( 'Updating CRVAL1 = {}'.format(ra_obj)) msgs.info( 'Updating CRVAL2 = {}'.format(dec_obj)) hdr0['CRVAL1'] = ra_obj hdr0['CRVAL2'] = dec_obj msgs.work('Updating CUNIT') hdr0['CUNIT1'] = 'deg' hdr0['CUNIT2'] = 'deg' msgs.work('Updating CTYPE') hdr0['CTYPE1'] = 'RA---TAN' hdr0['CTYPE2'] = 'DEC--TAN' msgs.work('Updating CRPIX') hdr0['CRPIX1'] = float( hdul[0].data.shape[1]) / 2. hdr0['CRPIX2'] = float( hdul[0].data.shape[0]) / 2. msgs.info('Updating CD1 and CD2') hdr0['CD1_1'] = hdr0[ 'PIXSCAL'] * 2.778E-4 / 1000. hdr0['CD2_2'] = hdr0[ 'PIXSCAL'] * 2.778E-4 / 1000. hdr0['CD1_2'] = 0. hdr0['CD2_1'] = 0. msgs.work('Updating RA, DEC') hdr0['RA'] = ra_obj hdr0.comments[ 'RA'] = object_coordinate.ra.to_string( u.hour) hdr0['DEC'] = dec_obj hdr0.comments[ 'DEC'] = object_coordinate.dec.to_string( u.degree, alwayssign=True) else: msgs.warning( 'Object - Pointing separation is {}'. format(separation)) msgs.warning( 'This is suspicious, CRVAL not updated' ) except name_resolve.NameResolveError: msgs.warning('Object {} not recognized'.format( str(hdr0['OBJECT']).strip())) msgs.warning('CRVAL not updated') # Updating file prodcatg msgs.work('Updating PRODCATG to SCIENCE.IMAGE') hdr0['PRODCATG'] = str('SCIENCE.IMAGE') if 'PROGID' not in hdr0.keys(): msgs.warning('PROG_ID missing') if 'ESO OBS PROG ID' in hdr0.keys(): msgs.info( 'Deriving PROG_ID from HIERARCH ESO OBS PROG ID') msgs.work('Updating PROG_ID to {}'.format( str(hdr0['HIERARCH ESO OBS PROG ID']))) hdr0['PROG_ID'] = str(hdr0['HIERARCH ESO OBS PROG ID']) # Update checksum and datasum msgs.work('Updating checksum and datasum') hdul[0].add_checksum(override_datasum=False) hdul.flush(output_verify='fix') hdul.close() else: msgs.warning( 'The Instrument {} is not supported \nThe file {} will not be processed' .format(instrument, fits_in)) msgs.end() '''
(WAVELMAX + WAVELMIN) * to_ang / 2., 2.) * whitelight_std * n_pixels_psf / (delta_wave_bin * to_ang) hdr0['ABMAGLIM'] = -2.5 * np.log10(five_sigma_nu / 3631.) msgs.warning( 'ABMAGLIM={}. This is most probably not correct at the moment' .format(hdr0['ABMAGLIM'])) # 6. update checksum and datasum msgs.work('Updating checksum and datasum') hdul[0].add_datasum() hdul[1].add_datasum() hdul[0].add_checksum(override_datasum=True) hdul[1].add_checksum(override_datasum=True) image_hdul[0].add_datasum() image_hdul[1].add_datasum() image_hdul[0].add_checksum(override_datasum=True) image_hdul[1].add_checksum(override_datasum=True) """ # CARDS are already present in hdr0 # HDUCLASS', 'HDUCLAS1', 'HDUCLAS2', 'HDUDOC', 'HDUVERS' """ image_hdul.writeto(image_out) hdul.flush() hdul.close() msgs.info('File {} produced.'.format(fits_out)) msgs.info('Image {} produced.'.format(image_out)) msgs.end()
def main(args): from ESOAsg.ancillary import astro from ESOAsg.ancillary import cleaning_lists from ESOAsg.ancillary import polygons from ESOAsg import archive_science_portal from ESOAsg import archive_observations from ESOAsg import msgs # Cleaning input lists input_fits_files = cleaning_lists.make_list_of_fits_files(args.input_fits) instruments = cleaning_lists.from_element_to_list(args.instruments, element_type=str) data_types = cleaning_lists.from_element_to_list(args.data_types, element_type=str) # Cleaning input values confidence_level = cleaning_lists.from_element_to_list( args.confidence_level, element_type=float)[0] maxrec = cleaning_lists.from_element_to_list(args.maxrec, element_type=int)[0] max_vertices = cleaning_lists.from_element_to_list(args.max_vertices, element_type=int)[0] # Cleaning bool show_figure = args.show_sky # Show link show_asp = args.asp_link # Download data download_data = args.download_data msgs.start() for fits_file in input_fits_files: msgs.info(' ') msgs.info('Working on file: {}'.format(fits_file)) msgs.info(' ') contours = astro.contours_from_gw_bayestar( fits_file, credible_level=confidence_level) astro.show_contours_from_gw_bayestar( fits_file, contours=contours, cmap='afmhot', contours_color='white', show_figure=show_figure, matplotlib_backend=STARTING_MATPLOTLIB_BACKEND) polygons_list = polygons.contours_to_polygons( contours, max_vertices=max_vertices) if show_asp: msgs.info('Opening links to ASP') archive_science_portal.query_from_polygons(polygons=polygons_list, open_link=True) results_from_query = archive_observations.query_from_polygons( polygons=polygons_list, maxrec=maxrec, instruments=instruments, data_types=data_types, verbose=False) msgs.info(' ') if download_data: if results_from_query is None: msgs.warning('No data retrieved') elif isinstance(results_from_query, list): for idx_poly, result_from_query in enumerate( results_from_query): msgs.info( 'Downloading data for polygon N.{}'.format(idx_poly + 1)) archive_observations.download(result_from_query['dp_id']) else: msgs.info('Downloading data') archive_observations.download(results_from_query['dp_id']) msgs.info(' ') msgs.end()
change0_comments.append(' ') change0_cards.append('CONTNORM') change0_values.append(False) change0_comments.append(' ') change0_cards.append('TOT_FLUX') change0_values.append(False) change0_comments.append(' ') change0_cards.append('FLUXERR') change0_values.append(-2) change0_comments.append(' ') change0_cards.append('SNR') msgs.info('Calculated SNR={}'.format( np.nanmedian(FLUX[FLUX > 0.] / ERR[FLUX > 0.]))) change0_values.append(np.nanmedian(FLUX[FLUX > 0.] / ERR[FLUX > 0.])) change0_comments.append(' ') change0_cards.append('NCOMBINE') change0_values.append(len(fullHeader['MJD-END'][this_object].data)) change0_comments.append(' ') change0_cards.append('DATE') change0_values.append(fullHeader['DATE'][this_object].data[0]) change0_comments.append(' ') #SPEC_RES change0_cards.append('SPEC_RES') change0_values.append(np.int(3000)) change0_comments.append('Spectral resolution')
def compare_with(self, second_list, check_cards=None, on_terminal=True, on_file=None): r"""Compare two list objects. The code will go through all cards and values present in the input list (or the `check_cards` subset if set not to None) and compare with cards and values in the `second_list`. Args: second_list (`Lists`): List that you want to compare with the input list. check_cards (`np.array`): List of cards that needs to be returned. on_terminal (`bool`): if `True`, the `cards` and respective `values` are printed on the terminal. on_file (`str`, `None`): if not `None`, `cards` and respective `values` will be stored in this text file Returns: first_cards (`np.array`): `cards` present in both the 1st and the 2nd lists. first_values (`np.array`): `values` from the 1st list associated to the `cards` second_values (`np.array`): `values` from the 2nd list associated to the `cards` """ if not isinstance(second_list, Lists): msgs.error('The second list is not a Lists object.') if not isinstance(on_terminal, bool): msgs.error('The on_terminal option should be a bool.') if check_cards is None: check_cards, _ = self.get_cards(check_cards) # loading the first list: _first_cards, _first_values, _first_missing = self.get_values( check_cards=check_cards) msgs.work('Checking for cards in the second_list') # comparing with values from the second list: _second_cards, _second_values, _second_missing = second_list.get_values( check_cards=check_cards) # define where cards are present in both lists _overlap = np.isin(_first_cards, _second_cards) first_cards, first_values, first_missing = self.get_values( check_cards=_first_cards[_overlap]) second_cards, second_values, second_missing = second_list.get_values( check_cards=_first_cards[_overlap]) # Printing results if on_terminal: msgs.info( 'Cards and values from both the first and the second list:') _print_1cards_2values(first_cards, first_values, second_values, on_terminal=on_terminal, on_file=on_file) if on_terminal: # Case of all cards present in the first and in the second lists if np.size(_first_missing) == 0 and np.size(_second_missing) == 0: msgs.info('All cards are present in both lists') # Case of some cards missing in the first list but present in the second: elif np.size(_first_missing) != 0 and np.size( _second_missing) == 0: _only_second = np.isin(check_cards, _first_missing) _only_second_cards, _only_second_values, _only_second_missing = second_list.get_values( check_cards=check_cards[_only_second]) msgs.warning('{} Cards are missing in the first list,'.format( np.size(_first_missing))) msgs.warning('but present in the second:') _print_1cards_1values(_only_second_cards, _only_second_values, on_terminal=on_terminal, on_file=None) # Case of some cards missing in the second list but present in the first: elif np.size( _first_missing) == 0 and np.size(_second_missing) != 0: _only_first = np.isin(check_cards, _second_missing) _only_first_cards, _only_first_values, _only_first_missing = self.get_values( check_cards=check_cards[_only_first]) msgs.warning('{} Cards are missing in the second list,'.format( np.size(_second_missing))) msgs.warning('but present in the first:') _print_1cards_1values(_only_first_cards, _only_first_values, on_terminal=on_terminal, on_file=None) # Case of some cards missing in the first and in the second list: else: _first_missing = ~np.isin(check_cards, _first_cards) _second_missing = ~np.isin(check_cards, _second_cards) _both_missing = _first_missing * _second_missing _first_only_missing = _first_missing * ~_both_missing _second_only_missing = _second_missing * ~_both_missing if np.size(check_cards[_both_missing]) != 0: msgs.warning( '{} Cards are missing from both lists:'.format( np.size(check_cards[_both_missing]))) for both_missing in check_cards[_both_missing]: print(' ----------> ' + both_missing) if np.size(check_cards[_second_only_missing]) != 0: _only_second_cards, _only_second_values, _only_second_missing = self.get_values( check_cards=check_cards[_second_only_missing]) msgs.warning( '{} Cards are missing in the first list,'.format( np.size(check_cards[_second_only_missing]))) msgs.warning('but present in the second:') _print_1cards_1values(_only_second_cards, _only_second_values, on_terminal=on_terminal, on_file=None) if np.size(check_cards[_first_only_missing]) != 0: _only_first_cards, _only_first_values, _only_first_missing = second_list.get_values( check_cards=check_cards[_first_only_missing]) msgs.warning( '{} Cards are missing in the second list,'.format( np.size(check_cards[_first_only_missing]))) msgs.warning('but present in the first:') _print_1cards_1values(_only_first_cards, _only_first_values, on_terminal=on_terminal, on_file=None) return first_cards, first_values, second_values
def query_from_radec(positions, radius=None, instruments=None, data_types=None, verbose=False, maxrec=default.get_value('maxrec')): r"""Query the ESO TAP service given a position in RA and Dec. The `positions` value (or list) needs to be given as an `astropy.coordinates.SkyCoord` object. Args: positions (`astropy.coordinates.SkyCoord`): Coordinates (or list of coordinates) of the sky you want to query in the format of an `astropy.coordinates.SkyCoord` object. For further detail see here: `astropy coordinates <https://docs.astropy.org/en/stable/coordinates/>`_ radius (`float`): Search radius you want to query in arcseconds. Note that in case `None` is given, the query will be performed with the `INTERSECT(POINT('',RA,Dec), s_region)` clause instead of the `INTERSECT(s_region,CIRCLE('',RA,Dec,radius/3600.))` one. See here for further examples: `tap obs examples <http://archive.eso.org/tap_obs/examples>`_ instruments (`list`): Limit the search to the selected list of instruments (e.g., `XSHOOTER`) data_types (`list`): Limit the search to the selected types of data (e.g., `spectrum`) verbose (`bool`): if set to `True` additional info will be displayed maxrec (`int`): Define the maximum number of file that a single query can return from the ESO archive. The default values is set in the `default.txt` file. Returns: results_from_query (`list`): Results from the query in a list with the same length of the input position. Currently it contains: target_name, dp_id, s_ra, s_dec, t_exptime, em_min, em_max, em_min, dataproduct_type, instrument_name, abmaglim, proposal_id, obs_collection """ # Check inputs: # Working on positions if isinstance(positions, list): positions_list = positions else: positions_list = [positions] for position in positions_list: assert isinstance( position, coordinates.SkyCoord), r'Input positions not a SkyCoord object' # Working on radius if radius is not None: if isinstance(radius, int): radius = float(radius) else: assert isinstance(radius, float), r'Input radius is not a number' # Working on instruments if instruments is not None: if isinstance(instruments, list): instruments_list = instruments else: instruments_list = [instruments] for instrument in instruments_list: assert isinstance( instrument, str), r'Input instrument: {} not valid'.format(instrument) # Working on data_types if data_types is not None: if isinstance(data_types, list): data_types_list = data_types else: data_types_list = [data_types] for data_type in data_types_list: assert isinstance( data_type, str), r'Input data type: {} not valid'.format(data_type) # Running over all positions if verbose: how_many_positions = len(positions_list) if how_many_positions > 1: msgs.work( 'Exploring ESO archive around {} locations in the sky'.format( how_many_positions)) else: msgs.work( 'Exploring ESO archive around the input location in the sky') results_from_query = [] for position, idx in zip(positions_list, range(len(positions_list))): position.transform_to(ICRS) ra, dec = np.float_(position.ra.degree), np.float_(position.dec.degree) msgs.work( 'Running query {} to the ESO archive (out of {} total)'.format( idx + 1, len(positions_list))) # Define query # base query: query = _query_obscore_base() # selection of the location: query = query + _query_obscore_intersect_ra_dec(ra, dec, radius=radius) # selection of the instrument(s) if instruments is not None: query = query + _query_obscore_select_instruments(instruments_list) # selection of the data_type(s) if data_types is not None: query = query + _query_obscore_select_data_types(data_types_list) # running query and append results to the list result_from_query = _run_query(query, verbose=verbose, remove_bytes=True, maxrec=maxrec) if len(result_from_query) < 1: msgs.warning('No data has been retrieved') else: msgs.info('A total of {} entries has been retrieved'.format( len(result_from_query))) if verbose: msgs.info('For the following instrument:') for inst_name in np.unique( result_from_query['instrument_name'].data): msgs.info(' - {}'.format(inst_name)) results_from_query.append(result_from_query) return results_from_query
def get_catalogues(collections=None, tables=None, columns=None, type_of_query='sync', all_versions=False, maxrec=None, verbose=False): r"""Query the ESO tap_cat service for specific catalogues There are two ways to select the catalogues you are interested in. Either you select directly the table_name (or the list of table_names) that you want to query, or you select a collection (or a list of collections). If you select this latter option, what happens in the background is that the code is going to search for the table(s) corresponding to the given collection and query them. If you are asking for more than one table, the result will be listed in a list of `astropy.tables` with one element per retrieved table Args: collections (any): list of `str` containing the names (or a single `str`) of the collections for which the query will be limited tables (any): list of `str` containing the table_name of the tables for which the query will be limited columns (any): list of the `column_name` that you want to download. The full list of the columns in a table can be found by running `columns_info()` all_versions (bool): if set to `True` also obsolete versions of the catalogues are searched in case `collections` is given type_of_query (str): type of query to be run maxrec (int, optional): define the maximum number of entries that a single query can return. If it is `None` the value is set by the limit of the service. verbose (bool): if set to `True` additional info will be displayed Returns: any: `astropy.table` or `list` of `astropy.tables` containing the queried catalogues """ # ToDo EMA: conditions to select properties from catalogues should be added in this. Both with ANDs and with ORs # Obtain list of all tables derived from the merger of collections and tables clean_tables = _is_collection_and_table_list_at_eso( collections=collections, tables=tables, all_versions=all_versions) # if maxrec is set to None, the entire length of the catalogue is considered: maxrec_list = _get_catalogue_length_from_tables(clean_tables, maxrec=maxrec, all_versions=all_versions) list_of_catalogues = [] for table_name, maxrec_for_table in zip(clean_tables, maxrec_list): # test for columns columns_in_table = _is_column_list_in_catalogues(columns, tables=table_name) # instantiate ESOCatalogues query_table = query_catalogues.ESOCatalogues( query=tap_queries.create_query_table(table_name, columns=columns_in_table), type_of_query=type_of_query, maxrec=maxrec_for_table) if verbose: query_table.print_query() query_table.run_query(to_string=True) catalogue = query_table.get_result_from_query() list_of_catalogues.append(catalogue) msgs.info( 'The query to {} returned {} entries (with a limit set to maxrec={})' .format(table_name, len(catalogue), maxrec_for_table)) if len(list_of_catalogues) == 0: return None elif len(list_of_catalogues) == 1: return list_of_catalogues[0] else: return list_of_catalogues
# cards input_cards = cleaning_lists.make_list_of_strings(args.cards) # Starting to read the header(s) msgs.start() for fits_file in input_fits_files: for which_hdu in hdu_numbers: hdr = fitsfiles.header_from_fits_file(fits_file, which_hdu=which_hdu) original_files, prov_files, original_cards, prov_cards = [], [], [], [] if 'PROV1' in hdr: for prov_number in hdr['PROV*']: file_id = hdr[prov_number].replace('.fits', '') phase3.get_header_from_archive(file_id, text_file=file_id + '.hdr') hdr_prov = fitsfiles.header_from_txt_file(file_id + '.hdr') for card in input_cards: original_files.append(fits_file) prov_files.append(file_id) original_cards.append(card) prov_cards.append(hdr_prov[card]) msgs.info('Summary:') msgs.info('Input <- Provenance - card -> Value') for original_file, prov_file, original_card, prov_card in zip( original_files, prov_files, original_cards, prov_cards): print('{} <- {} - {} -> {}'.format(original_file, prov_file, original_card, prov_card)) msgs.end()