def query_async(self, get_query_payload=False, cache=True, retry=False, **kwargs): """ Returns ------- response : `~requests.Response` The HTTP response returned from the service. """ request_payload = self._args_to_payload(**kwargs) if get_query_payload: return request_payload response = self._request('POST', self.DATA_URL, params=request_payload, timeout=self.TIMEOUT, cache=cache) self._last_response = response response.raise_for_status() # fail if response is entirely whitespace or if it is empty if not response.content.strip(): if cache: last_pickle = self._last_query.hash() + ".pickle" cache_fn = os.path.join(self.cache_location, last_pickle) os.remove(cache_fn) if retry > 0: log.warning("Query resulted in an empty result. Retrying {0}" " more times.".format(retry)) self.query_async(cache=cache, retry=retry - 1, **kwargs) else: raise ValueError("Query resulted in an empty result but " "the server did not raise an error.") return response
def _download_file(self, url, local_filepath, **kwargs): """ Wraps QueryWithLogin._download_file to detect if the authentication expired. """ trials = 1 while trials <= 2: resp = super(EsoClass, self)._download_file(url, local_filepath, **kwargs) # trying to detect the failing authentication: # - content type should not be html if (resp.headers['Content-Type'] == 'text/html;charset=UTF-8' and resp.url.startswith('https://www.eso.org/sso/login')): if trials == 1: log.warning("Session expired, trying to re-authenticate") self.login() trials += 1 else: raise LoginError("Could not authenticate") else: break return resp
def __init__(self): """ Show a warning message if the API key is not in the configuration file. """ super(AstrometryNetClass, self).__init__() if not conf.api_key: log.warning( "Astrometry.net API key not found in configuration file") log.warning( "You need to manually edit the configuration file and add it") log.warning( "You may also register it for this session with AstrometryNet.key = 'XXXXXXXX'" ) self._session_id = None
def _get_password(self, service_name, username, reenter=False): """Get password from keyring or prompt.""" password_from_keyring = None if reenter is False: try: password_from_keyring = keyring.get_password( service_name, username) except keyring.errors.KeyringError as exc: log.warning("Failed to get a valid keyring for password " "storage: {}".format(exc)) if password_from_keyring is None: log.warning("No password was found in the keychain for the " "provided username.") if system_tools.in_ipynb(): log.warning("You may be using an ipython notebook:" " the password form will appear in your terminal.") password = getpass.getpass( "{0}, enter your password:\n".format(username)) else: password = password_from_keyring return password, password_from_keyring
def get_epic_lightcurve(self, filename, source_number, *, instrument=[], path=""): """Extracts the EPIC sources light curve products from a given TAR file For a given TAR file obtained with ``XMMNewton.download_data``. This function extracts the EPIC sources light curve products in a given instrument (or instruments) from said TAR file The result is a dictionary containing the paths to the extracted EPIC sources light curve products with the key being the instrument If the instrument is not specified, this function will return all available instruments Parameters ---------- filename : string, mandatory The name of the tarfile to be proccessed source_number : integer, mandatory The source number, in decimal, in the observation instruments : array of strings, optional, default [] An array of strings indicating the desired instruments path: string, optional If set, extracts the EPIC images in the indicated path Returns ------- A dictionary with the full paths of the extracted EPIC sources light curve products. The key is the instrument Notes ----- The filenames will contain the source number in hexadecimal, as this is the convention used by the pipeline. The structure and the content of the extracted compressed FITS files are described in details in the Pipeline Products Description [XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf). """ _instrumnet = ["M1", "M2", "PN", "EP"] _band = [8] _product_type = ["SRCTSR", "FBKTSR"] _path = "" ret = None if instrument == []: instrument = _instrumnet else: for inst in instrument: if inst not in _instrumnet: log.warning("Invalid instrument %s" % inst) instrument.remove(inst) if path != "" and os.path.exists(path): _path = path try: with tarfile.open(filename, "r") as tar: ret = {} for member in tar.getmembers(): paths = os.path.split(member.name) fname = paths[1] paths = os.path.split(paths[0]) if paths[1] != "pps": continue fname_info = self._parse_filename(fname) if fname_info["X"] != "P": continue if not fname_info["I"] in instrument: continue if not int(fname_info["S"]) in _band: continue if not fname_info["T"] in _product_type: continue if int(fname_info["X-"], 16) != source_number: continue tar.extract(member, _path) key = fname_info["I"] path_inst_name = os.path.abspath(os.path.join(_path, member.name)) if fname_info["T"] == "FBKTSR": key = fname_info["I"] + "_bkg" if ret.get(key) and type(ret.get(key)) == str: log.warning("More than one file found with the " "instrument: %s" % key) ret[key] = [ret[key], path_inst_name] elif ret.get(key) and type(ret.get(key)) == list: ret[key].append(path_inst_name) else: ret[key] = path_inst_name except FileNotFoundError: log.error("File %s not found" % (filename)) return None if ret is None or ret == {}: log.info("Nothing to extract with the given parameters:\n" " PPS: %s\n" " Source Number: %u\n" " Instrument: %s\n" % (filename, source_number, instrument)) return ret
def get_epic_images(self, filename, band=[], instrument=[], get_detmask=False, get_exposure_map=False, path="", **kwargs): """Extracts the EPIC images from a given TAR file This function extracts the EPIC images in a given band (or bands) and instrument (or instruments) from it The result is a dictionary containing the paths to the extracted EPIC images with keys being the band and the instrument If the band or the instrument are not specified this function will return all the available bands and instruments Additionally, ``get_detmask`` and ``get_exposure_map`` can be set to True. If so, this function will also extract the exposure maps and detector masks within the specified bands and instruments Parameters ---------- filename : string, mandatory The name of the tarfile to be proccessed band : array of integers, optional, default [] An array of intergers indicating the desired bands instruments : array of strings, optional, default [] An array of strings indicating the desired instruments get_detmask : bool, optional If True, also extracts the detector masks get_exposure_map : bool, optional If True, also extracts the exposure maps path: string, optional If set, extracts the EPIC images in the indicated path Returns ------- A dictionary of dictionaries with the full paths of the extracted EPIC images. The keys of each dictionary are the band for the first level dictionary and the instrument for the second level dictionaries Notes ----- The structure and the content of the extracted compressed FITS files are described in details in the Pipeline Products Description [XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf). """ _product_type = ["IMAGE_"] _instrument = ["M1", "M2", "PN", "EP"] _band = [1, 2, 3, 4, 5, 8] _path = "" if get_detmask: _product_type.append("DETMSK") if get_exposure_map: _product_type.append("EXPMAP") if path != "" and os.path.exists(path): _path = path ret = None if band == []: band = _band else: for b in band: if b not in _band: log.warning("Invalid band %u" % b) band.remove(b) if instrument == []: instrument = _instrument else: for inst in instrument: if inst not in _instrument: log.warning("Invalid instrument %s" % inst) instrument.remove(inst) try: with tarfile.open(filename, "r") as tar: ret = {} for member in tar.getmembers(): paths = os.path.split(member.name) fname = paths[1] paths = os.path.split(paths[0]) if paths[1] != "pps": continue fname_info = self._parse_filename(fname) if fname_info["X"] != "P": continue if not fname_info["I"] in instrument: continue if not int(fname_info["S"]) in band: continue if not fname_info["T"] in _product_type: continue tar.extract(member, _path) if not ret.get(int(fname_info["S"])): ret[int(fname_info["S"])] = {} b = int(fname_info["S"]) ins = fname_info["I"] path_member_name = os.path.abspath(os.path.join(_path, member.name)) if fname_info["T"] == "DETMSK": ins = fname_info["I"] + "_det" elif fname_info["T"] == "EXPMAP": ins = fname_info["I"] + "_expo" if ret[b].get(ins) and type(ret[b].get(ins)) == str: log.warning("More than one file found with the " "band %u and " "the instrument: %s" % (b, ins)) ret[b][ins] = [ret[b][ins], path_member_name] elif ret[b].get(ins) and type(ret[b].get(ins)) == list: ret[b][ins].append(path_member_name) else: ret[b][ins] = path_member_name except FileNotFoundError: log.error("File %s not found" % (filename)) return None return ret
def get_epic_spectra(self, filename, source_number, *, instrument=[], path="", verbose=False): """Extracts in path (when set) the EPIC sources spectral products from a given TAR file. This function extracts the EPIC sources spectral products in a given instrument (or instruments) from it The result is a dictionary containing the paths to the extracted EPIC sources spectral products with key being the instrument If the instrument is not specified this function will return all the available instruments Parameters ---------- filename : string, mandatory The name of the tarfile to be processed source_number : integer, mandatory The source number, in decimal, in the observation instruments : array of strings, optional, default [] An array of strings indicating the desired instruments path: string, optional If set, extracts the EPIC images in the indicated path verbose : bool optional, default 'False' flag to display information about the process Returns ------- A dictionary with the full paths of the extracted EPIC sources spectral products. The key is the instrument Notes ----- The filenames will contain the source number in hexadecimal, as this is the convention used by the pipeline. The structure and the content of the extracted compressed FITS files are described in details in the Pipeline Products Description [XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf). """ _instrument = ["M1", "M2", "PN", "EP"] _product_type = ["SRSPEC", "BGSPEC", "SRCARF"] _path = "" ret = None if instrument == []: instrument = _instrument else: for inst in instrument: if inst not in _instrument: log.warning(f"Invalid instrument {inst}") instrument.remove(inst) if path != "" and os.path.exists(path): _path = path try: with tarfile.open(filename, "r") as tar: ret = {} for member in tar.getmembers(): paths = os.path.split(member.name) fname = paths[1] paths = os.path.split(paths[0]) if paths[1] != "pps": continue fname_info = self._parse_filename(fname) if fname_info["X"] != "P": continue if not fname_info["I"] in instrument: continue if not fname_info["T"] in _product_type: continue if int(fname_info["X-"], 16) != source_number: continue tar.extract(member, _path) key = fname_info["I"] path_inst_name = os.path.abspath(os.path.join(_path, member.name)) if fname_info["T"] == "BGSPEC": key = fname_info["I"] + "_bkg" elif fname_info["T"] == "SRCARF": key = fname_info["I"] + "_arf" else: # process the source spectrum with fits.open(path_inst_name) as hdul: # pick up the SAS version, needed for the pn RMF sasver = hdul[0].header["SASVERS"].split("-")[1][:-2] # # build a list with old RMF folders, going back two versions, last bulk in 18.0 # hardcoded_old_pn_rmfs = ['2020-10-28_sas19.0.0','2019-07-31_sas18.0.0'] old_rmf = False for irmf in hardcoded_old_pn_rmfs: if (sasver in irmf): rmf_path = irmf #if (verbose): # print (f'Info: picking up an old RMF file from {rmf_path} folder') old_rmf = True break # for ext in hdul: if ext.name != "SPECTRUM": continue rmf_fname = ext.header["RESPFILE"] if fname_info["I"] == "M1" or fname_info["I"] == "M2": inst = "MOS/" + str(ext.header["SPECDELT"]) + "eV/" elif fname_info["I"] == "PN": inst = "PN/" file_name, file_ext = os.path.splitext(rmf_fname) rmf_fname = file_name + "_v" + sasver + file_ext if (old_rmf): inst = f"old/pn/{rmf_path}/" link = self._rmf_ftp + inst + rmf_fname if verbose: log.info("rmf link is: %s" % link) response = self._request('GET', link) response.raise_for_status() rsp_filename = os.path.join(_path, paths[0], paths[1], ext.header["RESPFILE"]) with open(rsp_filename, 'wb') as f: f.write(response.content) ret[fname_info["I"] + "_rmf"] = rsp_filename if ret.get(key) and type(ret.get(key)) == str: log.warning("More than one file found with the instrument: %s" % key) ret[key] = [ret[key], path_inst_name] elif ret.get(key) and type(ret.get(key)) == list: ret[key].append(path_inst_name) else: ret[key] = path_inst_name except FileNotFoundError: log.error("File %s not found" % (filename)) return if not ret: log.info("Nothing to extract with the given parameters:\n" " PPS: %s\n" " Source Number: %u\n" " Instrument: %s\n" % (filename, source_number, instrument)) return ret
def download_file(self, data_product, local_path, cache=True): """ Takes a data product in the form of an `~astropy.table.Row` and downloads it from the cloud into the given directory. Parameters ---------- data_product : `~astropy.table.Row` Product to download. local_path : str The local filename to which toe downloaded file will be saved. cache : bool Default is True. If file is found on disc it will not be downloaded again. """ s3 = self.boto3.resource('s3', config=self.config) s3_client = self.boto3.client('s3', config=self.config) bkt = s3.Bucket(self.pubdata_bucket) with warnings.catch_warnings(): warnings.simplefilter("ignore") bucket_path = self.get_cloud_uri(data_product, False) if not bucket_path: raise Exception("Unable to locate file {}.".format( data_product['productFilename'])) # Ask the webserver (in this case S3) what the expected content length is and use that. info_lookup = s3_client.head_object(Bucket=self.pubdata_bucket, Key=bucket_path) length = info_lookup["ContentLength"] if cache and os.path.exists(local_path): if length is not None: statinfo = os.stat(local_path) if statinfo.st_size != length: log.warning("Found cached file {0} with size {1} that is " "different from expected size {2}".format( local_path, statinfo.st_size, length)) else: log.info( "Found cached file {0} with expected size {1}.".format( local_path, statinfo.st_size)) return with ProgressBarOrSpinner( length, ('Downloading URL s3://{0}/{1} to {2} ...'.format( self.pubdata_bucket, bucket_path, local_path))) as pb: # Bytes read tracks how much data has been received so far # This variable will be updated in multiple threads below global bytes_read bytes_read = 0 progress_lock = threading.Lock() def progress_callback(numbytes): # Boto3 calls this from multiple threads pulling the data from S3 global bytes_read # This callback can be called in multiple threads # Access to updating the console needs to be locked with progress_lock: bytes_read += numbytes pb.update(bytes_read) bkt.download_file(bucket_path, local_path, Callback=progress_callback)
def get_data_info(self, uids, *, expand_tarfiles=False, with_auxiliary=True, with_rawdata=True): """ Return information about the data associated with ALMA uid(s) Parameters ---------- uids : list or str A list of valid UIDs or a single UID. UIDs should have the form: 'uid://A002/X391d0b/X7b' expand_tarfiles : bool False to return information on the tarfiles packages containing the data or True to return information about individual files in these packages with_auxiliary : bool True to include the auxiliary packages, False otherwise with_rawdata : bool True to include raw data, False otherwise Returns ------- Table with results or None. Table has the following columns: id (UID), access_url (URL to access data), content_length, content_type (MIME type), semantics, description (optional), error_message (optional) """ if uids is None: raise AttributeError('UIDs required') if isinstance(uids, (str, bytes)): uids = [uids] if not isinstance(uids, (list, tuple, np.ndarray)): raise TypeError("Datasets must be given as a list of strings.") # TODO remove this loop and send uids at once when pyvo fixed result = None for uid in uids: res = self.datalink.run_sync(uid) if res.status[0] != 'OK': raise Exception('ERROR {}: {}'.format(res.status[0], res.status[1])) temp = res.to_table() if ASTROPY_LT_4_1: # very annoying for col in [x for x in temp.colnames if x not in ['content_length', 'readable']]: temp[col] = temp[col].astype(str) result = temp if result is None else vstack([result, temp]) to_delete = [] for index, rr in enumerate(result): if rr['error_message'] is not None and \ rr['error_message'].strip(): log.warning('Error accessing info about file {}: {}'. format(rr['access_url'], rr['error_message'])) # delete from results. Good thing to do? to_delete.append(index) result.remove_rows(to_delete) if not with_auxiliary: result = result[np.core.defchararray.find( result['semantics'], '#aux') == -1] if not with_rawdata: result = result[np.core.defchararray.find( result['semantics'], '#progenitor') == -1] # primary data delivery type is files packaged in tarballs. However # some type of data has an alternative way to retrieve each individual # file as an alternative (semantics='#datalink' and # 'content_type=application/x-votable+xml;content=datalink'). They also # require an extra call to the datalink service to get the list of # files. DATALINK_FILE_TYPE = 'application/x-votable+xml;content=datalink' DATALINK_SEMANTICS = '#datalink' if expand_tarfiles: # identify the tarballs that can be expandable and replace them # with the list of components expanded_result = None to_delete = [] for index, row in enumerate(result): if DATALINK_SEMANTICS in row['semantics'] and \ row['content_type'] == DATALINK_FILE_TYPE: # subsequent call to datalink file_id = row['access_url'].split('ID=')[1] expanded_tar = self.get_data_info(file_id) expanded_tar = expanded_tar[ expanded_tar['semantics'] != '#cutout'] if not expanded_result: expanded_result = expanded_tar else: expanded_result = vstack( [expanded_result, expanded_tar], join_type='exact') to_delete.append(index) # cleanup result.remove_rows(to_delete) # add the extra rows if expanded_result: result = vstack([result, expanded_result], join_type='exact') else: result = result[np.logical_or(np.core.defchararray.find( result['semantics'].astype(str), DATALINK_SEMANTICS) == -1, result['content_type'].astype(str) != DATALINK_FILE_TYPE)] return result
def _download_file(self, url, local_filepath, timeout=None, auth=None, continuation=True, cache=False, method="GET", head_safe=False, **kwargs): """ Download a file. Resembles `astropy.utils.data.download_file` but uses the local ``_session`` Parameters ---------- url : string local_filepath : string timeout : int auth : dict or None continuation : bool If the file has already been partially downloaded *and* the server supports HTTP "range" requests, the download will be continued where it left off. cache : bool method : "GET" or "POST" head_safe : bool """ if head_safe: response = self._session.request("HEAD", url, timeout=timeout, stream=True, auth=auth, **kwargs) else: response = self._session.request(method, url, timeout=timeout, stream=True, auth=auth, **kwargs) response.raise_for_status() if 'content-length' in response.headers: length = int(response.headers['content-length']) if length == 0: log.warn('URL {0} has length=0'.format(url)) else: length = None if ((os.path.exists(local_filepath) and ('Accept-Ranges' in response.headers) and continuation)): open_mode = 'ab' existing_file_length = os.stat(local_filepath).st_size if length is not None and existing_file_length >= length: # all done! log.info( "Found cached file {0} with expected size {1}.".format( local_filepath, existing_file_length)) return elif existing_file_length == 0: open_mode = 'wb' else: log.info("Continuing download of file {0}, with {1} bytes to " "go ({2}%)".format( local_filepath, length - existing_file_length, (length - existing_file_length) / length * 100)) # bytes are indexed from 0: # https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#range-request-header end = "{0}".format(length - 1) if length is not None else "" self._session.headers['Range'] = "bytes={0}-{1}".format( existing_file_length, end) response = self._session.request(method, url, timeout=timeout, stream=True, auth=auth, **kwargs) response.raise_for_status() del self._session.headers['Range'] elif cache and os.path.exists(local_filepath): if length is not None: statinfo = os.stat(local_filepath) if statinfo.st_size != length: log.warning("Found cached file {0} with size {1} that is " "different from expected size {2}".format( local_filepath, statinfo.st_size, length)) open_mode = 'wb' else: log.info( "Found cached file {0} with expected size {1}.".format( local_filepath, statinfo.st_size)) response.close() return else: log.info("Found cached file {0}.".format(local_filepath)) response.close() return else: open_mode = 'wb' if head_safe: response = self._session.request(method, url, timeout=timeout, stream=True, auth=auth, **kwargs) response.raise_for_status() blocksize = astropy.utils.data.conf.download_block_size log.debug( f"Downloading URL {url} to {local_filepath} with size {length} " f"by blocks of {blocksize}") bytes_read = 0 # Only show progress bar if logging level is INFO or lower. if log.getEffectiveLevel() <= 20: progress_stream = None # Astropy default else: progress_stream = io.StringIO() with ProgressBarOrSpinner( length, ('Downloading URL {0} to {1} ...'.format(url, local_filepath)), file=progress_stream) as pb: with open(local_filepath, open_mode) as f: for block in response.iter_content(blocksize): f.write(block) bytes_read += blocksize if length is not None: pb.update( bytes_read if bytes_read <= length else length) else: pb.update(bytes_read) response.close() return response
def get_epic_spectra(self, filename, source_number, *, instrument=[], path="", verbose=False): """Extracts in path (when set) the EPIC sources spectral products from a given TAR file. For a given TAR file obtained with: XMM.download_data(OBS_ID,level="PPS",extension="FTZ",filename=tarfile) This function extracts the EPIC sources spectral products in a given instrument (or instruments) from it The result is a dictionary containing the paths to the extracted EPIC sources spectral products with key being the instrument If the instrument is not specified this function will return all the available instruments Examples: Extracting all bands and instruments: result = XMM.get_epic_spectra(tarfile,83, instrument=['M1','M2','PN']) If we want to retrieve the source spectrum of the instrument PN fits_image = result['PN'] fits_image will be the full path to the extracted FTZ file Parameters ---------- filename : string, mandatory The name of the tarfile to be processed source_number : integer, mandatory The source number, in decimal, in the observation instruments : array of strings, optional, default [] An array of strings indicating the desired instruments path: string, optional If set, extracts the EPIC images in the indicated path verbose : bool optional, default 'False' flag to display information about the process Returns ------- A dictionary with the full paths of the extracted EPIC sources spectral products. The key is the instrument Notes ----- The filenames will contain the source number in hexadecimal, as this is the convention used by the pipeline. The structure and the content of the extracted compressed FITS files are described in details in the Pipeline Products Description [XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf). """ _instrument = ["M1", "M2", "PN", "EP"] _product_type = ["SRSPEC", "BGSPEC", "SRCARF"] _path = "" ret = None if instrument == []: instrument = _instrument else: for inst in instrument: if inst not in _instrument: log.warning(f"Invalid instrument {inst}") instrument.remove(inst) if path != "" and os.path.exists(path): _path = path try: with tarfile.open(filename, "r") as tar: ret = {} for member in tar.getmembers(): paths = os.path.split(member.name) fname = paths[1] paths = os.path.split(paths[0]) if paths[1] != "pps": continue fname_info = self._parse_filename(fname) if fname_info["X"] != "P": continue if not fname_info["I"] in instrument: continue if not fname_info["T"] in _product_type: continue if int(fname_info["X-"], 16) != source_number: continue tar.extract(member, _path) key = fname_info["I"] path_inst_name = os.path.abspath(os.path.join(_path, member.name)) if fname_info["T"] == "BGSPEC": key = fname_info["I"] + "_bkg" elif fname_info["T"] == "SRCARF": key = fname_info["I"] + "_arf" else: with fits.open(path_inst_name) as hdul: for ext in hdul: if ext.name != "SPECTRUM": continue rmf_fname = ext.header["RESPFILE"] if fname_info["I"] == "M1" or fname_info["I"] == "M2": inst = "MOS/" + str(ext.header["SPECDELT"]) + "eV/" elif fname_info["I"] == "PN": inst = "PN/" file_name, file_ext = os.path.splitext(rmf_fname) rmf_fname = file_name + "_v18.0" + file_ext link = self._rmf_ftp + inst + rmf_fname if verbose: log.info("rmf link is: %s" % link) response = self._request('GET', link) rsp_filename = os.path.join(_path, paths[0], paths[1], ext.header["RESPFILE"]) with open(rsp_filename, 'wb') as f: f.write(response.content) ret[fname_info["I"] + "_rmf"] = rsp_filename if ret.get(key) and type(ret.get(key)) == str: log.warning("More than one file found with the instrument: %s" % key) ret[key] = [ret[key], path_inst_name] elif ret.get(key) and type(ret.get(key)) == list: ret[key].append(path_inst_name) else: ret[key] = path_inst_name except FileNotFoundError: log.error("File %s not found" % (filename)) return if not ret: log.info("Nothing to extract with the given parameters:\n" " PPS: %s\n" " Source Number: %u\n" " Instrument: %s\n" % (filename, source_number, instrument)) return ret
def get_epic_images(self, filename, *, band=[], instrument=[], get_detmask=False, get_exposure_map=False, path=""): """Extracts the European Photon Imaging Camera (EPIC) images from a given TAR file For a given TAR file obtained with: XMM.download_data(OBS_ID,level="PPS",extension="FTZ",filename=tarfile) This function extracts the EPIC images in a given band (or bands) and instrument (or instruments) from it The result is a dictionary containing the paths to the extracted EPIC images with keys being the band and the instrument If the band or the instrument are not specified this function will return all the available bands and instruments Additionally, ``get_detmask`` and ``get_exposure_map`` can be set to True. If so, this function will also extract the exposure maps and detector masks within the specified bands and instruments Examples -------- Extract all bands and instruments:: result = XMM.get_epic_images(tarfile,band=[1,2,3,4,5,8], instrument=['M1','M2','PN'],**kwargs) If we want to retrieve the band 3 for the instrument PN (p-n junction):: fits_image = result[3]['PN'] ``fits_image`` will be the full path to the extracted FTZ file Extract the exposure and detector maps:: result = XMM.get_epic_images(tarfile,band=[1,2,3,4,5,8], instrument=['M1','M2','PN'], get_detmask=True, get_exposure_map=True) If we want to retrieve exposure map in the band 3 for the instrument PN:: fits_image = result[3]['PN_expo'] For retrieving the detector mask in the band 3 for the instrument PN:: fits_image = result[3]['PN_det'] Parameters ---------- filename : string, mandatory The name of the tarfile to be proccessed band : array of integers, optional, default [] An array of intergers indicating the desired bands instruments : array of strings, optional, default [] An array of strings indicating the desired instruments get_detmask : bool, optional If True, also extracts the detector masks get_exposure_map : bool, optional If True, also extracts the exposure maps path: string, optional If set, extracts the EPIC images in the indicated path Returns ------- A dictionary of dictionaries with the full paths of the extracted EPIC images. The keys of each dictionary are the band for the first level dictionary and the instrument for the second level dictionaries Notes ----- The structure and the content of the extracted compressed FITS files are described in details in the Pipeline Products Description [XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf). """ _product_type = ["IMAGE_"] _instrument = ["M1", "M2", "PN", "EP"] _band = [1, 2, 3, 4, 5, 8] _path = "" if get_detmask: _product_type.append("DETMSK") if get_exposure_map: _product_type.append("EXPMAP") if path != "" and os.path.exists(path): _path = path ret = {} if band == []: band = _band else: for i in band: if i not in _band: log.warning("Invalid band %u" % i) band.remove(i) if instrument == []: instrument = _instrument else: for i in instrument: if i not in _instrument: log.warning("Invalid instrument %s" % i) instrument.remove(i) with tarfile.open(filename, "r") as tar: for member in tar.getmembers(): paths = os.path.split(member.name) fname = paths[1] paths = os.path.split(paths[0]) if paths[1] != "pps": continue fname_info = self._parse_filename(fname) if fname_info["X"] != "P": continue if not fname_info["I"] in instrument: continue if not int(fname_info["S"]) in band: continue if not fname_info["T"] in _product_type: continue tar.extract(member, _path) if not ret.get(int(fname_info["S"])): ret[int(fname_info["S"])] = {} b = int(fname_info["S"]) ins = fname_info["I"] value = os.path.abspath(os.path.join(_path, member.name)) if fname_info["T"] == "DETMSK": ins = fname_info["I"] + "_det" elif fname_info["T"] == "EXPMAP": ins = fname_info["I"] + "_expo" if ret[b].get(ins) and type(ret[b].get(ins)) == str: log.warning("More than one file found with the " "band %u and " "the instrument: %s" % (b, ins)) ret[b][ins] = [ret[b][ins], value] elif ret[b].get(ins) and type(ret[b].get(ins)) == list: ret[b][ins].append(value) else: ret[b][ins] = value return ret