def query_async(self,
get_query_payload=False,
cache=True,
retry=False,
**kwargs):
"""
Returns
-------
response : `~requests.Response`
The HTTP response returned from the service.
"""
request_payload = self._args_to_payload(**kwargs)
if get_query_payload:
return request_payload
response = self._request('POST',
self.DATA_URL,
params=request_payload,
timeout=self.TIMEOUT,
cache=cache)
self._last_response = response
response.raise_for_status()
# fail if response is entirely whitespace or if it is empty
if not response.content.strip():
if cache:
last_pickle = self._last_query.hash() + ".pickle"
cache_fn = os.path.join(self.cache_location, last_pickle)
os.remove(cache_fn)
if retry > 0:
log.warning("Query resulted in an empty result. Retrying {0}"
" more times.".format(retry))
self.query_async(cache=cache, retry=retry - 1, **kwargs)
else:
raise ValueError("Query resulted in an empty result but "
"the server did not raise an error.")
return response
def _download_file(self, url, local_filepath, **kwargs):
""" Wraps QueryWithLogin._download_file to detect if the
authentication expired.
"""
trials = 1
while trials <= 2:
resp = super(EsoClass, self)._download_file(url, local_filepath,
**kwargs)
# trying to detect the failing authentication:
# - content type should not be html
if (resp.headers['Content-Type'] == 'text/html;charset=UTF-8' and
resp.url.startswith('https://www.eso.org/sso/login')):
if trials == 1:
log.warning("Session expired, trying to re-authenticate")
self.login()
trials += 1
else:
raise LoginError("Could not authenticate")
else:
break
return resp
def __init__(self):
""" Show a warning message if the API key is not in the configuration file. """
super(AstrometryNetClass, self).__init__()
if not conf.api_key:
log.warning(
"Astrometry.net API key not found in configuration file")
log.warning(
"You need to manually edit the configuration file and add it")
log.warning(
"You may also register it for this session with AstrometryNet.key = 'XXXXXXXX'"
)
self._session_id = None
def _get_password(self, service_name, username, reenter=False):
"""Get password from keyring or prompt."""
password_from_keyring = None
if reenter is False:
try:
password_from_keyring = keyring.get_password(
service_name, username)
except keyring.errors.KeyringError as exc:
log.warning("Failed to get a valid keyring for password "
"storage: {}".format(exc))
if password_from_keyring is None:
log.warning("No password was found in the keychain for the "
"provided username.")
if system_tools.in_ipynb():
log.warning("You may be using an ipython notebook:"
" the password form will appear in your terminal.")
password = getpass.getpass(
"{0}, enter your password:\n".format(username))
else:
password = password_from_keyring
return password, password_from_keyring
def get_epic_lightcurve(self, filename, source_number, *,
instrument=[], path=""):
"""Extracts the EPIC sources light curve products from a given TAR file
For a given TAR file obtained with ``XMMNewton.download_data``.
This function extracts the EPIC sources light curve products in a given
instrument (or instruments) from said TAR file
The result is a dictionary containing the paths to the extracted EPIC
sources light curve products with the key being the instrument
If the instrument is not specified, this function will
return all available instruments
Parameters
----------
filename : string, mandatory
The name of the tarfile to be proccessed
source_number : integer, mandatory
The source number, in decimal, in the observation
instruments : array of strings, optional, default []
An array of strings indicating the desired instruments
path: string, optional
If set, extracts the EPIC images in the indicated path
Returns
-------
A dictionary with the full paths of the extracted EPIC sources
light curve products. The key is the instrument
Notes
-----
The filenames will contain the source number in hexadecimal,
as this is the convention used by the pipeline.
The structure and the content of the extracted compressed FITS files
are described in details in the Pipeline Products Description
[XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf).
"""
_instrumnet = ["M1", "M2", "PN", "EP"]
_band = [8]
_product_type = ["SRCTSR", "FBKTSR"]
_path = ""
ret = None
if instrument == []:
instrument = _instrumnet
else:
for inst in instrument:
if inst not in _instrumnet:
log.warning("Invalid instrument %s" % inst)
instrument.remove(inst)
if path != "" and os.path.exists(path):
_path = path
try:
with tarfile.open(filename, "r") as tar:
ret = {}
for member in tar.getmembers():
paths = os.path.split(member.name)
fname = paths[1]
paths = os.path.split(paths[0])
if paths[1] != "pps":
continue
fname_info = self._parse_filename(fname)
if fname_info["X"] != "P":
continue
if not fname_info["I"] in instrument:
continue
if not int(fname_info["S"]) in _band:
continue
if not fname_info["T"] in _product_type:
continue
if int(fname_info["X-"], 16) != source_number:
continue
tar.extract(member, _path)
key = fname_info["I"]
path_inst_name = os.path.abspath(os.path.join(_path, member.name))
if fname_info["T"] == "FBKTSR":
key = fname_info["I"] + "_bkg"
if ret.get(key) and type(ret.get(key)) == str:
log.warning("More than one file found with the "
"instrument: %s" % key)
ret[key] = [ret[key], path_inst_name]
elif ret.get(key) and type(ret.get(key)) == list:
ret[key].append(path_inst_name)
else:
ret[key] = path_inst_name
except FileNotFoundError:
log.error("File %s not found" % (filename))
return None
if ret is None or ret == {}:
log.info("Nothing to extract with the given parameters:\n"
" PPS: %s\n"
" Source Number: %u\n"
" Instrument: %s\n" % (filename, source_number,
instrument))
return ret
def get_epic_images(self, filename, band=[], instrument=[],
get_detmask=False, get_exposure_map=False, path="", **kwargs):
"""Extracts the EPIC images from a given TAR file
This function extracts the EPIC images in a given band (or bands) and
instrument (or instruments) from it
The result is a dictionary containing the paths to the extracted EPIC
images with keys being the band and the instrument
If the band or the instrument are not specified this function will
return all the available bands and instruments
Additionally, ``get_detmask`` and ``get_exposure_map`` can be set to True.
If so, this function will also extract the exposure maps and detector
masks within the specified bands and instruments
Parameters
----------
filename : string, mandatory
The name of the tarfile to be proccessed
band : array of integers, optional, default []
An array of intergers indicating the desired bands
instruments : array of strings, optional, default []
An array of strings indicating the desired instruments
get_detmask : bool, optional
If True, also extracts the detector masks
get_exposure_map : bool, optional
If True, also extracts the exposure maps
path: string, optional
If set, extracts the EPIC images in the indicated path
Returns
-------
A dictionary of dictionaries with the full paths of the extracted
EPIC images. The keys of each dictionary are the band for the first
level dictionary and the instrument for the second level dictionaries
Notes
-----
The structure and the content of the extracted compressed FITS files
are described in details in the Pipeline Products Description
[XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf).
"""
_product_type = ["IMAGE_"]
_instrument = ["M1", "M2", "PN", "EP"]
_band = [1, 2, 3, 4, 5, 8]
_path = ""
if get_detmask:
_product_type.append("DETMSK")
if get_exposure_map:
_product_type.append("EXPMAP")
if path != "" and os.path.exists(path):
_path = path
ret = None
if band == []:
band = _band
else:
for b in band:
if b not in _band:
log.warning("Invalid band %u" % b)
band.remove(b)
if instrument == []:
instrument = _instrument
else:
for inst in instrument:
if inst not in _instrument:
log.warning("Invalid instrument %s" % inst)
instrument.remove(inst)
try:
with tarfile.open(filename, "r") as tar:
ret = {}
for member in tar.getmembers():
paths = os.path.split(member.name)
fname = paths[1]
paths = os.path.split(paths[0])
if paths[1] != "pps":
continue
fname_info = self._parse_filename(fname)
if fname_info["X"] != "P":
continue
if not fname_info["I"] in instrument:
continue
if not int(fname_info["S"]) in band:
continue
if not fname_info["T"] in _product_type:
continue
tar.extract(member, _path)
if not ret.get(int(fname_info["S"])):
ret[int(fname_info["S"])] = {}
b = int(fname_info["S"])
ins = fname_info["I"]
path_member_name = os.path.abspath(os.path.join(_path, member.name))
if fname_info["T"] == "DETMSK":
ins = fname_info["I"] + "_det"
elif fname_info["T"] == "EXPMAP":
ins = fname_info["I"] + "_expo"
if ret[b].get(ins) and type(ret[b].get(ins)) == str:
log.warning("More than one file found with the "
"band %u and "
"the instrument: %s" % (b, ins))
ret[b][ins] = [ret[b][ins], path_member_name]
elif ret[b].get(ins) and type(ret[b].get(ins)) == list:
ret[b][ins].append(path_member_name)
else:
ret[b][ins] = path_member_name
except FileNotFoundError:
log.error("File %s not found" % (filename))
return None
return ret
def get_epic_spectra(self, filename, source_number, *,
instrument=[], path="", verbose=False):
"""Extracts in path (when set) the EPIC sources spectral products from a
given TAR file.
This function extracts the EPIC sources spectral products in a given
instrument (or instruments) from it
The result is a dictionary containing the paths to the extracted EPIC
sources spectral products with key being the instrument
If the instrument is not specified this function will
return all the available instruments
Parameters
----------
filename : string, mandatory
The name of the tarfile to be processed
source_number : integer, mandatory
The source number, in decimal, in the observation
instruments : array of strings, optional, default []
An array of strings indicating the desired instruments
path: string, optional
If set, extracts the EPIC images in the indicated path
verbose : bool
optional, default 'False'
flag to display information about the process
Returns
-------
A dictionary with the full paths of the extracted EPIC sources
spectral products. The key is the instrument
Notes
-----
The filenames will contain the source number in hexadecimal,
as this is the convention used by the pipeline.
The structure and the content of the extracted compressed FITS files
are described in details in the Pipeline Products Description
[XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf).
"""
_instrument = ["M1", "M2", "PN", "EP"]
_product_type = ["SRSPEC", "BGSPEC", "SRCARF"]
_path = ""
ret = None
if instrument == []:
instrument = _instrument
else:
for inst in instrument:
if inst not in _instrument:
log.warning(f"Invalid instrument {inst}")
instrument.remove(inst)
if path != "" and os.path.exists(path):
_path = path
try:
with tarfile.open(filename, "r") as tar:
ret = {}
for member in tar.getmembers():
paths = os.path.split(member.name)
fname = paths[1]
paths = os.path.split(paths[0])
if paths[1] != "pps":
continue
fname_info = self._parse_filename(fname)
if fname_info["X"] != "P":
continue
if not fname_info["I"] in instrument:
continue
if not fname_info["T"] in _product_type:
continue
if int(fname_info["X-"], 16) != source_number:
continue
tar.extract(member, _path)
key = fname_info["I"]
path_inst_name = os.path.abspath(os.path.join(_path, member.name))
if fname_info["T"] == "BGSPEC":
key = fname_info["I"] + "_bkg"
elif fname_info["T"] == "SRCARF":
key = fname_info["I"] + "_arf"
else:
# process the source spectrum
with fits.open(path_inst_name) as hdul:
# pick up the SAS version, needed for the pn RMF
sasver = hdul[0].header["SASVERS"].split("-")[1][:-2]
#
# build a list with old RMF folders, going back two versions, last bulk in 18.0
#
hardcoded_old_pn_rmfs = ['2020-10-28_sas19.0.0','2019-07-31_sas18.0.0']
old_rmf = False
for irmf in hardcoded_old_pn_rmfs:
if (sasver in irmf):
rmf_path = irmf
#if (verbose):
# print (f'Info: picking up an old RMF file from {rmf_path} folder')
old_rmf = True
break
#
for ext in hdul:
if ext.name != "SPECTRUM":
continue
rmf_fname = ext.header["RESPFILE"]
if fname_info["I"] == "M1" or fname_info["I"] == "M2":
inst = "MOS/" + str(ext.header["SPECDELT"]) + "eV/"
elif fname_info["I"] == "PN":
inst = "PN/"
file_name, file_ext = os.path.splitext(rmf_fname)
rmf_fname = file_name + "_v" + sasver + file_ext
if (old_rmf):
inst = f"old/pn/{rmf_path}/"
link = self._rmf_ftp + inst + rmf_fname
if verbose:
log.info("rmf link is: %s" % link)
response = self._request('GET', link)
response.raise_for_status()
rsp_filename = os.path.join(_path, paths[0], paths[1], ext.header["RESPFILE"])
with open(rsp_filename, 'wb') as f:
f.write(response.content)
ret[fname_info["I"] + "_rmf"] = rsp_filename
if ret.get(key) and type(ret.get(key)) == str:
log.warning("More than one file found with the instrument: %s" % key)
ret[key] = [ret[key], path_inst_name]
elif ret.get(key) and type(ret.get(key)) == list:
ret[key].append(path_inst_name)
else:
ret[key] = path_inst_name
except FileNotFoundError:
log.error("File %s not found" % (filename))
return
if not ret:
log.info("Nothing to extract with the given parameters:\n"
" PPS: %s\n"
" Source Number: %u\n"
" Instrument: %s\n" % (filename, source_number,
instrument))
return ret
def download_file(self, data_product, local_path, cache=True):
"""
Takes a data product in the form of an `~astropy.table.Row` and downloads it from the cloud into
the given directory.
Parameters
----------
data_product : `~astropy.table.Row`
Product to download.
local_path : str
The local filename to which toe downloaded file will be saved.
cache : bool
Default is True. If file is found on disc it will not be downloaded again.
"""
s3 = self.boto3.resource('s3', config=self.config)
s3_client = self.boto3.client('s3', config=self.config)
bkt = s3.Bucket(self.pubdata_bucket)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
bucket_path = self.get_cloud_uri(data_product, False)
if not bucket_path:
raise Exception("Unable to locate file {}.".format(
data_product['productFilename']))
# Ask the webserver (in this case S3) what the expected content length is and use that.
info_lookup = s3_client.head_object(Bucket=self.pubdata_bucket,
Key=bucket_path)
length = info_lookup["ContentLength"]
if cache and os.path.exists(local_path):
if length is not None:
statinfo = os.stat(local_path)
if statinfo.st_size != length:
log.warning("Found cached file {0} with size {1} that is "
"different from expected size {2}".format(
local_path, statinfo.st_size, length))
else:
log.info(
"Found cached file {0} with expected size {1}.".format(
local_path, statinfo.st_size))
return
with ProgressBarOrSpinner(
length, ('Downloading URL s3://{0}/{1} to {2} ...'.format(
self.pubdata_bucket, bucket_path, local_path))) as pb:
# Bytes read tracks how much data has been received so far
# This variable will be updated in multiple threads below
global bytes_read
bytes_read = 0
progress_lock = threading.Lock()
def progress_callback(numbytes):
# Boto3 calls this from multiple threads pulling the data from S3
global bytes_read
# This callback can be called in multiple threads
# Access to updating the console needs to be locked
with progress_lock:
bytes_read += numbytes
pb.update(bytes_read)
bkt.download_file(bucket_path,
local_path,
Callback=progress_callback)
def get_data_info(self, uids, *, expand_tarfiles=False,
with_auxiliary=True, with_rawdata=True):
"""
Return information about the data associated with ALMA uid(s)
Parameters
----------
uids : list or str
A list of valid UIDs or a single UID.
UIDs should have the form: 'uid://A002/X391d0b/X7b'
expand_tarfiles : bool
False to return information on the tarfiles packages containing
the data or True to return information about individual files in
these packages
with_auxiliary : bool
True to include the auxiliary packages, False otherwise
with_rawdata : bool
True to include raw data, False otherwise
Returns
-------
Table with results or None. Table has the following columns: id (UID),
access_url (URL to access data), content_length, content_type (MIME
type), semantics, description (optional), error_message (optional)
"""
if uids is None:
raise AttributeError('UIDs required')
if isinstance(uids, (str, bytes)):
uids = [uids]
if not isinstance(uids, (list, tuple, np.ndarray)):
raise TypeError("Datasets must be given as a list of strings.")
# TODO remove this loop and send uids at once when pyvo fixed
result = None
for uid in uids:
res = self.datalink.run_sync(uid)
if res.status[0] != 'OK':
raise Exception('ERROR {}: {}'.format(res.status[0],
res.status[1]))
temp = res.to_table()
if ASTROPY_LT_4_1:
# very annoying
for col in [x for x in temp.colnames
if x not in ['content_length', 'readable']]:
temp[col] = temp[col].astype(str)
result = temp if result is None else vstack([result, temp])
to_delete = []
for index, rr in enumerate(result):
if rr['error_message'] is not None and \
rr['error_message'].strip():
log.warning('Error accessing info about file {}: {}'.
format(rr['access_url'], rr['error_message']))
# delete from results. Good thing to do?
to_delete.append(index)
result.remove_rows(to_delete)
if not with_auxiliary:
result = result[np.core.defchararray.find(
result['semantics'], '#aux') == -1]
if not with_rawdata:
result = result[np.core.defchararray.find(
result['semantics'], '#progenitor') == -1]
# primary data delivery type is files packaged in tarballs. However
# some type of data has an alternative way to retrieve each individual
# file as an alternative (semantics='#datalink' and
# 'content_type=application/x-votable+xml;content=datalink'). They also
# require an extra call to the datalink service to get the list of
# files.
DATALINK_FILE_TYPE = 'application/x-votable+xml;content=datalink'
DATALINK_SEMANTICS = '#datalink'
if expand_tarfiles:
# identify the tarballs that can be expandable and replace them
# with the list of components
expanded_result = None
to_delete = []
for index, row in enumerate(result):
if DATALINK_SEMANTICS in row['semantics'] and \
row['content_type'] == DATALINK_FILE_TYPE:
# subsequent call to datalink
file_id = row['access_url'].split('ID=')[1]
expanded_tar = self.get_data_info(file_id)
expanded_tar = expanded_tar[
expanded_tar['semantics'] != '#cutout']
if not expanded_result:
expanded_result = expanded_tar
else:
expanded_result = vstack(
[expanded_result, expanded_tar], join_type='exact')
to_delete.append(index)
# cleanup
result.remove_rows(to_delete)
# add the extra rows
if expanded_result:
result = vstack([result, expanded_result], join_type='exact')
else:
result = result[np.logical_or(np.core.defchararray.find(
result['semantics'].astype(str), DATALINK_SEMANTICS) == -1,
result['content_type'].astype(str) != DATALINK_FILE_TYPE)]
return result
def _download_file(self,
url,
local_filepath,
timeout=None,
auth=None,
continuation=True,
cache=False,
method="GET",
head_safe=False,
**kwargs):
"""
Download a file. Resembles `astropy.utils.data.download_file` but uses
the local ``_session``
Parameters
----------
url : string
local_filepath : string
timeout : int
auth : dict or None
continuation : bool
If the file has already been partially downloaded *and* the server
supports HTTP "range" requests, the download will be continued
where it left off.
cache : bool
method : "GET" or "POST"
head_safe : bool
"""
if head_safe:
response = self._session.request("HEAD",
url,
timeout=timeout,
stream=True,
auth=auth,
**kwargs)
else:
response = self._session.request(method,
url,
timeout=timeout,
stream=True,
auth=auth,
**kwargs)
response.raise_for_status()
if 'content-length' in response.headers:
length = int(response.headers['content-length'])
if length == 0:
log.warn('URL {0} has length=0'.format(url))
else:
length = None
if ((os.path.exists(local_filepath)
and ('Accept-Ranges' in response.headers) and continuation)):
open_mode = 'ab'
existing_file_length = os.stat(local_filepath).st_size
if length is not None and existing_file_length >= length:
# all done!
log.info(
"Found cached file {0} with expected size {1}.".format(
local_filepath, existing_file_length))
return
elif existing_file_length == 0:
open_mode = 'wb'
else:
log.info("Continuing download of file {0}, with {1} bytes to "
"go ({2}%)".format(
local_filepath, length - existing_file_length,
(length - existing_file_length) / length * 100))
# bytes are indexed from 0:
# https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#range-request-header
end = "{0}".format(length - 1) if length is not None else ""
self._session.headers['Range'] = "bytes={0}-{1}".format(
existing_file_length, end)
response = self._session.request(method,
url,
timeout=timeout,
stream=True,
auth=auth,
**kwargs)
response.raise_for_status()
del self._session.headers['Range']
elif cache and os.path.exists(local_filepath):
if length is not None:
statinfo = os.stat(local_filepath)
if statinfo.st_size != length:
log.warning("Found cached file {0} with size {1} that is "
"different from expected size {2}".format(
local_filepath, statinfo.st_size, length))
open_mode = 'wb'
else:
log.info(
"Found cached file {0} with expected size {1}.".format(
local_filepath, statinfo.st_size))
response.close()
return
else:
log.info("Found cached file {0}.".format(local_filepath))
response.close()
return
else:
open_mode = 'wb'
if head_safe:
response = self._session.request(method,
url,
timeout=timeout,
stream=True,
auth=auth,
**kwargs)
response.raise_for_status()
blocksize = astropy.utils.data.conf.download_block_size
log.debug(
f"Downloading URL {url} to {local_filepath} with size {length} "
f"by blocks of {blocksize}")
bytes_read = 0
# Only show progress bar if logging level is INFO or lower.
if log.getEffectiveLevel() <= 20:
progress_stream = None # Astropy default
else:
progress_stream = io.StringIO()
with ProgressBarOrSpinner(
length,
('Downloading URL {0} to {1} ...'.format(url, local_filepath)),
file=progress_stream) as pb:
with open(local_filepath, open_mode) as f:
for block in response.iter_content(blocksize):
f.write(block)
bytes_read += blocksize
if length is not None:
pb.update(
bytes_read if bytes_read <= length else length)
else:
pb.update(bytes_read)
response.close()
return response
def get_epic_spectra(self, filename, source_number, *,
instrument=[], path="", verbose=False):
"""Extracts in path (when set) the EPIC sources spectral products from a
given TAR file.
For a given TAR file obtained with:
XMM.download_data(OBS_ID,level="PPS",extension="FTZ",filename=tarfile)
This function extracts the EPIC sources spectral products in a given
instrument (or instruments) from it
The result is a dictionary containing the paths to the extracted EPIC
sources spectral products with key being the instrument
If the instrument is not specified this function will
return all the available instruments
Examples:
Extracting all bands and instruments:
result = XMM.get_epic_spectra(tarfile,83,
instrument=['M1','M2','PN'])
If we want to retrieve the source spectrum of the instrument PN
fits_image = result['PN']
fits_image will be the full path to the extracted FTZ file
Parameters
----------
filename : string, mandatory
The name of the tarfile to be processed
source_number : integer, mandatory
The source number, in decimal, in the observation
instruments : array of strings, optional, default []
An array of strings indicating the desired instruments
path: string, optional
If set, extracts the EPIC images in the indicated path
verbose : bool
optional, default 'False'
flag to display information about the process
Returns
-------
A dictionary with the full paths of the extracted EPIC sources
spectral products. The key is the instrument
Notes
-----
The filenames will contain the source number in hexadecimal,
as this is the convention used by the pipeline.
The structure and the content of the extracted compressed FITS files
are described in details in the Pipeline Products Description
[XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf).
"""
_instrument = ["M1", "M2", "PN", "EP"]
_product_type = ["SRSPEC", "BGSPEC", "SRCARF"]
_path = ""
ret = None
if instrument == []:
instrument = _instrument
else:
for inst in instrument:
if inst not in _instrument:
log.warning(f"Invalid instrument {inst}")
instrument.remove(inst)
if path != "" and os.path.exists(path):
_path = path
try:
with tarfile.open(filename, "r") as tar:
ret = {}
for member in tar.getmembers():
paths = os.path.split(member.name)
fname = paths[1]
paths = os.path.split(paths[0])
if paths[1] != "pps":
continue
fname_info = self._parse_filename(fname)
if fname_info["X"] != "P":
continue
if not fname_info["I"] in instrument:
continue
if not fname_info["T"] in _product_type:
continue
if int(fname_info["X-"], 16) != source_number:
continue
tar.extract(member, _path)
key = fname_info["I"]
path_inst_name = os.path.abspath(os.path.join(_path, member.name))
if fname_info["T"] == "BGSPEC":
key = fname_info["I"] + "_bkg"
elif fname_info["T"] == "SRCARF":
key = fname_info["I"] + "_arf"
else:
with fits.open(path_inst_name) as hdul:
for ext in hdul:
if ext.name != "SPECTRUM":
continue
rmf_fname = ext.header["RESPFILE"]
if fname_info["I"] == "M1" or fname_info["I"] == "M2":
inst = "MOS/" + str(ext.header["SPECDELT"]) + "eV/"
elif fname_info["I"] == "PN":
inst = "PN/"
file_name, file_ext = os.path.splitext(rmf_fname)
rmf_fname = file_name + "_v18.0" + file_ext
link = self._rmf_ftp + inst + rmf_fname
if verbose:
log.info("rmf link is: %s" % link)
response = self._request('GET', link)
rsp_filename = os.path.join(_path, paths[0], paths[1], ext.header["RESPFILE"])
with open(rsp_filename, 'wb') as f:
f.write(response.content)
ret[fname_info["I"] + "_rmf"] = rsp_filename
if ret.get(key) and type(ret.get(key)) == str:
log.warning("More than one file found with the instrument: %s" % key)
ret[key] = [ret[key], path_inst_name]
elif ret.get(key) and type(ret.get(key)) == list:
ret[key].append(path_inst_name)
else:
ret[key] = path_inst_name
except FileNotFoundError:
log.error("File %s not found" % (filename))
return
if not ret:
log.info("Nothing to extract with the given parameters:\n"
" PPS: %s\n"
" Source Number: %u\n"
" Instrument: %s\n" % (filename, source_number,
instrument))
return ret
def get_epic_images(self,
filename,
*,
band=[],
instrument=[],
get_detmask=False,
get_exposure_map=False,
path=""):
"""Extracts the European Photon Imaging Camera (EPIC) images from a given TAR file
For a given TAR file obtained with:
XMM.download_data(OBS_ID,level="PPS",extension="FTZ",filename=tarfile)
This function extracts the EPIC images in a given band (or bands) and
instrument (or instruments) from it
The result is a dictionary containing the paths to the extracted EPIC
images with keys being the band and the instrument
If the band or the instrument are not specified this function will
return all the available bands and instruments
Additionally, ``get_detmask`` and ``get_exposure_map`` can be set to True.
If so, this function will also extract the exposure maps and detector
masks within the specified bands and instruments
Examples
--------
Extract all bands and instruments::
result = XMM.get_epic_images(tarfile,band=[1,2,3,4,5,8],
instrument=['M1','M2','PN'],**kwargs)
If we want to retrieve the band 3 for the instrument PN (p-n junction)::
fits_image = result[3]['PN']
``fits_image`` will be the full path to the extracted FTZ file
Extract the exposure and detector maps::
result = XMM.get_epic_images(tarfile,band=[1,2,3,4,5,8],
instrument=['M1','M2','PN'],
get_detmask=True,
get_exposure_map=True)
If we want to retrieve exposure map in the band 3 for the instrument PN::
fits_image = result[3]['PN_expo']
For retrieving the detector mask in the band 3 for the instrument PN::
fits_image = result[3]['PN_det']
Parameters
----------
filename : string, mandatory
The name of the tarfile to be proccessed
band : array of integers, optional, default []
An array of intergers indicating the desired bands
instruments : array of strings, optional, default []
An array of strings indicating the desired instruments
get_detmask : bool, optional
If True, also extracts the detector masks
get_exposure_map : bool, optional
If True, also extracts the exposure maps
path: string, optional
If set, extracts the EPIC images in the indicated path
Returns
-------
A dictionary of dictionaries with the full paths of the extracted
EPIC images. The keys of each dictionary are the band for the first
level dictionary and the instrument for the second level dictionaries
Notes
-----
The structure and the content of the extracted compressed FITS files
are described in details in the Pipeline Products Description
[XMM-SOC-GEN-ICD-0024](https://xmm-tools.cosmos.esa.int/external/xmm_obs_info/odf/data/docs/XMM-SOC-GEN-ICD-0024.pdf).
"""
_product_type = ["IMAGE_"]
_instrument = ["M1", "M2", "PN", "EP"]
_band = [1, 2, 3, 4, 5, 8]
_path = ""
if get_detmask:
_product_type.append("DETMSK")
if get_exposure_map:
_product_type.append("EXPMAP")
if path != "" and os.path.exists(path):
_path = path
ret = {}
if band == []:
band = _band
else:
for i in band:
if i not in _band:
log.warning("Invalid band %u" % i)
band.remove(i)
if instrument == []:
instrument = _instrument
else:
for i in instrument:
if i not in _instrument:
log.warning("Invalid instrument %s" % i)
instrument.remove(i)
with tarfile.open(filename, "r") as tar:
for member in tar.getmembers():
paths = os.path.split(member.name)
fname = paths[1]
paths = os.path.split(paths[0])
if paths[1] != "pps":
continue
fname_info = self._parse_filename(fname)
if fname_info["X"] != "P":
continue
if not fname_info["I"] in instrument:
continue
if not int(fname_info["S"]) in band:
continue
if not fname_info["T"] in _product_type:
continue
tar.extract(member, _path)
if not ret.get(int(fname_info["S"])):
ret[int(fname_info["S"])] = {}
b = int(fname_info["S"])
ins = fname_info["I"]
value = os.path.abspath(os.path.join(_path, member.name))
if fname_info["T"] == "DETMSK":
ins = fname_info["I"] + "_det"
elif fname_info["T"] == "EXPMAP":
ins = fname_info["I"] + "_expo"
if ret[b].get(ins) and type(ret[b].get(ins)) == str:
log.warning("More than one file found with the "
"band %u and "
"the instrument: %s" % (b, ins))
ret[b][ins] = [ret[b][ins], value]
elif ret[b].get(ins) and type(ret[b].get(ins)) == list:
ret[b][ins].append(value)
else:
ret[b][ins] = value
return ret
