def download_GBM_trigger_data(trigger_name,
detectors=None,
destination_directory='.',
compress_tte=True):
"""
Download the latest GBM TTE and RSP files from the HEASARC server. Will get the
latest file version and prefer RSP2s over RSPs. If the files already exist in your destination
directory, they will be skipped in the download process. The output dictionary can be used
as input to the FermiGBMTTELike class.
example usage: download_GBM_trigger_data('080916009', detectors=['n0','na','b0'], destination_directory='.')
:param trigger_name: trigger number (str) e.g. '080916009' or 'bn080916009' or 'GRB080916009'
:param detectors: list of detectors, default is all detectors
:param destination_directory: download directory
:param compress_tte: compress the TTE files via gzip (default True)
:return: a dictionary with information about the download
"""
# Let's doctor up the input just in case the user tried something strange
sanitized_trigger_name_ = _validate_fermi_trigger_name(trigger_name)
# create output directory if it does not exists
destination_directory = sanitize_filename(destination_directory,
abspath=True)
if_directory_not_existing_then_make(destination_directory)
# Sanitize detector list (if any)
if detectors is not None:
for det in detectors:
assert det in _detector_list, "Detector %s in the provided list is not a valid detector. " \
"Valid choices are: %s" % (det, _detector_list)
else:
detectors = list(_detector_list)
# Open heasarc web page
url = threeML_config['gbm']['public HTTP location']
year = '20%s' % sanitized_trigger_name_[:2]
directory = '/triggers/%s/bn%s/current' % (year, sanitized_trigger_name_)
heasarc_web_page_url = '%s/%s' % (url, directory)
try:
downloader = ApacheDirectory(heasarc_web_page_url)
except RemoteDirectoryNotFound:
raise TriggerDoesNotExist(
"Trigger %s does not exist at %s" %
(sanitized_trigger_name_, heasarc_web_page_url))
# Now select the files we want to download, then we will download them later
# We do it in two steps because we want to be able to choose what to download once we
# have the complete picture
# Get the list of remote files
remote_file_list = downloader.files
# This is the dictionary to keep track of the classification
remote_files_info = DictWithPrettyPrint([(det, {}) for det in detectors])
# Classify the files detector by detector
for this_file in remote_file_list:
# this_file is something like glg_tte_n9_bn100101988_v00.fit
tokens = this_file.split("_")
if len(tokens) != 5:
# Not a data file
continue
else:
# The "map" is necessary to transform the tokens to normal string (instead of unicode),
# because u"b0" != "b0" as a key for a dictionary
_, file_type, detname, _, version_ext = list(map(str, tokens))
version, ext = version_ext.split(".")
# We do not care here about the other files (tcat, bcat and so on),
# nor about files which pertain to other detectors
if file_type not in ['cspec', 'tte'] or ext not in [
'rsp', 'rsp2', 'pha', 'fit'
] or detname not in detectors:
continue
# cspec files can be rsp, rsp2 or pha files. Classify them
if file_type == 'cspec':
if ext == 'rsp':
remote_files_info[detname]['rsp'] = this_file
elif ext == 'rsp2':
remote_files_info[detname]['rsp2'] = this_file
elif ext == 'pha':
remote_files_info[detname]['cspec'] = this_file
else:
raise RuntimeError("Should never get here")
else:
remote_files_info[detname][file_type] = this_file
# Now download the files
download_info = DictWithPrettyPrint([(det, DictWithPrettyPrint())
for det in detectors])
for detector in list(remote_files_info.keys()):
remote_detector_info = remote_files_info[detector]
local_detector_info = download_info[detector]
# Get CSPEC file
local_detector_info['cspec'] = downloader.download(
remote_detector_info['cspec'],
destination_directory,
progress=True)
# Get the RSP2 file if it exists, otherwise get the RSP file
if 'rsp2' in remote_detector_info:
local_detector_info['rsp'] = downloader.download(
remote_detector_info['rsp2'],
destination_directory,
progress=True)
else:
local_detector_info['rsp'] = downloader.download(
remote_detector_info['rsp'],
destination_directory,
progress=True)
# Get TTE file (compressing it if requested)
local_detector_info['tte'] = downloader.download(
remote_detector_info['tte'],
destination_directory,
progress=True,
compress=compress_tte)
return download_info
def download_LLE_trigger_data(trigger_name, destination_directory='.'):
"""
Download the latest Fermi LAT LLE and RSP files from the HEASARC server. Will get the
latest file versions. If the files already exist in your destination
directory, they will be skipped in the download process. The output dictionary can be used
as input to the FermiLATLLELike class.
example usage: download_LLE_trigger_data('080916009', destination_directory='.')
:param trigger_name: trigger number (str) with no leading letter e.g. '080916009'
:param destination_directory: download directory
:return: a dictionary with information about the download
"""
sanitized_trigger_name_ = _validate_fermi_trigger_name(trigger_name)
# create output directory if it does not exists
destination_directory = sanitize_filename(destination_directory,
abspath=True)
if_directory_not_existing_then_make(destination_directory)
# Figure out the directory on the server
url = threeML_config['LAT']['public HTTP location']
year = '20%s' % sanitized_trigger_name_[:2]
directory = 'triggers/%s/bn%s/current' % (year, sanitized_trigger_name_)
heasarc_web_page_url = '%s/%s' % (url, directory)
try:
downloader = ApacheDirectory(heasarc_web_page_url)
except RemoteDirectoryNotFound:
raise TriggerDoesNotExist(
"Trigger %s does not exist at %s" %
(sanitized_trigger_name_, heasarc_web_page_url))
# Download only the lle, pt, cspec and rsp file (i.e., do not get all the png, pdf and so on)
pattern = 'gll_(lle|pt|cspec)_bn.+\.(fit|rsp|pha)'
destination_directory_sanitized = sanitize_filename(destination_directory)
downloaded_files = downloader.download_all_files(
destination_directory_sanitized, progress=True, pattern=pattern)
# Put the files in a structured dictionary
download_info = DictWithPrettyPrint()
for download in downloaded_files:
file_type = _file_type_match.match(os.path.basename(download)).group(1)
if file_type == 'cspec':
# a cspec file can be 2 things: a CSPEC spectral set (with .pha) extension,
# or a response matrix (with a .rsp extension)
ext = os.path.splitext(os.path.basename(download))[1]
if ext == '.rsp':
file_type = 'rsp'
elif ext == '.pha':
file_type = 'cspec'
else:
raise RuntimeError("Should never get here")
# The pt file is really an ft2 file
if file_type == 'pt':
file_type = 'ft2'
download_info[file_type] = download
return download_info
def download_GBM_trigger_data(trigger_name: str,
detectors: Optional[List[str]] = None,
destination_directory: str = ".",
compress_tte: bool = True) -> Dict[str, Any]:
"""
Download the latest GBM TTE and RSP files from the HEASARC server. Will get the
latest file version and prefer RSP2s over RSPs. If the files already exist in your destination
directory, they will be skipped in the download process. The output dictionary can be used
as input to the FermiGBMTTELike class.
example usage: download_GBM_trigger_data('080916009', detectors=['n0','na','b0'], destination_directory='.')
:param trigger_name: trigger number (str) e.g. '080916009' or 'bn080916009' or 'GRB080916009'
:param detectors: list of detectors, default is all detectors
:param destination_directory: download directory
:param compress_tte: compress the TTE files via gzip (default True)
:return: a dictionary with information about the download
"""
# Let's doctor up the input just in case the user tried something strange
sanitized_trigger_name_: str = _validate_fermi_trigger_name(trigger_name)
# create output directory if it does not exists
destination_directory: Path = sanitize_filename(destination_directory,
abspath=True)
if_directory_not_existing_then_make(destination_directory)
# Sanitize detector list (if any)
if detectors is not None:
for det in detectors:
if det not in _detector_list:
log.error(
f"Detector {det} in the provided list is not a valid detector. "
f"Valid choices are: {_detector_list}")
raise DetDoesNotExist()
else:
detectors: List[str] = list(_detector_list)
# Open heasarc web page
url = threeML_config.GBM.public_http_location
year = f"20{sanitized_trigger_name_[:2]}"
directory = f"/triggers/{year}/bn{sanitized_trigger_name_}/current"
heasarc_web_page_url = f"{url}/{directory}"
log.debug(f"going to look in {heasarc_web_page_url}")
try:
downloader = ApacheDirectory(heasarc_web_page_url)
except RemoteDirectoryNotFound:
log.exception(
f"Trigger {sanitized_trigger_name_} does not exist at {heasarc_web_page_url}"
)
raise TriggerDoesNotExist()
# Now select the files we want to download, then we will download them later
# We do it in two steps because we want to be able to choose what to download once we
# have the complete picture
# Get the list of remote files
remote_file_list = downloader.files
# This is the dictionary to keep track of the classification
remote_files_info = DictWithPrettyPrint([(det, {}) for det in detectors])
# Classify the files detector by detector
for this_file in remote_file_list:
# this_file is something like glg_tte_n9_bn100101988_v00.fit
tokens = this_file.split("_")
if len(tokens) != 5:
# Not a data file
continue
else:
# The "map" is necessary to transform the tokens to normal string (instead of unicode),
# because u"b0" != "b0" as a key for a dictionary
_, file_type, detname, _, version_ext = list(map(str, tokens))
version, ext = version_ext.split(".")
# We do not care here about the other files (tcat, bcat and so on),
# nor about files which pertain to other detectors
if (file_type not in ["cspec", "tte"]
or ext not in ["rsp", "rsp2", "pha", "fit"]
or detname not in detectors):
continue
# cspec files can be rsp, rsp2 or pha files. Classify them
if file_type == "cspec":
if ext == "rsp":
remote_files_info[detname]["rsp"] = this_file
elif ext == "rsp2":
remote_files_info[detname]["rsp2"] = this_file
elif ext == "pha":
remote_files_info[detname]["cspec"] = this_file
else:
raise RuntimeError("Should never get here")
else:
remote_files_info[detname][file_type] = this_file
# Now download the files
download_info = DictWithPrettyPrint([(det, DictWithPrettyPrint())
for det in detectors])
for detector in list(remote_files_info.keys()):
log.debug(f"trying to download GBM detector {detector}")
remote_detector_info = remote_files_info[detector]
local_detector_info = download_info[detector]
# Get CSPEC file
local_detector_info["cspec"] = downloader.download(
remote_detector_info["cspec"],
destination_directory,
progress=True)
# Get the RSP2 file if it exists, otherwise get the RSP file
if "rsp2" in remote_detector_info:
log.debug(f"{detector} has RSP2 responses")
local_detector_info["rsp"] = downloader.download(
remote_detector_info["rsp2"],
destination_directory,
progress=True)
else:
log.debug(f"{detector} has RSP responses")
local_detector_info["rsp"] = downloader.download(
remote_detector_info["rsp"],
destination_directory,
progress=True)
# Get TTE file (compressing it if requested)
local_detector_info["tte"] = downloader.download(
remote_detector_info["tte"],
destination_directory,
progress=True,
compress=compress_tte,
)
return download_info