Ejemplo n.º 1
0
def download_GBM_trigger_data(trigger_name,
                              detectors=None,
                              destination_directory='.',
                              compress_tte=True):
    """
    Download the latest GBM TTE and RSP files from the HEASARC server. Will get the
    latest file version and prefer RSP2s over RSPs. If the files already exist in your destination
    directory, they will be skipped in the download process. The output dictionary can be used
    as input to the FermiGBMTTELike class.

    example usage: download_GBM_trigger_data('080916009', detectors=['n0','na','b0'], destination_directory='.')

    :param trigger_name: trigger number (str) e.g. '080916009' or 'bn080916009' or 'GRB080916009'
    :param detectors: list of detectors, default is all detectors
    :param destination_directory: download directory
    :param compress_tte: compress the TTE files via gzip (default True)
    :return: a dictionary with information about the download
    """

    # Let's doctor up the input just in case the user tried something strange

    sanitized_trigger_name_ = _validate_fermi_trigger_name(trigger_name)

    # create output directory if it does not exists
    destination_directory = sanitize_filename(destination_directory,
                                              abspath=True)

    if_directory_not_existing_then_make(destination_directory)

    # Sanitize detector list (if any)
    if detectors is not None:

        for det in detectors:

            assert det in _detector_list, "Detector %s in the provided list is not a valid detector. " \
                                          "Valid choices are: %s" % (det, _detector_list)

    else:

        detectors = list(_detector_list)

    # Open heasarc web page

    url = threeML_config['gbm']['public HTTP location']
    year = '20%s' % sanitized_trigger_name_[:2]
    directory = '/triggers/%s/bn%s/current' % (year, sanitized_trigger_name_)

    heasarc_web_page_url = '%s/%s' % (url, directory)

    try:

        downloader = ApacheDirectory(heasarc_web_page_url)

    except RemoteDirectoryNotFound:

        raise TriggerDoesNotExist(
            "Trigger %s does not exist at %s" %
            (sanitized_trigger_name_, heasarc_web_page_url))

    # Now select the files we want to download, then we will download them later
    # We do it in two steps because we want to be able to choose what to download once we
    # have the complete picture

    # Get the list of remote files
    remote_file_list = downloader.files

    # This is the dictionary to keep track of the classification
    remote_files_info = DictWithPrettyPrint([(det, {}) for det in detectors])

    # Classify the files detector by detector

    for this_file in remote_file_list:

        # this_file is something like glg_tte_n9_bn100101988_v00.fit
        tokens = this_file.split("_")

        if len(tokens) != 5:

            # Not a data file

            continue

        else:

            # The "map" is necessary to transform the tokens to normal string (instead of unicode),
            # because u"b0" != "b0" as a key for a dictionary

            _, file_type, detname, _, version_ext = list(map(str, tokens))

        version, ext = version_ext.split(".")

        # We do not care here about the other files (tcat, bcat and so on),
        # nor about files which pertain to other detectors

        if file_type not in ['cspec', 'tte'] or ext not in [
                'rsp', 'rsp2', 'pha', 'fit'
        ] or detname not in detectors:

            continue

        # cspec files can be rsp, rsp2 or pha files. Classify them

        if file_type == 'cspec':

            if ext == 'rsp':

                remote_files_info[detname]['rsp'] = this_file

            elif ext == 'rsp2':

                remote_files_info[detname]['rsp2'] = this_file

            elif ext == 'pha':

                remote_files_info[detname]['cspec'] = this_file

            else:

                raise RuntimeError("Should never get here")

        else:

            remote_files_info[detname][file_type] = this_file

    # Now download the files

    download_info = DictWithPrettyPrint([(det, DictWithPrettyPrint())
                                         for det in detectors])

    for detector in list(remote_files_info.keys()):

        remote_detector_info = remote_files_info[detector]
        local_detector_info = download_info[detector]

        # Get CSPEC file
        local_detector_info['cspec'] = downloader.download(
            remote_detector_info['cspec'],
            destination_directory,
            progress=True)

        # Get the RSP2 file if it exists, otherwise get the RSP file
        if 'rsp2' in remote_detector_info:

            local_detector_info['rsp'] = downloader.download(
                remote_detector_info['rsp2'],
                destination_directory,
                progress=True)

        else:

            local_detector_info['rsp'] = downloader.download(
                remote_detector_info['rsp'],
                destination_directory,
                progress=True)

        # Get TTE file (compressing it if requested)
        local_detector_info['tte'] = downloader.download(
            remote_detector_info['tte'],
            destination_directory,
            progress=True,
            compress=compress_tte)

    return download_info
Ejemplo n.º 2
0
def download_LLE_trigger_data(trigger_name, destination_directory='.'):
    """
    Download the latest Fermi LAT LLE and RSP files from the HEASARC server. Will get the
    latest file versions. If the files already exist in your destination
    directory, they will be skipped in the download process. The output dictionary can be used
    as input to the FermiLATLLELike class.

    example usage: download_LLE_trigger_data('080916009', destination_directory='.')

    :param trigger_name: trigger number (str) with no leading letter e.g. '080916009'
    :param destination_directory: download directory
    :return: a dictionary with information about the download
    """

    sanitized_trigger_name_ = _validate_fermi_trigger_name(trigger_name)

    # create output directory if it does not exists
    destination_directory = sanitize_filename(destination_directory,
                                              abspath=True)
    if_directory_not_existing_then_make(destination_directory)

    # Figure out the directory on the server
    url = threeML_config['LAT']['public HTTP location']

    year = '20%s' % sanitized_trigger_name_[:2]
    directory = 'triggers/%s/bn%s/current' % (year, sanitized_trigger_name_)

    heasarc_web_page_url = '%s/%s' % (url, directory)

    try:

        downloader = ApacheDirectory(heasarc_web_page_url)

    except RemoteDirectoryNotFound:

        raise TriggerDoesNotExist(
            "Trigger %s does not exist at %s" %
            (sanitized_trigger_name_, heasarc_web_page_url))

    # Download only the lle, pt, cspec and rsp file (i.e., do not get all the png, pdf and so on)
    pattern = 'gll_(lle|pt|cspec)_bn.+\.(fit|rsp|pha)'

    destination_directory_sanitized = sanitize_filename(destination_directory)

    downloaded_files = downloader.download_all_files(
        destination_directory_sanitized, progress=True, pattern=pattern)

    # Put the files in a structured dictionary

    download_info = DictWithPrettyPrint()

    for download in downloaded_files:

        file_type = _file_type_match.match(os.path.basename(download)).group(1)

        if file_type == 'cspec':

            # a cspec file can be 2 things: a CSPEC spectral set (with .pha) extension,
            # or a response matrix (with a .rsp extension)

            ext = os.path.splitext(os.path.basename(download))[1]

            if ext == '.rsp':

                file_type = 'rsp'

            elif ext == '.pha':

                file_type = 'cspec'

            else:

                raise RuntimeError("Should never get here")

        # The pt file is really an ft2 file

        if file_type == 'pt':

            file_type = 'ft2'

        download_info[file_type] = download

    return download_info
Ejemplo n.º 3
0
def download_GBM_trigger_data(trigger_name: str,
                              detectors: Optional[List[str]] = None,
                              destination_directory: str = ".",
                              compress_tte: bool = True) -> Dict[str, Any]:
    """
    Download the latest GBM TTE and RSP files from the HEASARC server. Will get the
    latest file version and prefer RSP2s over RSPs. If the files already exist in your destination
    directory, they will be skipped in the download process. The output dictionary can be used
    as input to the FermiGBMTTELike class.

    example usage: download_GBM_trigger_data('080916009', detectors=['n0','na','b0'], destination_directory='.')

    :param trigger_name: trigger number (str) e.g. '080916009' or 'bn080916009' or 'GRB080916009'
    :param detectors: list of detectors, default is all detectors
    :param destination_directory: download directory
    :param compress_tte: compress the TTE files via gzip (default True)
    :return: a dictionary with information about the download
    """

    # Let's doctor up the input just in case the user tried something strange

    sanitized_trigger_name_: str = _validate_fermi_trigger_name(trigger_name)

    # create output directory if it does not exists
    destination_directory: Path = sanitize_filename(destination_directory,
                                                    abspath=True)

    if_directory_not_existing_then_make(destination_directory)

    # Sanitize detector list (if any)
    if detectors is not None:

        for det in detectors:

            if det not in _detector_list:
                log.error(
                    f"Detector {det} in the provided list is not a valid detector. "
                    f"Valid choices are: {_detector_list}")
                raise DetDoesNotExist()

    else:

        detectors: List[str] = list(_detector_list)

    # Open heasarc web page

    url = threeML_config.GBM.public_http_location
    year = f"20{sanitized_trigger_name_[:2]}"
    directory = f"/triggers/{year}/bn{sanitized_trigger_name_}/current"

    heasarc_web_page_url = f"{url}/{directory}"

    log.debug(f"going to look in {heasarc_web_page_url}")

    try:

        downloader = ApacheDirectory(heasarc_web_page_url)

    except RemoteDirectoryNotFound:

        log.exception(
            f"Trigger {sanitized_trigger_name_} does not exist at {heasarc_web_page_url}"
        )

        raise TriggerDoesNotExist()

    # Now select the files we want to download, then we will download them later
    # We do it in two steps because we want to be able to choose what to download once we
    # have the complete picture

    # Get the list of remote files
    remote_file_list = downloader.files

    # This is the dictionary to keep track of the classification
    remote_files_info = DictWithPrettyPrint([(det, {}) for det in detectors])

    # Classify the files detector by detector

    for this_file in remote_file_list:

        # this_file is something like glg_tte_n9_bn100101988_v00.fit
        tokens = this_file.split("_")

        if len(tokens) != 5:

            # Not a data file

            continue

        else:

            # The "map" is necessary to transform the tokens to normal string (instead of unicode),
            # because u"b0" != "b0" as a key for a dictionary

            _, file_type, detname, _, version_ext = list(map(str, tokens))

        version, ext = version_ext.split(".")

        # We do not care here about the other files (tcat, bcat and so on),
        # nor about files which pertain to other detectors

        if (file_type not in ["cspec", "tte"]
                or ext not in ["rsp", "rsp2", "pha", "fit"]
                or detname not in detectors):

            continue

        # cspec files can be rsp, rsp2 or pha files. Classify them

        if file_type == "cspec":

            if ext == "rsp":

                remote_files_info[detname]["rsp"] = this_file

            elif ext == "rsp2":

                remote_files_info[detname]["rsp2"] = this_file

            elif ext == "pha":

                remote_files_info[detname]["cspec"] = this_file

            else:

                raise RuntimeError("Should never get here")

        else:

            remote_files_info[detname][file_type] = this_file

    # Now download the files

    download_info = DictWithPrettyPrint([(det, DictWithPrettyPrint())
                                         for det in detectors])

    for detector in list(remote_files_info.keys()):

        log.debug(f"trying to download GBM detector {detector}")

        remote_detector_info = remote_files_info[detector]
        local_detector_info = download_info[detector]

        # Get CSPEC file
        local_detector_info["cspec"] = downloader.download(
            remote_detector_info["cspec"],
            destination_directory,
            progress=True)

        # Get the RSP2 file if it exists, otherwise get the RSP file
        if "rsp2" in remote_detector_info:

            log.debug(f"{detector} has RSP2 responses")

            local_detector_info["rsp"] = downloader.download(
                remote_detector_info["rsp2"],
                destination_directory,
                progress=True)

        else:

            log.debug(f"{detector} has RSP responses")

            local_detector_info["rsp"] = downloader.download(
                remote_detector_info["rsp"],
                destination_directory,
                progress=True)

        # Get TTE file (compressing it if requested)
        local_detector_info["tte"] = downloader.download(
            remote_detector_info["tte"],
            destination_directory,
            progress=True,
            compress=compress_tte,
        )

    return download_info