示例#1
0
def main_method(config):
    """main method for processing spectra.

    Parameters
    ----------
    config : :obj:`Config`
        Configuration object

    Returns
    -------
    int
        0 on success
    """
    # initialize logger
    init_logger("process_spectra", config.logdir, config.log_level)
    start_message = "Running process_spectra {}".format(VERSION)
    logger.info(start_message)

    if config.process_method.lower() == 'amazed':
        amazed(config)
    elif config.process_method.lower() == 'dummy':
        dummy(config)
    else:
        raise "Unknown process_method {}".format(config.process_method)

    return 0
示例#2
0
def main_method(config):
    """main_method

    Parameters
    ----------
    config : :obj:`Config`
        Configuration object

    Returns
    -------
    int
        0 on success
    """

    # initialize logger
    logger = init_logger("pre_process", config.logdir, config.log_level)
    start_message = "Running pre_process {}".format(VERSION)
    logger.info(start_message)

    spectra_dir = normpath(config.workdir, config.spectra_dir)

    # bunch
    bunch_list = []
    for i, spc_list in enumerate(bunch(config.bunch_size, spectra_dir)):
        spectralist_file = os.path.join(config.output_dir,
                                        'spectralist_B{}.json'.format(str(i)))
        with open(spectralist_file, "w") as ff:
            json.dump(spc_list, ff)
        bunch_list.append(spectralist_file)

    # create json containing list of bunches
    with open(config.bunch_list, 'w') as f:
        json.dump(bunch_list, f)

    return 0
示例#3
0
def main_method(config):
    """main_method

    Parameters
    ----------
    config : :obj:`Config`
        Configuration object

    Returns
    -------
    int
        0 on success
    """

    # initialize logger
    logger = init_logger("merge_results", config.logdir, config.log_level)
    start_message = "Running merge_results {}".format(VERSION)
    logger.info(start_message)

    if not os.path.exists(config.bunch_listfile):
        raise FileNotFoundError("Bunch list file not found : {}".format(
            config.bunch_listfile))

    with open(config.bunch_listfile, "r") as ff:
        bunch_list = json.load(ff)

    data_dir = os.path.join(config.output_dir, 'data')
    os.makedirs(data_dir, exist_ok=True)
    for bunch in bunch_list:
        if not os.path.exists(bunch):
            raise FileNotFoundError(
                "Bunch directory not found : {}".format(bunch))
        bunch_data_dir = os.path.join(bunch, "data")
        if not os.path.exists(bunch_data_dir):
            raise FileNotFoundError(
                "Bunch data directory not found : {}".format(bunch_data_dir))
        to_merge = os.listdir(bunch_data_dir)
        for pfs_candidate in to_merge:
            shutil.move(os.path.join(bunch_data_dir, pfs_candidate),
                        os.path.join(data_dir, pfs_candidate))

    return 0
示例#4
0
def main_method(config):
    """main_method

    Parameters
    ----------
    config : :obj:`Config`
        Configuration object

    Returns
    -------
    int
        0 on success
    """

    # initialize logger
    logger = init_logger("merge_results", config.logdir, config.log_level)
    start_message = "Running merge_results {}".format(VERSION)
    logger.info(start_message)

    if not os.path.exists(config.bunch_listfile):
        raise FileNotFoundError("Bunch list file not found : {}".format(
            config.bunch_listfile))

    with open(config.bunch_listfile, "r") as ff:
        bunch_list = json.load(ff)

    data_dir = os.path.join(config.output_dir, 'data')
    os.makedirs(data_dir, exist_ok=True)
    galaxy_summary_list = []
    stellar_summary_list = []
    qso_summary_list = []
    for bunch in bunch_list:
        if not os.path.exists(bunch):
            raise FileNotFoundError(
                "Bunch directory not found : {}".format(bunch))
        bunch_data_dir = os.path.join(bunch, "data")
        if not os.path.exists(bunch_data_dir):
            raise FileNotFoundError(
                "Bunch data directory not found : {}".format(bunch_data_dir))
        to_merge = os.listdir(bunch_data_dir)
        for pfs_candidate in to_merge:
            shutil.move(os.path.join(bunch_data_dir, pfs_candidate),
                        os.path.join(data_dir, pfs_candidate))

        try:
            amazed_results = RedshiftSummary(output_dir=bunch)
            amazed_results.read()
            galaxy_summary_list.extend(amazed_results.summary)
        except FileNotFoundError:
            raise FileNotFoundError(
                "Redshift summary file not found in {}".format(bunch))

        try:
            amazed_results = StellarSummary(output_dir=bunch)
            amazed_results.read()
            stellar_summary_list.extend(amazed_results.summary)
        except:
            pass

        try:
            amazed_results = QsoSummary(output_dir=bunch)
            amazed_results.read()
            qso_summary_list.extend(amazed_results.summary)
        except:
            pass

    gsr = RedshiftSummary(output_dir=config.output_dir)
    gsr.summary = galaxy_summary_list
    gsr.write()
    ssr = StellarSummary(output_dir=config.output_dir)
    ssr.summary = stellar_summary_list
    ssr.write()
    qsr = QsoSummary(output_dir=config.output_dir)
    qsr.summary = qso_summary_list
    qsr.write()

    return 0
示例#5
0
def main_method(config):
    """Run the 1D Data Reduction Pipeline.

    Returns
    -------
    int
        0 on success
    """

    # initialize logger
    logger = init_logger('scheduler', config.logdir, config.log_level)
    start_message = "Running drp_1dpipe {}".format(VERSION)
    logger.info(start_message)

    # Launch banner
    print(start_message)

    # set workdir environment
    init_environ(config.workdir)

    runner_class = get_runner(config.scheduler)
    # if not runner_class:
    #     error_message = "Unknown runner {}".format(config.scheduler)
    #     logger.error(error_message)
    #     raise error_message

    notifier = init_notifier(config.notification_url)

    json_bunch_list = normpath(config.output_dir, 'bunchlist.json')

    notifier.update('root', 'RUNNING')
    notifier.update('pre_process', 'RUNNING')

    with TemporaryFilesSet(keep_tempfiles=config.log_level <= logging.DEBUG) as tmpcontext:

        runner = runner_class(config, tmpcontext)

        # prepare workdir
        try:
            runner.single('pre_process',
                        args={'workdir': normpath(config.workdir),
                                'logdir': normpath(config.logdir),
                                'bunch_size': config.bunch_size,
                                'spectra_dir': normpath(config.spectra_dir),
                                'bunch_list': json_bunch_list,
                                'output_dir': normpath(config.output_dir)
                                })
        except Exception as e:
            traceback.print_exc()
            notifier.update('pre_process', 'ERROR')
            return 1
        else:
            notifier.update('pre_process', 'SUCCESS')
            # tmpcontext.add_files(json_bunch_list)

        # process spectra
        bunch_list, output_list, logdir_list = map_process_spectra_entries(
            json_bunch_list, config.output_dir, config.logdir)
        try:
            # runner.parallel('process_spectra', bunch_list,
            #                 'spectra-listfile', ['output-dir','logdir'],
            runner.parallel('process_spectra',
                            parallel_args={
                                'spectra_listfile': bunch_list,
                                'output_dir': output_list,
                                'logdir': logdir_list
                            },
                            args={
                                'workdir': normpath(config.workdir),
                                'lineflux': config.lineflux,
                                'spectra_dir': normpath(config.spectra_dir),
                                'parameters_file': config.parameters_file,
                                'linemeas_parameters_file': config.linemeas_parameters_file,
                                'stellar': config.stellar
                            })
        except Exception as e:
            traceback.print_exc()
            notifier.update('root', 'ERROR')
        else:
            notifier.update('root', 'SUCCESS')
        
        json_reduce = normpath(config.output_dir, 'reduce.json')
        reduce_process_spectra_output(json_bunch_list, config.output_dir, json_reduce)
        try:
            runner.single('merge_results',
                            args={
                                'workdir': normpath(config.workdir),
                                'logdir': normpath(config.logdir),
                                'output_dir': normpath(config.output_dir),
                                'bunch_listfile': json_reduce
                        })
        except Exception as e:
            traceback.print_exc()
            notifier.update('merge_results', 'ERROR')
            return 1
        else:
            notifier.update('merge_results', 'SUCCESS')

        aux_data_list = list_aux_data(json_bunch_list, config.output_dir)
        for aux_dir in aux_data_list:
            tmpcontext.add_dirs(aux_dir)


    return 0