Exemplo n.º 1
0
def density_plot(model_name,
                 ref_name,
                 model_values,
                 ref_values,
                 axis_min=None,
                 axis_max=None,
                 target_file=None,
                 unit=None,
                 log_scaled=None,
                 config=None):
    """
    Creates the density plot for the given matchups and variables and possible writes it to the given target file.
    @param model_name: the name of the model variable.
    @param ref_name: the name of the reference variable.
    @param model_name: the name of the model variable.
    @param ref_name: the name of the reference variable.
    @param target_file: the optional target diagram file. If None, nothing will be written.
    @param config: the optional configuration.
    @return: the density plot.
    """
    op = Output(config=config)
    if log_scaled is None and config is not None:
        log_scaled = config.density_plot_log_scaled
    elif log_scaled is None:
        log_scaled = False
    return op.density_plot(model_name, ref_name, model_values, ref_values,
                           log_scaled, target_file, axis_min, axis_max, unit)
Exemplo n.º 2
0
def target_diagram(statistics, target_file=None, config=None):
    """
    Creates the target diagram derived from the statistics and possibly writes it to the given target file.
    @param statistics: An iterable of statistics to write the target diagram for.
    @param target_file: the name of the target diagram file. If None, nothing will be written.
    @param config: the optional configuration.
    @return: the target diagram.
    """
    op = Output(config=config)
    return op.target_diagram(statistics, target_file)
Exemplo n.º 3
0
def target_diagram(statistics, target_file=None, config=None):
    """
    Creates the target diagram derived from the statistics and possibly writes it to the given target file.
    @param statistics: An iterable of statistics to write the target diagram for.
    @param target_file: the name of the target diagram file. If None, nothing will be written.
    @param config: the optional configuration.
    @return: the target diagram.
    """
    op = Output(config=config)
    return op.target_diagram(statistics, target_file)
Exemplo n.º 4
0
def taylor_diagrams(statistics, target_file=None, config=None):
    """
    Creates the taylor diagrams derived from the statistics and possibly writes them to the given target file.
    @param statistics: an iterable of statistics to write taylor diagrams for.
    @param target_file: the basename for the file where to write the taylor diagrams. If None, nothing will be written.
    @param config: the optional configuration.
    @return: a list of the taylor diagrams.
    """
    op = Output(config=config)
    t, diagrams = op.taylor(statistics, target_file)
    return diagrams
Exemplo n.º 5
0
def taylor_diagrams(statistics, target_file=None, config=None):
    """
    Creates the taylor diagrams derived from the statistics and possibly writes them to the given target file.
    @param statistics: an iterable of statistics to write taylor diagrams for.
    @param target_file: the basename for the file where to write the taylor diagrams. If None, nothing will be written.
    @param config: the optional configuration.
    @return: a list of the taylor diagrams.
    """
    op = Output(config=config)
    t, diagrams = op.taylor(statistics, target_file)
    return diagrams
Exemplo n.º 6
0
def write_xhtml_report(statistics_list, matchups, data, target_file, taylor_target_files=None, target_diagram_file=None, density_plot_files=None, config=None):
    """
    Writes the xhtml report to the given target file.
    @param statistics_list: the list of statistics to mention in the report.
    @param matchups: the matchups the statistics have been calculated for.
    @param target_file: the target xhtml file.
    @param taylor_target_files: the optional paths to the taylor diagrams.
    @param target_diagram_file: the optional paths to the target diagram.
    @param density_plot_files: the optional paths to the density plots.
    @param config: the optional configuration.
    @return: None.
    """
    op = Output(config=config)
    op.xhtml(statistics_list, len(matchups), matchups, data, target_file, taylor_target_files, target_diagram_file, density_plot_files)
Exemplo n.º 7
0
def write_csv(statistics, model_name, ref_name, matchups, data, target_file=None, config=None):
    """
    Returns the given statistics as CSV string; writes this string if target_file is provided.
    @param model_name: the model variable name.
    @param ref_name: the reference variable name.
    @param statistics: a dictionary containing all statistics.
    @param matchups: the matchups.
    @param data: the input data file.
    @param target_file: the optional target file; if not None, result string will be written to that file.
    @param config: the optional configuration.
    @return: the given statistics as CSV string.
    """
    op = Output(config=config)
    op.csv(data, [(model_name, ref_name)], {(model_name, ref_name): statistics}, len(matchups), matchups=matchups, target_file=target_file)
Exemplo n.º 8
0
    def test_output_csv(self):
        chl_ref_chl = ('chl', 'Ref_chl')
        chl_ref2_chl = ('chl', 'Ref2_chl')
        sst_ref_sst = ('sst', 'sst_reference')
        sst_sst = ('sst', 'sst')
        mappings = [chl_ref_chl, chl_ref2_chl, sst_ref_sst, sst_sst]

        statistics = {}
        statistics[chl_ref_chl] = processor.calculate_statistics(np.array([11, 9, 11.2, 10.5]), np.array([10, 10, 10, 10]), 'chl', 'Ref_chl')
        statistics[chl_ref2_chl] = processor.calculate_statistics(np.array([12, 2, 3, 5]), np.array([2, 3, 4, 6]), 'chl', 'Ref2_chl')
        statistics[sst_ref_sst] = processor.calculate_statistics(np.array([8, 9, 15, 4]), np.array([6, 8, 2, 1]), 'sst', 'Ref_sst')
        statistics[sst_sst] = processor.calculate_statistics(np.array([8, 10, 2, 55]), np.array([99, 5, 5, 23]), 'sst', 'sst')

        output = Output()
        output.csv(mappings, statistics, 10957, matchups=None, target_file='c:\\temp\\output\\benchmark\\test.csv')
Exemplo n.º 9
0
def density_plot(model_name, ref_name, model_values, ref_values, axis_min=None, axis_max=None, target_file=None, unit=None, log_scaled=None, config=None):
    """
    Creates the density plot for the given matchups and variables and possible writes it to the given target file.
    @param model_name: the name of the model variable.
    @param ref_name: the name of the reference variable.
    @param model_name: the name of the model variable.
    @param ref_name: the name of the reference variable.
    @param target_file: the optional target diagram file. If None, nothing will be written.
    @param config: the optional configuration.
    @return: the density plot.
    """
    op = Output(config=config)
    if log_scaled is None and config is not None:
        log_scaled = config.density_plot_log_scaled
    elif log_scaled is None:
        log_scaled = False
    return op.density_plot(model_name, ref_name, model_values, ref_values, log_scaled, target_file, axis_min, axis_max, unit)
Exemplo n.º 10
0
def write_xhtml_report(statistics_list,
                       matchups,
                       data,
                       target_file,
                       taylor_target_files=None,
                       target_diagram_file=None,
                       density_plot_files=None,
                       config=None):
    """
    Writes the xhtml report to the given target file.
    @param statistics_list: the list of statistics to mention in the report.
    @param matchups: the matchups the statistics have been calculated for.
    @param target_file: the target xhtml file.
    @param taylor_target_files: the optional paths to the taylor diagrams.
    @param target_diagram_file: the optional paths to the target diagram.
    @param density_plot_files: the optional paths to the density plots.
    @param config: the optional configuration.
    @return: None.
    """
    op = Output(config=config)
    op.xhtml(statistics_list, len(matchups), matchups, data, target_file,
             taylor_target_files, target_diagram_file, density_plot_files)
Exemplo n.º 11
0
    def test_output_csv(self):
        chl_ref_chl = ('chl', 'Ref_chl')
        chl_ref2_chl = ('chl', 'Ref2_chl')
        sst_ref_sst = ('sst', 'sst_reference')
        sst_sst = ('sst', 'sst')
        mappings = [chl_ref_chl, chl_ref2_chl, sst_ref_sst, sst_sst]

        statistics = {}
        statistics[chl_ref_chl] = processor.calculate_statistics(
            np.array([11, 9, 11.2, 10.5]), np.array([10, 10, 10, 10]), 'chl',
            'Ref_chl')
        statistics[chl_ref2_chl] = processor.calculate_statistics(
            np.array([12, 2, 3, 5]), np.array([2, 3, 4, 6]), 'chl', 'Ref2_chl')
        statistics[sst_ref_sst] = processor.calculate_statistics(
            np.array([8, 9, 15, 4]), np.array([6, 8, 2, 1]), 'sst', 'Ref_sst')
        statistics[sst_sst] = processor.calculate_statistics(
            np.array([8, 10, 2, 55]), np.array([99, 5, 5, 23]), 'sst', 'sst')

        output = Output()
        output.csv(mappings,
                   statistics,
                   10957,
                   matchups=None,
                   target_file='c:\\temp\\output\\benchmark\\test.csv')
Exemplo n.º 12
0
def write_csv(statistics,
              model_name,
              ref_name,
              matchups,
              data,
              target_file=None,
              config=None):
    """
    Returns the given statistics as CSV string; writes this string if target_file is provided.
    @param model_name: the model variable name.
    @param ref_name: the reference variable name.
    @param statistics: a dictionary containing all statistics.
    @param matchups: the matchups.
    @param data: the input data file.
    @param target_file: the optional target file; if not None, result string will be written to that file.
    @param config: the optional configuration.
    @return: the given statistics as CSV string.
    """
    op = Output(config=config)
    op.csv(data, [(model_name, ref_name)],
           {(model_name, ref_name): statistics},
           len(matchups),
           matchups=matchups,
           target_file=target_file)
Exemplo n.º 13
0
def main():
    parsed_args = parse_arguments(sys.argv[1:])
    config = Configuration(properties_file_name=parsed_args.config, target_dir=parsed_args.output_dir,
                           target_prefix=parsed_args.prefix)
    file_handler = setup_logging(config)
    if parsed_args.reference_file is not None:
        data = Data(parsed_args.path, parsed_args.reference_file, config.max_cache_size)
    else:
        data = Data(parsed_args.path, max_cache_size=config.max_cache_size)

    output = Output(config=config)

    matchups = None
    if data.has_one_dim_ref_var():
        me = MatchupEngine(data, config)
        matchups = me.find_all_matchups()
        if not matchups:
            logging.warning('No matchups found. System will exit.')
            exit(0)
        if config.remove_empty_matchups:
            matchups = me.remove_empty_matchups(matchups)

    if not os.name == 'nt':
        logging.debug('Memory after matchups have been found: %s' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

    matchup_count = 0 if matchups is None else len(matchups)
    collected_statistics = {}
    density_plot_files = []
    target_files = []
    density_plots = {}

    for (model_name, ref_name) in parsed_args.variable_mappings:
        unit = data.unit(model_name)
        is_gridded = len(data.get_reference_dimensions(ref_name)) > 1
        if is_gridded:
            reference_values, model_values = data.get_values(ref_name, model_name)
            matchup_count += ma.count(reference_values)
        else:
            reference_values, model_values = utils.extract_values(matchups, data, ref_name, model_name)
            reference_values, model_values = utils.harmonise(reference_values, model_values)
            logging.debug('Compressing ref-variable %s' % ref_name)
            reference_values = reference_values.compressed()
            logging.debug('Compressing model-variable %s' % model_name)
            model_values = model_values.compressed()

        logging.info('Calculating statistics for \'%s\' with \'%s\'' % (model_name, ref_name))
        stats = processor.calculate_statistics(model_values, reference_values, model_name, ref_name, unit, config)
        collected_statistics[(model_name, ref_name)] = stats

        if config.write_density_plots:
            axis_min = min(stats['min'], stats['ref_min'])
            axis_max = max(stats['p90'], stats['ref_p90'])
            logging.info('Creating density plot for \'%s\' and \'%s\'' % (model_name, ref_name))
            density_plots[model_name + ref_name] = output.density_plot(model_name, ref_name, model_values,
                                                                       reference_values, config.density_plot_log_scaled,
                                                                       None, axis_min, axis_max, data.unit(model_name))

    if not os.name == 'nt':
        logging.debug(
            'Memory after statistics have been computed: %s' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

    if config.write_csv:
        csv_target_file = '%s/%sstatistics.csv' % (parsed_args.output_dir, config.target_prefix)
        target_files.append(csv_target_file)
        output.csv(data, parsed_args.variable_mappings, collected_statistics, matchup_count, matchups=matchups, source_file=parsed_args.path, target_file=csv_target_file)
        logging.info('CSV output written to \'%s\'' % csv_target_file)
        if matchups is not None:
            matchup_filename = '%s_matchups.csv' % os.path.splitext(csv_target_file)[0]
            logging.info('Matchups written to \'%s\'' % matchup_filename)
            target_files.append(matchup_filename)

    taylor_target_files = []
    if config.write_taylor_diagrams:
        taylor_target_file = '%s/%staylor.png' % (parsed_args.output_dir, config.target_prefix)
        written_taylor_diagrams, d = output.taylor(list(collected_statistics.values()), taylor_target_file)
        del d
        if written_taylor_diagrams:
            for written_taylor_diagram in written_taylor_diagrams:
                logging.info('Taylor diagram written to \'%s\'' % written_taylor_diagram)
                target_files.append(written_taylor_diagram)
                taylor_target_files.append(written_taylor_diagram)

    if config.write_density_plots:
        for (model_name, ref_name) in parsed_args.variable_mappings:
            density_target = '%s/density-%s-%s.png' % (parsed_args.output_dir, model_name, ref_name)
            density_plot_files.append(density_target)
            target_files.append(density_target)
            output.write_density_plot(density_plots[model_name + ref_name], density_target)
            logging.info('Density plot written to \'%s\'' % density_target)

    target_diagram_file = None
    if config.write_target_diagram:
        target_diagram_file = '%s/%starget.png' % (parsed_args.output_dir, config.target_prefix)
        output.target_diagram(list(collected_statistics.values()), target_diagram_file)
        logging.info('Target diagram written to \'%s\'' % target_diagram_file)
        target_files.append(target_diagram_file)

    if config.write_xhtml:
        xml_target_file = '%s/%sreport.xml' % (parsed_args.output_dir, config.target_prefix)
        path = str(os.path.dirname(os.path.realpath(__file__))) + '/../resources/'
        xsl = path + 'analysis-summary.xsl'
        css = path + 'styleset.css'
        xsl_target = '%s/%s' % (parsed_args.output_dir, os.path.basename(xsl))
        css_target = '%s/%s' % (parsed_args.output_dir, os.path.basename(css))
        output.xhtml(list(collected_statistics.values()), matchup_count, matchups, data, xml_target_file, taylor_target_files,
                     target_diagram_file, density_plot_files)
        logging.info('XHTML report written to \'%s\'' % xml_target_file)
        shutil.copy(xsl, parsed_args.output_dir)
        logging.info('XHTML support file written to \'%s/%s\'' % (parsed_args.output_dir, 'analysis-summary.xsl'))
        shutil.copy(css, parsed_args.output_dir)
        logging.info('XHTML support file written to \'%s/%s\'' % (parsed_args.output_dir, 'styleset.xsl'))
        target_files.append(xml_target_file)
        target_files.append(xsl_target)
        target_files.append(css_target)

    if config.zip:
        create_zip(target_files, config, file_handler, parsed_args)

    logging.info('End of process')