def wrap_batch_physical_sim_run(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the physical sim processing
    """
    params_init = {"simkey": "point to the sims to run",
                   "spec_ini": "ini file for the spectral estimation",
                   "output_tag": "tag identifying the output somehow"}
    prefix="csp_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['simkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    datapath_db = data_paths.DataPath()

    return batch_physical_sim_run(params["simkey"],
                                  inifile=params["spec_ini"],
                                  datapath_db=datapath_db,
                                  outdir=output_root,
                                  output_tag=output_tag)
def wrap_batch_physical_sim_run(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the physical sim processing
    """
    params_init = {
        "simkey": "point to the sims to run",
        "spec_ini": "ini file for the spectral estimation",
        "output_tag": "tag identifying the output somehow"
    }
    prefix = "csp_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['simkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)

    datapath_db = data_paths.DataPath()

    return batch_physical_sim_run(params["simkey"],
                                  inifile=params["spec_ini"],
                                  datapath_db=datapath_db,
                                  outdir=output_root,
                                  output_tag=output_tag)
def wrap_batch_single_crosspwr(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the single crosspwr calculator
    """
    params_init = {"left_mapkey": "some preparation of a map, cleaned",
                   "right_simkey": "a simulation to cross it with",
                   "right_weightkey": "weight to use for that sim",
                   "multiplier": "multiply the 1D and 2D spectra",
                   "spec_ini": "ini file for the spectral estimation",
                   "output_tag": "tag identifying the output somehow"}
    prefix="csc_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['left_mapkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root, output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    datapath_db = data_paths.DataPath()

    return batch_single_crosspwr(params["left_mapkey"],
                                 params["right_simkey"],
                                 params["right_weightkey"],
                                 multiplier=params["multiplier"],
                                 inifile=params["spec_ini"],
                                 datapath_db=datapath_db,
                                 outdir=output_root,
                                 output_tag=output_tag)
def batch_physical_sim_run(sim_key,
                           inifile=None,
                           datapath_db=None,
                           output_tag=None,
                           outdir="./plots/"):
    """Test the power spectral estimator using simulations"""
    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    mock_cases = datapath_db.fileset_cases(sim_key, "realization")

    funcname = "correlate.batch_quadratic.call_phys_space_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname,
                                        cache_path,
                                        generate=generate,
                                        verbose=True)

    if output_tag:
        output_root = "%s/%s/" % (outdir, output_tag)
        file_tools.mkparents(output_root)

    pwr_1d = []
    pwr_1d_from_2d = []
    pwr_2d = []
    for index in mock_cases['realization']:
        mapfile = datapath_db.fetch("%s:%s" % (sim_key, index))

        pwrspec_out = caller.execute(mapfile, mapfile, inifile=inifile)

        if output_tag:
            pwr_1d_from_2d.append(
                pe.convert_2d_to_1d(pwrspec_out[0], transfer=None))

            pwr_2d.append(pwrspec_out[0])
            pwr_1d.append(pwrspec_out[1])

    if output_tag:
        pe.summarize_agg_pwrspec(pwr_1d,
                                 pwr_1d_from_2d,
                                 pwr_2d,
                                 output_tag,
                                 outdir=output_root)

        retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
    else:
        caller.multiprocess_stack()
        retval = None

    return retval
def wrap_batch_gbtxwigglez_data_run(inifile, generate=False,
                                    outdir="./plots/"):
    r"""Wrapper to the GBT x WiggleZ calculation"""
    params_init = {"gbt_mapkey": "cleaned GBT map",
                   "wigglez_deltakey": "WiggleZ overdensity map",
                   "wigglez_mockkey": "WiggleZ overdensities from mocks",
                   "wigglez_selectionkey": "WiggleZ selection function",
                   "mode_transfer_1d_ini": "ini file -> 1d trans. function",
                   "mode_transfer_2d_ini": "ini file -> 2d trans. function",
                   "beam_transfer_ini": "ini file -> 2d beam trans. function",
                   "spec_ini": "ini file for the spectral estimation",
                   "output_tag": "tag identifying the output somehow"}
    prefix = "cwx_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['gbt_mapkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    datapath_db = data_paths.DataPath()

    mode_transfer_1d = None
    if params["mode_transfer_1d_ini"]:
        mode_transfer_1d = cct.wrap_batch_crosspwr_transfer(
                                            params["mode_transfer_1d_ini"],
                                            generate=generate,
                                            outdir=outdir)

    batch_gbtxwigglez_data_run(params["gbt_mapkey"],
                               params["wigglez_deltakey"],
                               params["wigglez_mockkey"],
                               params["wigglez_selectionkey"],
                               inifile=params["spec_ini"],
                               datapath_db=datapath_db,
                               outdir=output_root,
                               output_tag=output_tag,
                               beam_transfer=None,
                               mode_transfer_1d=mode_transfer_1d,
                               mode_transfer_2d=None,
                               theory_curve=None)
示例#6
0
def wrap_batch_gbtxwigglez_data_run(inifile,
                                    generate=False,
                                    outdir="./plots/"):
    r"""Wrapper to the GBT x WiggleZ calculation"""
    params_init = {
        "gbt_mapkey": "cleaned GBT map",
        "wigglez_deltakey": "WiggleZ overdensity map",
        "wigglez_mockkey": "WiggleZ overdensities from mocks",
        "wigglez_selectionkey": "WiggleZ selection function",
        "mode_transfer_1d_ini": "ini file -> 1d trans. function",
        "mode_transfer_2d_ini": "ini file -> 2d trans. function",
        "beam_transfer_ini": "ini file -> 2d beam trans. function",
        "spec_ini": "ini file for the spectral estimation",
        "output_tag": "tag identifying the output somehow"
    }
    prefix = "cwx_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['gbt_mapkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)

    datapath_db = data_paths.DataPath()

    mode_transfer_1d = None
    if params["mode_transfer_1d_ini"]:
        mode_transfer_1d = cct.wrap_batch_crosspwr_transfer(
            params["mode_transfer_1d_ini"], generate=generate, outdir=outdir)

    batch_gbtxwigglez_data_run(params["gbt_mapkey"],
                               params["wigglez_deltakey"],
                               params["wigglez_mockkey"],
                               params["wigglez_selectionkey"],
                               inifile=params["spec_ini"],
                               datapath_db=datapath_db,
                               outdir=output_root,
                               output_tag=output_tag,
                               beam_transfer=None,
                               mode_transfer_1d=mode_transfer_1d,
                               mode_transfer_2d=None,
                               theory_curve=None)
def write_map_cleanerini(mapname, cutlist, nfreq, factorizable=True, meansub=True,
                         regenerate=False, convolve=False,
                         subtract_inputmap_from_sim = True,
                         subtract_sim_from_inputmap = False,
                         sim_multiplier = 1., username="******",
                         modes = range(0, 105, 5), simfile=None, prefix="fs_",
                         inidir="./input/ers/map_cleaning_autogen/"):
    file_tools.mkparents(inidir)
    params = {}

    simtag = ""
    if simfile:
        simtag = "_plussim"
        if subtract_inputmap_from_sim:
            simtag = "_plussim_minusmap"
        if subtract_sim_from_inputmap:
            simtag = "_plussim_minussim"

    alt = ""
    if sim_multiplier != 1.:
        multstring = "%5.3g" % sim_multiplier
        alt = "_simx" + multstring.replace(".", "p").strip()

    key = '%s_cleaned%s%s' % (mapname, simtag, alt)
    params["output_root"] = '%s_path_%s' % (key, username)

    params["SVD_root"] = None
    params["modes"] = modes
    params["map1"] = mapname
    params["map2"] = mapname
    params["noise_inv1"] = mapname
    params["noise_inv2"] = mapname
    params["no_weights"] = False
    params["sub_weighted_mean"] = meansub
    params["factorizable_noise"] = factorizable
    params["convolve"] = convolve
    params["regenerate_noise_inv"] = regenerate
    params["freq_list"] = tuple([ind for ind in range(nfreq) \
                                 if ind not in cutlist])

    if simfile:
        params["simfile"] = simfile
        params["sim_multiplier"] = sim_multiplier
        params["subtract_inputmap_from_sim"] = subtract_inputmap_from_sim
        params["subtract_sim_from_inputmap"] = subtract_sim_from_inputmap

    filename = "%s/%s.ini" % (inidir, params["output_root"])
    print filename
    parse_ini.write_params(params, filename, prefix=prefix)
def batch_physical_sim_run(sim_key, inifile=None, datapath_db=None,
                           output_tag=None,
                           outdir="./plots/"):
    """Test the power spectral estimator using simulations"""
    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    mock_cases = datapath_db.fileset_cases(sim_key, "realization")

    funcname = "correlate.batch_quadratic.call_phys_space_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname, cache_path,
                                        generate=generate, verbose=True)

    if output_tag:
        output_root = "%s/%s/" % (outdir, output_tag)
        file_tools.mkparents(output_root)

    pwr_1d = []
    pwr_1d_from_2d = []
    pwr_2d = []
    for index in mock_cases['realization']:
        mapfile = datapath_db.fetch("%s:%s" % (sim_key, index))

        pwrspec_out = caller.execute(mapfile, mapfile, inifile=inifile)

        if output_tag:
            pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
                                                      transfer=None))

            pwr_2d.append(pwrspec_out[0])
            pwr_1d.append(pwrspec_out[1])

    if output_tag:
        pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d,
                                 output_tag, outdir=output_root)

        retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
    else:
        caller.multiprocess_stack()
        retval = None

    return retval
def wrap_batch_gbtpwrspec_data_run(inifile, generate=False,
                                    outdir="./plots/"):
    r"""Wrapper to the GBT x GBT calculation"""
    params_init = {"gbt_mapkey": "cleaned GBT map",
                   "mode_transfer_1d_ini": "ini file -> 1d trans. function",
                   "mode_transfer_2d_ini": "ini file -> 2d trans. function",
                   "beam_transfer_ini": "ini file -> 2d beam trans. function",
                   "square_1dmodetrans": False,
                   "spec_ini": "ini file for the spectral estimation",
                   "output_tag": "tag identifying the output somehow"}
    prefix="cp_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['gbt_mapkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    datapath_db = data_paths.DataPath()

    mode_transfer_1d=None
    if params["mode_transfer_1d_ini"]:
        mode_transfer_1d = cct.wrap_batch_crosspwr_transfer(
                                            params["mode_transfer_1d_ini"],
                                            generate=generate,
                                            outdir=outdir)

    return batch_gbtpwrspec_data_run(params["gbt_mapkey"],
                         inifile=params["spec_ini"],
                         datapath_db=datapath_db,
                         outdir=output_root,
                         output_tag=output_tag,
                         beam_transfer=None,
                         square_1dmodetrans = params["square_1dmodetrans"],
                         mode_transfer_1d=mode_transfer_1d,
                         mode_transfer_2d=None)
示例#10
0
def wrap_batch_gbtpwrspec_data_run(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the GBT x GBT calculation"""
    params_init = {
        "gbt_mapkey": "cleaned GBT map",
        "mode_transfer_1d_ini": "ini file -> 1d trans. function",
        "mode_transfer_2d_ini": "ini file -> 2d trans. function",
        "beam_transfer_ini": "ini file -> 2d beam trans. function",
        "square_1dmodetrans": False,
        "spec_ini": "ini file for the spectral estimation",
        "output_tag": "tag identifying the output somehow"
    }
    prefix = "cp_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['gbt_mapkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)

    datapath_db = data_paths.DataPath()

    mode_transfer_1d = None
    if params["mode_transfer_1d_ini"]:
        mode_transfer_1d = cct.wrap_batch_crosspwr_transfer(
            params["mode_transfer_1d_ini"], generate=generate, outdir=outdir)

    return batch_gbtpwrspec_data_run(
        params["gbt_mapkey"],
        inifile=params["spec_ini"],
        datapath_db=datapath_db,
        outdir=output_root,
        output_tag=output_tag,
        beam_transfer=None,
        square_1dmodetrans=params["square_1dmodetrans"],
        mode_transfer_1d=mode_transfer_1d,
        mode_transfer_2d=None)
示例#11
0
def wrap_batch_sim_run(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the sim processing
    TODO: add transfer function to test its application
    """
    params_init = {"left_simkey": "sim to put on the left side",
                   "right_simkey": "sim to put on the right side",
                   "left_weightkey": "weight to use on the left side",
                   "right_weightkey": "weight to use on the right side",
                   "spec_ini": "ini file for the spectral estimation",
                   "output_tag": "tag identifying the output somehow"}
    prefix="csp_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%sx%s_%s" % (params['left_simkey'], \
                               params['right_simkey'], \
                               params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    datapath_db = data_paths.DataPath()

    return batch_sim_run(params["left_simkey"],
                         params["right_simkey"],
                         params["left_weightkey"],
                         params["right_weightkey"],
                         inifile=params["spec_ini"],
                         datapath_db=datapath_db,
                         outdir=output_root,
                         output_tag=output_tag)
def write_map_cleanerini_old(mapname, cutlist, nfreq, factorizable=True, meansub=True,
                         regenerate=False, noconv=False,
                         subtract_inputmap_from_sim = True,
                         subtract_sim_from_inputmap = False,
                         modes = range(0, 105, 5), simfile=None, prefix="fs_",
                         inidir="./input/ers/map_cleaning_autogen_old/"):
    file_tools.mkparents(inidir)

    params = {}
    params["SVD_root"] = None
    params["modes"] = modes
    params["map1"] = mapname
    params["map2"] = mapname
    params["noise_inv1"] = mapname
    params["noise_inv2"] = mapname
    params["no_weights"] = False
    params["sub_weighted_mean"] = meansub
    params["factorizable_noise"] = factorizable
    params["regenerate_noise_inv"] = regenerate
    params["freq_list"] = tuple([ind for ind in range(nfreq) \
                                 if ind not in cutlist])

    tag = "_sims" if simfile else ""
    if simfile:
        params["simfile"] = simfile
        params["sim_multiplier"] = 1.
        params["subtract_inputmap_from_sim"] = subtract_inputmap_from_sim
        params["subtract_sim_from_inputmap"] = subtract_sim_from_inputmap

    params["convolve"] = False
    # TODO: move this to direct path rather than db
    params["output_root"] = "%s_cleaned%s_noconv_path_Eric" % (mapname, tag)
    filename = "%s/%s_cleaned%s_noconv.ini" % (inidir, mapname, tag)
    parse_ini.write_params(params, filename, prefix=prefix)

    params["convolve"] = True
    params["output_root"] = "%s_cleaned%s_path_Eric" % (mapname, tag)
    filename = "%s/%s_cleaned%s.ini" % (inidir, mapname, tag)
    parse_ini.write_params(params, filename, prefix=prefix)
def wrap_batch_crosspwr_transfer(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the transfer function calculator
    """
    params_init = {
        "cleaned_simkey": "cleaned sims for transfer func",
        "truesignal_simkey": "pure signal",
        "truesignal_weightkey": "weight to use for pure signal",
        "reference_simkey": "reference signal",
        "reference_weightkey": "weight to use for reference signal",
        "spec_ini": "ini file for the spectral estimation",
        "output_tag": "tag identifying the output somehow"
    }
    prefix = "cct_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['cleaned_simkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)

    datapath_db = data_paths.DataPath()

    return batch_crosspwr_transfer(params["cleaned_simkey"],
                                   params["truesignal_simkey"],
                                   params["truesignal_weightkey"],
                                   params["reference_simkey"],
                                   params["reference_weightkey"],
                                   inifile=params["spec_ini"],
                                   datapath_db=datapath_db,
                                   outdir=output_root,
                                   output_tag=output_tag)
def wrap_batch_crosspwr_transfer(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the transfer function calculator
    """
    params_init = {"cleaned_simkey": "cleaned sims for transfer func",
                   "truesignal_simkey": "pure signal",
                   "truesignal_weightkey": "weight to use for pure signal",
                   "reference_simkey": "reference signal",
                   "reference_weightkey": "weight to use for reference signal",
                   "spec_ini": "ini file for the spectral estimation",
                   "output_tag": "tag identifying the output somehow"}
    prefix="cct_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['cleaned_simkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root
    print output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    datapath_db = data_paths.DataPath()

    return batch_crosspwr_transfer(params["cleaned_simkey"],
                                   params["truesignal_simkey"],
                                   params["truesignal_weightkey"],
                                   params["reference_simkey"],
                                   params["reference_weightkey"],
                                   inifile=params["spec_ini"],
                                   datapath_db=datapath_db,
                                   outdir=output_root,
                                   output_tag=output_tag)
示例#15
0
def wrap_batch_single_crosspwr(inifile, generate=False, outdir="./plots/"):
    r"""Wrapper to the single crosspwr calculator
    """
    params_init = {
        "left_mapkey": "some preparation of a map, cleaned",
        "right_simkey": "a simulation to cross it with",
        "right_weightkey": "weight to use for that sim",
        "multiplier": "multiply the 1D and 2D spectra",
        "spec_ini": "ini file for the spectral estimation",
        "output_tag": "tag identifying the output somehow"
    }
    prefix = "csc_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_tag = "%s_%s" % (params['left_mapkey'], params['output_tag'])
    output_root = "%s/%s/" % (outdir, output_tag)

    if generate:
        output_tag = None

    print output_root, output_tag
    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)

    datapath_db = data_paths.DataPath()

    return batch_single_crosspwr(params["left_mapkey"],
                                 params["right_simkey"],
                                 params["right_weightkey"],
                                 multiplier=params["multiplier"],
                                 inifile=params["spec_ini"],
                                 datapath_db=datapath_db,
                                 outdir=output_root,
                                 output_tag=output_tag)
示例#16
0
    'output_root': '',
    'sim_physical_key': '',
    'sim_key': '',
    'sim_beam_key': '',
    'sim_beam_plus_fg_key': '',
    'sim_beam_plus_data_key': '',
    'sim_delta_key': '',
    'sim_beam_meansub_key': '',
    'sim_beam_conv_key': '',
    'sim_beam_meansubconv_key': '',
    'template_key': '',
    'weight_key': '',
    'pwrspec_scenario': '',
    'refinement': 2,
    'omega_HI': ''
}
prefix = 'sg_'

if __name__ == '__main__':
    params = parse_ini.parse(str(sys.argv[1]), params_init, prefix=prefix)
    print params

    datapath_db = data_paths.DataPath()
    output_root = datapath_db.fetch(params['output_root'])

    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)

    generate_sim(params, parallel=True, datapath_db=datapath_db)
    generate_aux_simset(params, datapath_db=datapath_db)
示例#17
0
def batch_sim_run(simleft_key, simright_key,
                  weightleft_key, weightright_key,
                  inifile=None, datapath_db=None,
                  outdir="./plots/",
                  usecache_output_tag=None, transfer=None):
    r"""
    typical weight matrix:
    db:GBT_15hr_map_cleaned_0mode:A_with_B;noise_inv
    """

    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    mock_cases = datapath_db.fileset_cases(simleft_key, "realization")

    funcname = "correlate.batch_quadratic.call_xspec_run"
    generate = False if usecache_output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname, cache_path,
                                        generate=generate, verbose=True)

    if usecache_output_tag:
        output_root = "%s/%s/" % (outdir, usecache_output_tag)
        file_tools.mkparents(output_root)

    pwr_1d = []
    pwr_1d_from_2d = []
    pwr_2d = []
    for index in mock_cases['realization']:
        dbkeydict = {}
        dbkeydict['map1_key'] = "%s:%s" % (simleft_key, index)
        dbkeydict['map2_key'] = "%s:%s" % (simright_key, index)
        dbkeydict['noiseinv1_key'] = weightleft_key
        dbkeydict['noiseinv2_key'] = weightright_key
        files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
                                                    datapath_db=datapath_db)

        pwrspec_out = caller.execute(files['map1_key'], files['map2_key'],
                                     files['noiseinv1_key'],
                                     files['noiseinv2_key'],
                                     inifile=inifile)

        if usecache_output_tag:
            pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
                                                  transfer=transfer))

            pwr_2d.append(pwrspec_out[0])
            pwr_1d.append(pwrspec_out[1])

    if usecache_output_tag:
        pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d,
                                 usecache_output_tag, outdir=output_root)
        retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
    else:
        caller.multiprocess_stack()
        retval = None

    return retval
def batch_crosspwr_transfer(cleaned_simkey,
                            truesignal_simkey, truesignal_weightkey,
                            reference_simkey, reference_weightkey,
                            inifile=None, datapath_db=None,
                            outdir="./plots",
                            output_tag=None):
    r"""take relevant cross-powers
    cleaned_simkey(map) * cleaned_simkey(weight) x
    truesignal_weightkey * truesignal_simkey
    divided by:
    reference_simkey * reference_weightkey x truesignal_weightkey * truesignal_simkey
    """

    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    map_cases = datapath_db.fileset_cases(cleaned_simkey, "type;treatment")

    funcname = "correlate.batch_quadratic.call_xspec_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname, cache_path,
                                        generate=generate, verbose=True)

    dbkeydict = {}
    dbkeydict['map1_key'] = reference_simkey
    dbkeydict['map2_key'] = truesignal_simkey
    dbkeydict['noiseinv1_key'] = reference_weightkey
    dbkeydict['noiseinv2_key'] = truesignal_weightkey
    files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
                                                     datapath_db=datapath_db)

    reference_pwrspec_out = caller.execute(files['map1_key'],
                                        files['map2_key'],
                                        files['noiseinv1_key'],
                                        files['noiseinv2_key'],
                                        inifile=inifile)

    if output_tag:
        file_tools.mkparents(outdir)
        ref_pwr_1d_from_2d = pe.convert_2d_to_1d(reference_pwrspec_out[0])

        ref_pwr_1d = reference_pwrspec_out[1]['binavg']
        ref_pwr_2d = reference_pwrspec_out[0]['binavg']
        ref_pwr_1d_from_2d = ref_pwr_1d_from_2d['binavg']

        bin_left = reference_pwrspec_out[1]['bin_left']
        bin_center = reference_pwrspec_out[1]['bin_center']
        bin_right = reference_pwrspec_out[1]['bin_right']
        counts_histo = reference_pwrspec_out[1]['counts_histo']

    transfer_functions = {}
    for treatment in map_cases['treatment']:
        dbkeydict = {}
        dbkeydict['map1_key'] = "%s:map;%s" % (cleaned_simkey, treatment)
        dbkeydict['map2_key'] = truesignal_simkey
        dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (cleaned_simkey, treatment)
        dbkeydict['noiseinv2_key'] = truesignal_weightkey
        files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
                                                      datapath_db=datapath_db)

        pwrspec_out_signal = caller.execute(files['map1_key'],
                                            files['map2_key'],
                                            files['noiseinv1_key'],
                                            files['noiseinv2_key'],
                                            inifile=inifile)

        if output_tag:
            pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])

            pwr_1d = pwrspec_out_signal[1]['binavg']
            pwr_2d = pwrspec_out_signal[0]['binavg']
            pwr_1d_from_2d = pwr_1d_from_2d['binavg']

            trans1d_mode = pwr_1d / ref_pwr_1d
            trans1d_from2d_mode = pwr_1d_from_2d / ref_pwr_1d_from_2d
            trans2d_mode = pwr_2d / ref_pwr_2d

            transfer_functions[treatment] = (trans1d_mode,
                                             trans1d_from2d_mode,
                                             trans2d_mode)

            filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)

            outfile = open(filename, "w")
            for specdata in zip(bin_left, bin_center,
                                bin_right, counts_histo,
                                pwr_1d, pwr_1d_from_2d,
                                trans1d_mode, trans1d_from2d_mode):
                outfile.write(("%10.15g " * 8 + "\n") % specdata)
            outfile.close()

    if not output_tag:
        caller.multiprocess_stack()
        return None
    else:
        return transfer_functions
def batch_gbtxwigglez_data_run(gbt_map_key, wigglez_map_key,
                               wigglez_mock_key, wigglez_selection_key,
                               inifile=None, datapath_db=None,
                               outdir="./plots",
                               output_tag=None,
                               beam_transfer=None,
                               mode_transfer_1d=None,
                               mode_transfer_2d=None,
                               theory_curve=None):
    r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""

    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
    mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")

    funcname = "correlate.batch_quadratic.call_xspec_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname, cache_path,
                                        generate=generate, verbose=True)

    if output_tag:
        file_tools.mkparents(outdir)

    for treatment in map_cases['treatment']:
        # TODO: make this more elegant
        # TODO: convert treatment into mode num
        transfer_2d = None
        if (mode_transfer_2d is not None) and (beam_transfer is None):
            transfer_2d = mode_transfer_2d[treatment][2]

        if (mode_transfer_2d is None) and (beam_transfer is not None):
            transfer_2d = beam_transfer

        if (mode_transfer_2d is not None) and (beam_transfer is not None):
            transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer

        pwr_1d = []
        pwr_2d = []
        pwr_1d_from_2d = []
        for index in mock_cases['realization']:
            dbkeydict = {}
            dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
            dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
            dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
                                         (gbt_map_key, treatment)

            dbkeydict['noiseinv2_key'] = wigglez_selection_key
            files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
                                                      datapath_db=datapath_db)

            pwrspec_out = caller.execute(files['map1_key'], files['map2_key'],
                                         files['noiseinv1_key'],
                                         files['noiseinv2_key'],
                                         inifile=inifile)

            if output_tag:
                pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
                                      transfer=transfer_2d))

                pwr_2d.append(pwrspec_out[0])
                pwr_1d.append(pwrspec_out[1])

        if output_tag:
            if mode_transfer_1d is not None:
                transfunc = mode_transfer_1d[treatment][1]
            else:
                transfunc = None

            mtag = output_tag + "_%s_mock" % treatment
            agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
                              pwr_1d_from_2d, pwr_2d, mtag,
                              outdir=outdir,
                              apply_1d_transfer=transfunc)

            mean1dmock = agg_pwrspec["mean_1d"]
            std1dmock = agg_pwrspec["std_1d"]
            covmock = agg_pwrspec["covmat_1d"]

        # now recover the xspec with the real data
        dbkeydict = {}
        dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
        dbkeydict['map2_key'] = wigglez_map_key
        dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
        dbkeydict['noiseinv2_key'] = wigglez_selection_key
        files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
                                                      datapath_db=datapath_db)

        pwrspec_out_signal = caller.execute(files['map1_key'],
                                            files['map2_key'],
                                            files['noiseinv1_key'],
                                            files['noiseinv2_key'],
                                            inifile=inifile)

        if output_tag:
            pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
                                                 transfer=transfer_2d)

            pwr_1d = pwrspec_out_signal[1]['binavg']
            pwr_1d_from_2d = pwr_1d_from_2d['binavg']
            if mode_transfer_1d is not None:
                pwr_1d /= mode_transfer_1d[treatment][1]
                pwr_1d_from_2d /= mode_transfer_1d[treatment][1]

            # assume that they all have the same binning
            bin_left = pwrspec_out_signal[1]['bin_left']
            bin_center = pwrspec_out_signal[1]['bin_center']
            bin_right = pwrspec_out_signal[1]['bin_right']
            counts_histo = pwrspec_out_signal[1]['counts_histo']

            filename = "%s/%s_%s.dat" % (outdir,
                                         output_tag,
                                         treatment)

            outfile = open(filename, "w")
            for specdata in zip(bin_left, bin_center,
                                bin_right, counts_histo,
                                pwr_1d, pwr_1d_from_2d,
                                mean1dmock, std1dmock):
                outfile.write(("%10.15g " * 8 + "\n") % specdata)
            outfile.close()

            # TODO: kludge to make a fast fit; remove
            theory_curve = np.genfromtxt("plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat")
            theory_curve = theory_curve[:, 4]

            if theory_curve is not None:
                restrict = np.where(np.logical_and(bin_center > 0.09,
                                                   bin_center < 1.1))
                res_slice = slice(min(restrict[0]), max(restrict[0]))

                #restrict_alt = np.where(restrict)[0][np.newaxis, :]
                #restricted_cov = covmock[restrict_alt][0]

                from core import utils
                amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
                                         covmock[res_slice, res_slice],
                                         theory_curve[res_slice])
                print "AMP:", mtag, treatment, amplitude

    if not output_tag:
        caller.multiprocess_stack()

    return None
def spectrum_arithmetic(inifile, outdir="./plots/"):
    r"""Perform the operations"""
    params_init = {"pwr_a_ini": None,
                   "pwr_b_ini": None,
                   "pwr_c_ini": None,
                   "pwr_d_ini": None,
                   "mul_a": "1.",
                   "mul_b": "1.",
                   "mul_c": "1.",
                   "mul_d": "1.",
                   "output_tag": "tag identifying the output somehow"}
    prefix="sa_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_root = "%s/%s/" % (outdir, params["output_tag"])
    print output_root

    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    if not params["pwr_a_ini"]:
        print "ERROR: at least spectrum A must be given (and C for ratios)"
        return

    pwr_a_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_a_ini"],
                                                  generate=False,
                                                  outdir=outdir)

    if params['pwr_b_ini']:
        pwr_b_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_b_ini"],
                                                      generate=False,
                                                      outdir=outdir)

    if params['pwr_c_ini']:
        pwr_c_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_c_ini"],
                                                      generate=False,
                                                      outdir=outdir)

    if params['pwr_d_ini']:
        pwr_d_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_d_ini"],
                                                      generate=False,
                                                      outdir=outdir)

    for treatment in pwr_a_all:
        pwr_a = pwr_a_all[treatment]
        pwr_a = mul_pwrspec(pwr_a, params['mul_a'])
        pwr_numerator = copy.deepcopy(pwr_a)

        if params['pwr_b_ini']:
            pwr_b = pwr_b_all[treatment]
            pwr_b = mul_pwrspec(pwr_b, params['mul_b'])
            pwr_numerator = add_pwrspec(pwr_numerator, pwr_b)

        if params['pwr_c_ini']:
            pwr_c = pwr_c_all[treatment]
            pwr_c = mul_pwrspec(pwr_c, params['mul_c'])
            pwr_denominator = copy.deepcopy(pwr_c)

            if params['pwr_d_ini']:
                pwr_d = pwr_d_all[treatment]
                pwr_d = mul_pwrspec(pwr_d, params['mul_d'])
                pwr_denominator = add_pwrspec(pwr_denominator, pwr_d)

            pwr_final = divide_pwrspec(pwr_numerator, pwr_denominator)
        else:
            pwr_final = pwr_numerator

        filename = "%s/%s_%s.dat" % (output_root, params['output_tag'], treatment)
        outfile = open(filename, "w")
        for specdata in zip(pwr_final['bin_left'], pwr_final['bin_center'],
                            pwr_final['bin_right'], pwr_final['counts_histo'],
                            pwr_final['mean_1d'], pwr_final['std_1d']):
            outfile.write(("%10.15g " * 6 + "\n") % specdata)

        outfile.close()
示例#21
0
def spectrum_arithmetic(inifile, outdir="./plots/"):
    r"""Perform the operations"""
    params_init = {
        "pwr_a_ini": None,
        "pwr_b_ini": None,
        "pwr_c_ini": None,
        "pwr_d_ini": None,
        "mul_a": "1.",
        "mul_b": "1.",
        "mul_c": "1.",
        "mul_d": "1.",
        "output_tag": "tag identifying the output somehow"
    }
    prefix = "sa_"

    params = parse_ini.parse(inifile, params_init, prefix=prefix)
    print params

    output_root = "%s/%s/" % (outdir, params["output_tag"])
    print output_root

    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)

    if not params["pwr_a_ini"]:
        print "ERROR: at least spectrum A must be given (and C for ratios)"
        return

    pwr_a_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_a_ini"],
                                                  generate=False,
                                                  outdir=outdir)

    if params['pwr_b_ini']:
        pwr_b_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_b_ini"],
                                                      generate=False,
                                                      outdir=outdir)

    if params['pwr_c_ini']:
        pwr_c_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_c_ini"],
                                                      generate=False,
                                                      outdir=outdir)

    if params['pwr_d_ini']:
        pwr_d_all = cp.wrap_batch_gbtpwrspec_data_run(params["pwr_d_ini"],
                                                      generate=False,
                                                      outdir=outdir)

    for treatment in pwr_a_all:
        pwr_a = pwr_a_all[treatment]
        pwr_a = mul_pwrspec(pwr_a, params['mul_a'])
        pwr_numerator = copy.deepcopy(pwr_a)

        if params['pwr_b_ini']:
            pwr_b = pwr_b_all[treatment]
            pwr_b = mul_pwrspec(pwr_b, params['mul_b'])
            pwr_numerator = add_pwrspec(pwr_numerator, pwr_b)

        if params['pwr_c_ini']:
            pwr_c = pwr_c_all[treatment]
            pwr_c = mul_pwrspec(pwr_c, params['mul_c'])
            pwr_denominator = copy.deepcopy(pwr_c)

            if params['pwr_d_ini']:
                pwr_d = pwr_d_all[treatment]
                pwr_d = mul_pwrspec(pwr_d, params['mul_d'])
                pwr_denominator = add_pwrspec(pwr_denominator, pwr_d)

            pwr_final = divide_pwrspec(pwr_numerator, pwr_denominator)
        else:
            pwr_final = pwr_numerator

        filename = "%s/%s_%s.dat" % (output_root, params['output_tag'],
                                     treatment)
        outfile = open(filename, "w")
        for specdata in zip(pwr_final['bin_left'], pwr_final['bin_center'],
                            pwr_final['bin_right'], pwr_final['counts_histo'],
                            pwr_final['mean_1d'], pwr_final['std_1d']):
            outfile.write(("%10.15g " * 6 + "\n") % specdata)

        outfile.close()
示例#22
0
               'sim_beam_key': '',
               'sim_beam_plus_fg_key': '',
               'sim_beam_plus_data_key': '',
               'sim_delta_key': '',
               'sim_beam_meansub_key': '',
               'sim_beam_conv_key': '',
               'sim_beam_meansubconv_key': '',
               'template_key': '',
               'weight_key': '',
               'pwrspec_scenario': '',
               'refinement': 2,
               'omega_HI': ''
               }
prefix = 'sg_'

if __name__ == '__main__':
    params = parse_ini.parse(str(sys.argv[1]), params_init,
                             prefix=prefix)
    print params

    datapath_db = data_paths.DataPath()
    output_root = datapath_db.fetch(params['output_root'])

    file_tools.mkparents(output_root)
    parse_ini.write_params(params, output_root + 'params.ini',
                           prefix=prefix)

    generate_sim(params, parallel=True, datapath_db=datapath_db)
    generate_aux_simset(params, datapath_db=datapath_db)

def batch_crosspwr_transfer(cleaned_simkey,
                            truesignal_simkey,
                            truesignal_weightkey,
                            reference_simkey,
                            reference_weightkey,
                            inifile=None,
                            datapath_db=None,
                            outdir="./plots",
                            output_tag=None):
    r"""take relevant cross-powers
    cleaned_simkey(map) * cleaned_simkey(weight) x
    truesignal_weightkey * truesignal_simkey
    divided by:
    reference_simkey * reference_weightkey x truesignal_weightkey * truesignal_simkey
    """

    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    map_cases = datapath_db.fileset_cases(cleaned_simkey, "type;treatment")

    funcname = "correlate.batch_quadratic.call_xspec_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname,
                                        cache_path,
                                        generate=generate,
                                        verbose=True)

    dbkeydict = {}
    dbkeydict['map1_key'] = reference_simkey
    dbkeydict['map2_key'] = truesignal_simkey
    dbkeydict['noiseinv1_key'] = reference_weightkey
    dbkeydict['noiseinv2_key'] = truesignal_weightkey
    files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
                                                     datapath_db=datapath_db)

    reference_pwrspec_out = caller.execute(files['map1_key'],
                                           files['map2_key'],
                                           files['noiseinv1_key'],
                                           files['noiseinv2_key'],
                                           inifile=inifile)

    if output_tag:
        file_tools.mkparents(outdir)
        ref_pwr_1d_from_2d = pe.convert_2d_to_1d(reference_pwrspec_out[0])

        ref_pwr_1d = reference_pwrspec_out[1]['binavg']
        ref_pwr_2d = reference_pwrspec_out[0]['binavg']
        ref_pwr_1d_from_2d = ref_pwr_1d_from_2d['binavg']

        bin_left = reference_pwrspec_out[1]['bin_left']
        bin_center = reference_pwrspec_out[1]['bin_center']
        bin_right = reference_pwrspec_out[1]['bin_right']
        counts_histo = reference_pwrspec_out[1]['counts_histo']

    transfer_functions = {}
    for treatment in map_cases['treatment']:
        dbkeydict = {}
        dbkeydict['map1_key'] = "%s:map;%s" % (cleaned_simkey, treatment)
        dbkeydict['map2_key'] = truesignal_simkey
        dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (cleaned_simkey,
                                                       treatment)
        dbkeydict['noiseinv2_key'] = truesignal_weightkey
        files = data_paths.convert_dbkeydict_to_filedict(
            dbkeydict, datapath_db=datapath_db)

        pwrspec_out_signal = caller.execute(files['map1_key'],
                                            files['map2_key'],
                                            files['noiseinv1_key'],
                                            files['noiseinv2_key'],
                                            inifile=inifile)

        if output_tag:
            pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])

            pwr_1d = pwrspec_out_signal[1]['binavg']
            pwr_2d = pwrspec_out_signal[0]['binavg']
            pwr_1d_from_2d = pwr_1d_from_2d['binavg']

            trans1d_mode = pwr_1d / ref_pwr_1d
            trans1d_from2d_mode = pwr_1d_from_2d / ref_pwr_1d_from_2d
            trans2d_mode = pwr_2d / ref_pwr_2d

            transfer_functions[treatment] = (trans1d_mode, trans1d_from2d_mode,
                                             trans2d_mode)

            filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)

            outfile = open(filename, "w")
            for specdata in zip(bin_left, bin_center, bin_right, counts_histo,
                                pwr_1d, pwr_1d_from_2d, trans1d_mode,
                                trans1d_from2d_mode):
                outfile.write(("%10.15g " * 8 + "\n") % specdata)
            outfile.close()

    if not output_tag:
        caller.multiprocess_stack()
        return None
    else:
        return transfer_functions
示例#24
0
def batch_gbtpwrspec_data_run(map_key, inifile=None, datapath_db=None,
                   output_tag=None, beam_transfer=None,
                   outdir="./plots/",
                   square_1dmodetrans=False,
                   mode_transfer_1d=None,
                   mode_transfer_2d=None):
    r"""Form the pairs of maps1*weight1 x map0*weight0 for calculating the
    auto-power of the GBT data"""

    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")

    funcname = "correlate.batch_quadratic.call_xspec_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname, cache_path,
                                        generate=generate, verbose=True)

    if output_tag:
        file_tools.mkparents(outdir)

    pwrspec_collection = {}
    for treatment in map_cases['treatment']:
        unique_pairs = data_paths.GBTauto_cross_pairs(map_cases['pair'],
                                                    map_cases['pair'],
                                                    cross_sym="_with_")

        # TODO: make this more elegant
        transfer_2d = None
        if (mode_transfer_2d is not None) and (beam_transfer is None):
            transfer_2d = mode_transfer_2d[treatment][1]

        if (mode_transfer_2d is None) and (beam_transfer is not None):
            transfer_2d = beam_transfer

        if (mode_transfer_2d is not None) and (beam_transfer is not None):
            transfer_2d = mode_transfer_2d[treatment][1] * beam_transfer

        pwr_1d = []
        pwr_2d = []
        pwr_1d_from_2d = []
        for item in unique_pairs:
            dbkeydict = {}
            # NOTE: formerly had "weight" instead of noise_inv
            mapset0 = (map_key, item[0], treatment)
            mapset1 = (map_key, item[1], treatment)
            dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
            dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
            dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
            dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
            files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
                                                      datapath_db=datapath_db)

            pwrspec_out = caller.execute(files['map1_key'],
                                         files['map2_key'],
                                         files['noiseinv1_key'],
                                         files['noiseinv2_key'],
                                         inifile=inifile)

            if output_tag:
                pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
                                      transfer=transfer_2d))

                pwr_2d.append(pwrspec_out[0])
                pwr_1d.append(pwrspec_out[1])

        if output_tag:
            mtag = output_tag + "_%s" % treatment
            if mode_transfer_1d is not None:
                transfunc = mode_transfer_1d[treatment][0]
                if square_1dmodetrans:
                    transfunc *= transfunc
            else:
                transfunc = None

            agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
                                                   pwr_1d_from_2d, pwr_2d, mtag,
                                                   outdir=outdir,
                                                   apply_1d_transfer=transfunc)

            # (mean_1d, std_1d, covmat_1d)
            pwrspec_collection[treatment] = agg_pwrspec

    if output_tag:
        return pwrspec_collection
    else:
        caller.multiprocess_stack()
        return None
示例#25
0
def batch_gbtpwrspec_data_run(map_key,
                              inifile=None,
                              datapath_db=None,
                              output_tag=None,
                              beam_transfer=None,
                              outdir="./plots/",
                              square_1dmodetrans=False,
                              mode_transfer_1d=None,
                              mode_transfer_2d=None):
    r"""Form the pairs of maps1*weight1 x map0*weight0 for calculating the
    auto-power of the GBT data"""

    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")

    funcname = "correlate.batch_quadratic.call_xspec_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname,
                                        cache_path,
                                        generate=generate,
                                        verbose=True)

    if output_tag:
        file_tools.mkparents(outdir)

    pwrspec_collection = {}
    for treatment in map_cases['treatment']:
        unique_pairs = data_paths.GBTauto_cross_pairs(map_cases['pair'],
                                                      map_cases['pair'],
                                                      cross_sym="_with_")

        # TODO: make this more elegant
        transfer_2d = None
        if (mode_transfer_2d is not None) and (beam_transfer is None):
            transfer_2d = mode_transfer_2d[treatment][1]

        if (mode_transfer_2d is None) and (beam_transfer is not None):
            transfer_2d = beam_transfer

        if (mode_transfer_2d is not None) and (beam_transfer is not None):
            transfer_2d = mode_transfer_2d[treatment][1] * beam_transfer

        pwr_1d = []
        pwr_2d = []
        pwr_1d_from_2d = []
        for item in unique_pairs:
            dbkeydict = {}
            # NOTE: formerly had "weight" instead of noise_inv
            mapset0 = (map_key, item[0], treatment)
            mapset1 = (map_key, item[1], treatment)
            dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
            dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
            dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
            dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
            files = data_paths.convert_dbkeydict_to_filedict(
                dbkeydict, datapath_db=datapath_db)

            pwrspec_out = caller.execute(files['map1_key'],
                                         files['map2_key'],
                                         files['noiseinv1_key'],
                                         files['noiseinv2_key'],
                                         inifile=inifile)

            if output_tag:
                pwr_1d_from_2d.append(
                    pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d))

                pwr_2d.append(pwrspec_out[0])
                pwr_1d.append(pwrspec_out[1])

        if output_tag:
            mtag = output_tag + "_%s" % treatment
            if mode_transfer_1d is not None:
                transfunc = mode_transfer_1d[treatment][0]
                if square_1dmodetrans:
                    transfunc *= transfunc
            else:
                transfunc = None

            agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
                                                   pwr_1d_from_2d,
                                                   pwr_2d,
                                                   mtag,
                                                   outdir=outdir,
                                                   apply_1d_transfer=transfunc)

            # (mean_1d, std_1d, covmat_1d)
            pwrspec_collection[treatment] = agg_pwrspec

    if output_tag:
        return pwrspec_collection
    else:
        caller.multiprocess_stack()
        return None
示例#26
0
def batch_gbtxwigglez_data_run(gbt_map_key,
                               wigglez_map_key,
                               wigglez_mock_key,
                               wigglez_selection_key,
                               inifile=None,
                               datapath_db=None,
                               outdir="./plots",
                               output_tag=None,
                               beam_transfer=None,
                               mode_transfer_1d=None,
                               mode_transfer_2d=None,
                               theory_curve=None):
    r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""

    if datapath_db is None:
        datapath_db = data_paths.DataPath()

    cache_path = datapath_db.fetch("quadratic_batch_data")

    map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
    mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")

    funcname = "correlate.batch_quadratic.call_xspec_run"
    generate = False if output_tag else True
    if generate:
        print "REGENERATING the power spectrum result cache: "

    caller = batch_handler.MemoizeBatch(funcname,
                                        cache_path,
                                        generate=generate,
                                        verbose=True)

    if output_tag:
        file_tools.mkparents(outdir)

    for treatment in map_cases['treatment']:
        # TODO: make this more elegant
        # TODO: convert treatment into mode num
        transfer_2d = None
        if (mode_transfer_2d is not None) and (beam_transfer is None):
            transfer_2d = mode_transfer_2d[treatment][2]

        if (mode_transfer_2d is None) and (beam_transfer is not None):
            transfer_2d = beam_transfer

        if (mode_transfer_2d is not None) and (beam_transfer is not None):
            transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer

        pwr_1d = []
        pwr_2d = []
        pwr_1d_from_2d = []
        for index in mock_cases['realization']:
            dbkeydict = {}
            dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
            dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
            dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
                                         (gbt_map_key, treatment)

            dbkeydict['noiseinv2_key'] = wigglez_selection_key
            files = data_paths.convert_dbkeydict_to_filedict(
                dbkeydict, datapath_db=datapath_db)

            pwrspec_out = caller.execute(files['map1_key'],
                                         files['map2_key'],
                                         files['noiseinv1_key'],
                                         files['noiseinv2_key'],
                                         inifile=inifile)

            if output_tag:
                pwr_1d_from_2d.append(
                    pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d))

                pwr_2d.append(pwrspec_out[0])
                pwr_1d.append(pwrspec_out[1])

        if output_tag:
            if mode_transfer_1d is not None:
                transfunc = mode_transfer_1d[treatment][1]
            else:
                transfunc = None

            mtag = output_tag + "_%s_mock" % treatment
            agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
                                                   pwr_1d_from_2d,
                                                   pwr_2d,
                                                   mtag,
                                                   outdir=outdir,
                                                   apply_1d_transfer=transfunc)

            mean1dmock = agg_pwrspec["mean_1d"]
            std1dmock = agg_pwrspec["std_1d"]
            covmock = agg_pwrspec["covmat_1d"]

        # now recover the xspec with the real data
        dbkeydict = {}
        dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
        dbkeydict['map2_key'] = wigglez_map_key
        dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
        dbkeydict['noiseinv2_key'] = wigglez_selection_key
        files = data_paths.convert_dbkeydict_to_filedict(
            dbkeydict, datapath_db=datapath_db)

        pwrspec_out_signal = caller.execute(files['map1_key'],
                                            files['map2_key'],
                                            files['noiseinv1_key'],
                                            files['noiseinv2_key'],
                                            inifile=inifile)

        if output_tag:
            pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
                                                 transfer=transfer_2d)

            pwr_1d = pwrspec_out_signal[1]['binavg']
            pwr_1d_from_2d = pwr_1d_from_2d['binavg']
            if mode_transfer_1d is not None:
                pwr_1d /= mode_transfer_1d[treatment][1]
                pwr_1d_from_2d /= mode_transfer_1d[treatment][1]

            # assume that they all have the same binning
            bin_left = pwrspec_out_signal[1]['bin_left']
            bin_center = pwrspec_out_signal[1]['bin_center']
            bin_right = pwrspec_out_signal[1]['bin_right']
            counts_histo = pwrspec_out_signal[1]['counts_histo']

            filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)

            outfile = open(filename, "w")
            for specdata in zip(bin_left, bin_center, bin_right, counts_histo,
                                pwr_1d, pwr_1d_from_2d, mean1dmock, std1dmock):
                outfile.write(("%10.15g " * 8 + "\n") % specdata)
            outfile.close()

            # TODO: kludge to make a fast fit; remove
            theory_curve = np.genfromtxt(
                "plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat"
            )
            theory_curve = theory_curve[:, 4]

            if theory_curve is not None:
                restrict = np.where(
                    np.logical_and(bin_center > 0.09, bin_center < 1.1))
                res_slice = slice(min(restrict[0]), max(restrict[0]))

                #restrict_alt = np.where(restrict)[0][np.newaxis, :]
                #restricted_cov = covmock[restrict_alt][0]

                from core import utils
                amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
                                         covmock[res_slice, res_slice],
                                         theory_curve[res_slice])
                print "AMP:", mtag, treatment, amplitude

    if not output_tag:
        caller.multiprocess_stack()

    return None