def test_create_gp(use_interpolators):
"""Create valid GP with reasonable parameters, kernel, mean fct"""
redshiftGrid = np.logspace(-2, np.log10(4), num=numZ)
f_mod_interp = np.zeros((numTemplates, numBands), dtype=object)
for it in range(numTemplates):
for jf in range(numBands):
data = np.random.randn(numZ)
f_mod_interp[it, jf] = interp1d(redshiftGrid, data,
kind='linear', bounds_error=False,
fill_value='extrapolate')
gp = PhotozGP(
f_mod_interp,
fcoefs_amp, fcoefs_mu, fcoefs_sig,
lines_mu, lines_sig,
var_C, var_L, alpha_C, alpha_L,
redshiftGrid,
use_interpolators=use_interpolators
)
gp.setData(X, Y, Yvar, np.random.randint(numTemplates))
if use_interpolators is False:
gp.optimizeHyperparamaters()
return gp
def test_create_gp(use_interpolators):
"""Create valid GP with reasonable parameters, kernel, mean fct"""
redshiftGrid = np.logspace(-2, np.log10(4), num=numZ)
f_mod_interp = np.zeros((numTemplates, numBands), dtype=object)
for it in range(numTemplates):
for jf in range(numBands):
data = np.random.randn(numZ)
f_mod_interp[it, jf] = interp1d(redshiftGrid,
data,
kind='linear',
bounds_error=False,
fill_value='extrapolate')
gp = PhotozGP(f_mod_interp,
fcoefs_amp,
fcoefs_mu,
fcoefs_sig,
lines_mu,
lines_sig,
var_C,
var_L,
alpha_C,
alpha_L,
redshiftGrid,
use_interpolators=use_interpolators)
gp.setData(X, Y, Yvar, np.random.randint(numTemplates))
if use_interpolators is False:
gp.optimizeHyperparamaters()
return gp
min(numObjectsTarget,
(threadNum + 1) * numObjectsTarget / float(numThreads)))
numLines = lastLine - firstLine
if threadNum == 0:
print('Number of Training Objects', numObjectsTraining)
print('Number of Target Objects', numObjectsTarget)
comm.Barrier()
print('Thread ', threadNum, ' analyzes lines ', firstLine, ' to ', lastLine)
DL = approx_DL()
gp = PhotozGP(f_mod_interp,
bandCoefAmplitudes,
bandCoefPositions,
bandCoefWidths,
params['lines_pos'],
params['lines_width'],
params['V_C'],
params['V_L'],
params['alpha_C'],
params['alpha_L'],
redshiftGridGP,
use_interpolators=True)
# Create local files to store results
numMetrics = 7 + len(params['confidenceLevels'])
localPDFs = np.zeros((numLines, numZ))
localMetrics = np.zeros((numLines, numMetrics))
localCompressIndices = np.zeros((numLines, Ncompress), dtype=int)
localCompEvidences = np.zeros((numLines, Ncompress))
# Looping over chunks of the training set to prepare model predictions over z
numChunks = params['training_numChunks']
def delightApply(configfilename):
"""
:param configfilename:
:return:
"""
threadNum = 0
numThreads = 1
params = parseParamFile(configfilename, verbose=False, catFilesNeeded=True)
if threadNum == 0:
#print("--- DELIGHT-APPLY ---")
logger.info("--- DELIGHT-APPLY ---")
# Read filter coefficients, compute normalization of filters
bandCoefAmplitudes, bandCoefPositions, bandCoefWidths, norms = readBandCoefficients(
params)
numBands = bandCoefAmplitudes.shape[0]
redshiftDistGrid, redshiftGrid, redshiftGridGP = createGrids(params)
f_mod_interp = readSEDs(params)
nt = f_mod_interp.shape[0]
nz = redshiftGrid.size
dir_seds = params['templates_directory']
dir_filters = params['bands_directory']
lambdaRef = params['lambdaRef']
sed_names = params['templates_names']
f_mod_grid = np.zeros(
(redshiftGrid.size, len(sed_names), len(params['bandNames'])))
for t, sed_name in enumerate(sed_names):
f_mod_grid[:, t, :] = np.loadtxt(dir_seds + '/' + sed_name +
'_fluxredshiftmod.txt')
numZbins = redshiftDistGrid.size - 1
numZ = redshiftGrid.size
numObjectsTraining = np.sum(1 for line in open(params['training_catFile']))
numObjectsTarget = np.sum(1 for line in open(params['target_catFile']))
redshiftsInTarget = ('redshift' in params['target_bandOrder'])
Ncompress = params['Ncompress']
firstLine = int(threadNum * numObjectsTarget / float(numThreads))
lastLine = int(
min(numObjectsTarget,
(threadNum + 1) * numObjectsTarget / float(numThreads)))
numLines = lastLine - firstLine
if threadNum == 0:
msg = 'Number of Training Objects ' + str(numObjectsTraining)
logger.info(msg)
msg = 'Number of Target Objects ' + str(numObjectsTarget)
logger.info(msg)
msg = 'Thread ' + str(threadNum) + ' , analyzes lines ' + str(
firstLine) + ' to ' + str(lastLine)
logger.info(msg)
DL = approx_DL()
gp = PhotozGP(f_mod_interp,
bandCoefAmplitudes,
bandCoefPositions,
bandCoefWidths,
params['lines_pos'],
params['lines_width'],
params['V_C'],
params['V_L'],
params['alpha_C'],
params['alpha_L'],
redshiftGridGP,
use_interpolators=True)
# Create local files to store results
numMetrics = 7 + len(params['confidenceLevels'])
localPDFs = np.zeros((numLines, numZ))
localMetrics = np.zeros((numLines, numMetrics))
localCompressIndices = np.zeros((numLines, Ncompress), dtype=int)
localCompEvidences = np.zeros((numLines, Ncompress))
# Looping over chunks of the training set to prepare model predictions over z
numChunks = params['training_numChunks']
for chunk in range(numChunks):
TR_firstLine = int(chunk * numObjectsTraining / float(numChunks))
TR_lastLine = int(
min(numObjectsTraining,
(chunk + 1) * numObjectsTarget / float(numChunks)))
targetIndices = np.arange(TR_firstLine, TR_lastLine)
numTObjCk = TR_lastLine - TR_firstLine
redshifts = np.zeros((numTObjCk, ))
model_mean = np.zeros((numZ, numTObjCk, numBands))
model_covar = np.zeros((numZ, numTObjCk, numBands))
bestTypes = np.zeros((numTObjCk, ), dtype=int)
ells = np.zeros((numTObjCk, ), dtype=int)
# loop on training data and training GP coefficients produced by delight_learn
# It fills the model_mean and model_covar predicted by GP
loc = TR_firstLine - 1
trainingDataIter = getDataFromFile(params,
TR_firstLine,
TR_lastLine,
prefix="training_",
ftype="gpparams")
# loop on training data to load the GP parameter
for loc, (z, ell, bands, X, B,
flatarray) in enumerate(trainingDataIter):
t1 = time()
redshifts[loc] = z # redshift of all training samples
gp.setCore(X, B, nt, flatarray[0:nt + B + B * (B + 1) // 2])
bestTypes[
loc] = gp.bestType # retrieve the best-type found by delight-learn
ells[loc] = ell # retrieve the luminosity parameter l
# here is the model prediction of Gaussian Process for that particular trainning galaxy
model_mean[:,
loc, :], model_covar[:,
loc, :] = gp.predictAndInterpolate(
redshiftGrid, ell=ell)
t2 = time()
# print(loc, t2-t1)
#Redshift prior on training galaxy
# p_t = params['p_t'][bestTypes][None, :]
# p_z_t = params['p_z_t'][bestTypes][None, :]
# compute the prior for taht training sample
prior = np.exp(-0.5 * ((redshiftGrid[:, None] - redshifts[None, :]) /
params['zPriorSigma'])**2)
# prior[prior < 1e-6] = 0
# prior *= p_t * redshiftGrid[:, None] *
# np.exp(-0.5 * redshiftGrid[:, None]**2 / p_z_t) / p_z_t
if params['useCompression'] and params['compressionFilesFound']:
fC = open(params['compressMargLikFile'])
fCI = open(params['compressIndicesFile'])
itCompM = itertools.islice(fC, firstLine, lastLine)
iterCompI = itertools.islice(fCI, firstLine, lastLine)
targetDataIter = getDataFromFile(params,
firstLine,
lastLine,
prefix="target_",
getXY=False,
CV=False)
# loop on target samples
for loc, (z, normedRefFlux, bands, fluxes, fluxesVar, bCV, dCV,
dVCV) in enumerate(targetDataIter):
t1 = time()
ell_hat_z = normedRefFlux * 4 * np.pi * params[
'fluxLuminosityNorm'] * (DL(redshiftGrid)**2. *
(1 + redshiftGrid))
ell_hat_z[:] = 1
if params['useCompression'] and params['compressionFilesFound']:
indices = np.array(next(iterCompI).split(' '), dtype=int)
sel = np.in1d(targetIndices, indices, assume_unique=True)
# same likelihood as for template fitting
like_grid2 = approx_flux_likelihood(
fluxes,
fluxesVar,
model_mean[:, sel, :][:, :, bands],
f_mod_covar=model_covar[:, sel, :][:, :, bands],
marginalizeEll=True,
normalized=False,
ell_hat=ell_hat_z,
ell_var=(ell_hat_z * params['ellPriorSigma'])**2)
like_grid *= prior[:, sel]
else:
like_grid = np.zeros((nz, model_mean.shape[1]))
# same likelihood as for template fitting, but cython
approx_flux_likelihood_cy(
like_grid,
nz,
model_mean.shape[1],
bands.size,
fluxes,
fluxesVar, # target galaxy fluxes and variance
model_mean[:, :,
bands], # prediction with Gaussian process
model_covar[:, :, bands],
ell_hat=ell_hat_z, # it will find internally the ell
ell_var=(ell_hat_z * params['ellPriorSigma'])**2)
like_grid *= prior[:, :] #likelihood multiplied by redshift training galaxies priors
t2 = time()
localPDFs[loc, :] += like_grid.sum(
axis=1
) # the final redshift posterior is sum over training galaxies posteriors
# compute the evidence for each model
evidences = np.trapz(like_grid, x=redshiftGrid, axis=0)
t3 = time()
if params['useCompression'] and not params['compressionFilesFound']:
if localCompressIndices[loc, :].sum() == 0:
sortind = np.argsort(evidences)[::-1][0:Ncompress]
localCompressIndices[loc, :] = targetIndices[sortind]
localCompEvidences[loc, :] = evidences[sortind]
else:
dind = np.concatenate(
(targetIndices, localCompressIndices[loc, :]))
devi = np.concatenate(
(evidences, localCompEvidences[loc, :]))
sortind = np.argsort(devi)[::-1][0:Ncompress]
localCompressIndices[loc, :] = dind[sortind]
localCompEvidences[loc, :] = devi[sortind]
if chunk == numChunks - 1\
and redshiftsInTarget\
and localPDFs[loc, :].sum() > 0:
localMetrics[loc, :] = computeMetrics(
z, redshiftGrid, localPDFs[loc, :],
params['confidenceLevels'])
t4 = time()
if loc % 100 == 0:
print(loc, t2 - t1, t3 - t2, t4 - t3)
if params['useCompression'] and params['compressionFilesFound']:
fC.close()
fCI.close()
#comm.Barrier()
if threadNum == 0:
globalPDFs = np.zeros((numObjectsTarget, numZ))
globalCompressIndices = np.zeros((numObjectsTarget, Ncompress),
dtype=int)
globalCompEvidences = np.zeros((numObjectsTarget, Ncompress))
globalMetrics = np.zeros((numObjectsTarget, numMetrics))
firstLines = [
int(k * numObjectsTarget / numThreads) for k in range(numThreads)
]
lastLines = [
int(min(numObjectsTarget, (k + 1) * numObjectsTarget / numThreads))
for k in range(numThreads)
]
numLines = [lastLines[k] - firstLines[k] for k in range(numThreads)]
sendcounts = tuple([numLines[k] * numZ for k in range(numThreads)])
displacements = tuple([firstLines[k] * numZ for k in range(numThreads)])
#comm.Gatherv(localPDFs,[globalPDFs, sendcounts, displacements, MPI.DOUBLE])
globalPDFs = localPDFs
sendcounts = tuple([numLines[k] * Ncompress for k in range(numThreads)])
displacements = tuple(
[firstLines[k] * Ncompress for k in range(numThreads)])
#comm.Gatherv(localCompressIndices,[globalCompressIndices, sendcounts, displacements, MPI.LONG])
#comm.Gatherv(localCompEvidences,[globalCompEvidences, sendcounts, displacements, MPI.DOUBLE])
globalCompressIndices = localCompressIndices
globalCompEvidences = localCompEvidences
#comm.Barrier()
sendcounts = tuple([numLines[k] * numMetrics for k in range(numThreads)])
displacements = tuple(
[firstLines[k] * numMetrics for k in range(numThreads)])
#comm.Gatherv(localMetrics,[globalMetrics, sendcounts, displacements, MPI.DOUBLE])
globalMetrics = localMetrics
#comm.Barrier()
if threadNum == 0:
fmt = '%.2e'
fname = params['redshiftpdfFileComp'] if params['compressionFilesFound']\
else params['redshiftpdfFile']
np.savetxt(fname, globalPDFs, fmt=fmt)
if redshiftsInTarget:
np.savetxt(params['metricsFile'], globalMetrics, fmt=fmt)
if params['useCompression'] and not params['compressionFilesFound']:
np.savetxt(params['compressMargLikFile'],
globalCompEvidences,
fmt=fmt)
np.savetxt(params['compressIndicesFile'],
globalCompressIndices,
fmt="%i")
redshiftDistGrid, redshiftGrid, redshiftGridGP = createGrids(params)
f_mod = readSEDs(params)
numObjectsTraining = np.sum(1 for line in open(params['training_catFile']))
print('Number of Training Objects', numObjectsTraining)
firstLine = int(threadNum * numObjectsTraining / numThreads)
lastLine = int(min(numObjectsTraining,
(threadNum + 1) * numObjectsTraining / numThreads))
numLines = lastLine - firstLine
#comm.Barrier()
print('Thread ', threadNum, ' analyzes lines ', firstLine, ' to ', lastLine)
DL = approx_DL()
gp = PhotozGP(f_mod, bandCoefAmplitudes, bandCoefPositions, bandCoefWidths,
params['lines_pos'], params['lines_width'],
params['V_C'], params['V_L'],
params['alpha_C'], params['alpha_L'],
redshiftGridGP, use_interpolators=True)
B = numBands
numCol = 3 + B + B*(B+1)//2 + B + f_mod.shape[0]
localData = np.zeros((numLines, numCol))
fmt = '%i ' + '%.12e ' * (localData.shape[1] - 1)
loc = - 1
crossValidate = params['training_crossValidate']
trainingDataIter1 = getDataFromFile(params, firstLine, lastLine,
prefix="training_", getXY=True,
CV=crossValidate)
if crossValidate:
chi2sLocal = None
from delight.utils import *
if len(sys.argv) < 2:
raise Exception('Please provide a parameter file')
params = parseParamFile(sys.argv[1], verbose=False, catFilesNeeded=False)
redshiftDistGrid, redshiftGrid, redshiftGridGP = createGrids(params)
numObjects = params['numObjects']
noiseLevel = params['noiseLevel']
f_mod = readSEDs(params)
bandCoefAmplitudes, bandCoefPositions, bandCoefWidths, norms\
= readBandCoefficients(params)
gp = PhotozGP(f_mod, bandCoefAmplitudes, bandCoefPositions, bandCoefWidths,
params['lines_pos'], params['lines_width'],
params['V_C'], params['V_L'],
params['alpha_C'], params['alpha_L'],
redshiftGridGP, use_interpolators=True)
DL = approx_DL()
bandIndices_TRN, bandNames_TRN, bandColumns_TRN,\
bandVarColumns_TRN, redshiftColumn_TRN,\
refBandColumn_TRN = readColumnPositions(params, prefix='training_')
bandIndices_TAR, bandNames_TAR, bandColumns_TAR,\
bandVarColumns_TAR, redshiftColumn_TAR,\
refBandColumn_TAR = readColumnPositions(params, prefix='target_')
bandNames = params['bandNames']
refBandLoc = np.where(bandNames_TRN == params['training_referenceBand'])[0]
refBandNorm = norms[bandNames.index(params['training_referenceBand'])]
loc = -1
numConfLevels = len(params['confidenceLevels'])
numObjectsTraining = np.sum(1 for line in open(params['training_catFile']))
numObjectsTarget = np.sum(1 for line in open(params['target_catFile']))
print('Number of Training Objects', numObjectsTraining)
print('Number of Target Objects', numObjectsTarget)
for ellPriorSigma in [1.0, 10.0]:
alpha_C = 1e3
alpha_L = 1e2
V_C, V_L = 1.0, 1.0
gp = PhotozGP(f_mod,
bandCoefAmplitudes,
bandCoefPositions,
bandCoefWidths,
params['lines_pos'],
params['lines_width'],
V_C,
V_L,
alpha_C,
alpha_L,
redshiftGridGP,
use_interpolators=True)
for extraFracFluxError in [1e-2]:
redshifts = np.zeros((numObjectsTraining, ))
bestTypes = np.zeros((numObjectsTraining, ), dtype=int)
ellMLs = np.zeros((numObjectsTraining, ))
model_mean = np.zeros((numZ, numObjectsTraining, numBands))
model_covar = np.zeros((numZ, numObjectsTraining, numBands))
# params['training_extraFracFluxError'] = extraFracFluxError
params['target_extraFracFluxError'] = extraFracFluxError
params = parseParamFile(sys.argv[1], verbose=False)
# Read filter coefficients, compute normalization of filters
bandCoefAmplitudes, bandCoefPositions, bandCoefWidths, norms\
= readBandCoefficients(params)
numBands = bandCoefAmplitudes.shape[0]
redshiftDistGrid, redshiftGrid, redshiftGridGP = createGrids(params)
f_mod = readSEDs(params)
numObjectsTraining = np.sum(1 for line in open(params['training_catFile']))
print('Number of Training Objects', numObjectsTraining)
gp = PhotozGP(f_mod, bandCoefAmplitudes, bandCoefPositions, bandCoefWidths,
params['lines_pos'], params['lines_width'],
params['V_C'], params['V_L'],
params['alpha_C'], params['alpha_L'],
redshiftGridGP, use_interpolators=True)
numZ = redshiftGrid.size
all_z = np.zeros((numObjectsTraining, ))
all_fluxes = np.zeros((numObjectsTraining, numBands))
all_fluxes_var = np.zeros((numObjectsTraining, numBands))
bestTypes = np.zeros((numObjectsTraining, ), dtype=int)
model_mean = np.zeros((numZ, numObjectsTraining, numBands))
model_covar = np.zeros((numZ, numObjectsTraining, numBands))
bandIndices_TRN, bandNames_TRN, bandColumns_TRN,\
bandVarColumns_TRN, redshiftColumn_TRN,\
refBandColumn_TRN = readColumnPositions(params, prefix='training_')
bandIndices, bandNames, bandColumns,\
bandVarColumns, redshiftColumn,\
def delightLearn(configfilename):
"""
:param configfilename:
:return:
"""
threadNum = 0
numThreads = 1
#parse arguments
params = parseParamFile(configfilename,
verbose=False,
catFilesNeeded=False)
if threadNum == 0:
logger.info("--- DELIGHT-LEARN ---")
# Read filter coefficients, compute normalization of filters
bandCoefAmplitudes, bandCoefPositions, bandCoefWidths, norms = readBandCoefficients(
params)
numBands = bandCoefAmplitudes.shape[0]
redshiftDistGrid, redshiftGrid, redshiftGridGP = createGrids(params)
f_mod = readSEDs(params)
numObjectsTraining = np.sum(1 for line in open(params['training_catFile']))
msg = 'Number of Training Objects ' + str(numObjectsTraining)
logger.info(msg)
firstLine = int(threadNum * numObjectsTraining / numThreads)
lastLine = int(
min(numObjectsTraining,
(threadNum + 1) * numObjectsTraining / numThreads))
numLines = lastLine - firstLine
msg = 'Thread ' + str(threadNum) + ' , analyzes lines ' + str(
firstLine) + ' , to ' + str(lastLine)
logger.info(msg)
DL = approx_DL()
gp = PhotozGP(f_mod,
bandCoefAmplitudes,
bandCoefPositions,
bandCoefWidths,
params['lines_pos'],
params['lines_width'],
params['V_C'],
params['V_L'],
params['alpha_C'],
params['alpha_L'],
redshiftGridGP,
use_interpolators=True)
B = numBands
numCol = 3 + B + B * (B + 1) // 2 + B + f_mod.shape[0]
localData = np.zeros((numLines, numCol))
fmt = '%i ' + '%.12e ' * (localData.shape[1] - 1)
loc = -1
crossValidate = params['training_crossValidate']
trainingDataIter1 = getDataFromFile(params,
firstLine,
lastLine,
prefix="training_",
getXY=True,
CV=crossValidate)
if crossValidate:
chi2sLocal = None
bandIndicesCV, bandNamesCV, bandColumnsCV, bandVarColumnsCV, redshiftColumnCV = readColumnPositions(
params, prefix="training_CV_", refFlux=False)
for z, normedRefFlux,\
bands, fluxes, fluxesVar,\
bandsCV, fluxesCV, fluxesVarCV,\
X, Y, Yvar in trainingDataIter1:
loc += 1
themod = np.zeros((1, f_mod.shape[0], bands.size))
for it in range(f_mod.shape[0]):
for ib, band in enumerate(bands):
themod[0, it, ib] = f_mod[it, band](z)
# really calibrate the luminosity parameter l compared to the model
# according the best type of galaxy
chi2_grid, ellMLs = scalefree_flux_likelihood(fluxes,
fluxesVar,
themod,
returnChi2=True)
bestType = np.argmin(chi2_grid) # best type
ell = ellMLs[0, bestType] # the luminosity factor
X[:, 2] = ell
gp.setData(X, Y, Yvar, bestType)
lB = bands.size
localData[loc, 0] = lB
localData[loc, 1] = z
localData[loc, 2] = ell
localData[loc, 3:3 + lB] = bands
localData[loc, 3 + lB:3 + f_mod.shape[0] + lB + lB * (lB + 1) // 2 +
lB] = gp.getCore()
if crossValidate:
model_mean, model_covar = gp.predictAndInterpolate(np.array([z]),
ell=ell)
if chi2sLocal is None:
chi2sLocal = np.zeros((numObjectsTraining, bandIndicesCV.size))
ind = np.array([list(bandIndicesCV).index(b) for b in bandsCV])
chi2sLocal[firstLine + loc,
ind] = -0.5 * (model_mean[0, bandsCV] - fluxesCV)**2 / (
model_covar[0, bandsCV] + fluxesVarCV)
if threadNum == 0:
reducedData = np.zeros((numObjectsTraining, numCol))
if crossValidate:
chi2sGlobal = np.zeros_like(chi2sLocal)
#comm.Allreduce(chi2sLocal, chi2sGlobal, op=MPI.SUM)
#comm.Barrier()
chi2sGlobal = chi2sLocal
firstLines = [
int(k * numObjectsTraining / numThreads) for k in range(numThreads)
]
lastLines = [
int(min(numObjectsTraining, (k + 1) * numObjectsTraining / numThreads))
for k in range(numThreads)
]
sendcounts = tuple([(lastLines[k] - firstLines[k]) * numCol
for k in range(numThreads)])
displacements = tuple([firstLines[k] * numCol for k in range(numThreads)])
reducedData = localData
# parameters for the GP process on traniing data are transfered to reduced data and saved in file
#'training_paramFile'
if threadNum == 0:
np.savetxt(params['training_paramFile'], reducedData, fmt=fmt)
if crossValidate:
np.savetxt(params['training_CVfile'], chi2sGlobal)
Ncompress = params['Ncompress']
firstLine = int(threadNum * numObjectsTarget / float(numThreads))
lastLine = int(min(numObjectsTarget,
(threadNum + 1) * numObjectsTarget / float(numThreads)))
numLines = lastLine - firstLine
if threadNum == 0:
print('Number of Training Objects', numObjectsTraining)
print('Number of Target Objects', numObjectsTarget)
comm.Barrier()
print('Thread ', threadNum, ' analyzes lines ', firstLine, ' to ', lastLine)
DL = approx_DL()
gp = PhotozGP(f_mod_interp,
bandCoefAmplitudes, bandCoefPositions, bandCoefWidths,
params['lines_pos'], params['lines_width'],
params['V_C'], params['V_L'],
params['alpha_C'], params['alpha_L'],
redshiftGridGP, use_interpolators=True)
# Create local files to store results
numMetrics = 7 + len(params['confidenceLevels'])
localPDFs = np.zeros((numLines, numZ))
localMetrics = np.zeros((numLines, numMetrics))
localCompressIndices = np.zeros((numLines, Ncompress), dtype=int)
localCompEvidences = np.zeros((numLines, Ncompress))
# Looping over chunks of the training set to prepare model predictions over z
numChunks = params['training_numChunks']
for chunk in range(numChunks):
TR_firstLine = int(chunk * numObjectsTraining / float(numChunks))
TR_lastLine = int(min(numObjectsTraining,
bandCoefAmplitudes, bandCoefPositions, bandCoefWidths, norms\
= readBandCoefficients(params)
numBands = bandCoefAmplitudes.shape[0]
redshiftDistGrid, redshiftGrid, redshiftGridGP = createGrids(params)
f_mod = readSEDs(params)
numObjectsTraining = np.sum(1 for line in open(params['training_catFile']))
print('Number of Training Objects', numObjectsTraining)
gp = PhotozGP(f_mod,
bandCoefAmplitudes,
bandCoefPositions,
bandCoefWidths,
params['lines_pos'],
params['lines_width'],
params['V_C'],
params['V_L'],
params['alpha_C'],
params['alpha_L'],
redshiftGridGP,
use_interpolators=True)
numZ = redshiftGrid.size
all_z = np.zeros((numObjectsTraining, ))
all_fluxes = np.zeros((numObjectsTraining, numBands))
all_fluxes_var = np.zeros((numObjectsTraining, numBands))
bestTypes = np.zeros((numObjectsTraining, ), dtype=int)
model_mean = np.zeros((numZ, numObjectsTraining, numBands))
model_covar = np.zeros((numZ, numObjectsTraining, numBands))
bandIndices_TRN, bandNames_TRN, bandColumns_TRN,\
bandVarColumns_TRN, redshiftColumn_TRN,\
redshiftDistGrid, redshiftGrid, redshiftGridGP = createGrids(params)
f_mod = readSEDs(params)
numObjectsTraining = np.sum(1 for line in open(params['training_catFile']))
print('Number of Training Objects', numObjectsTraining)
firstLine = int(threadNum * numObjectsTraining / numThreads)
lastLine = int(min(numObjectsTraining,
(threadNum + 1) * numObjectsTraining / numThreads))
numLines = lastLine - firstLine
comm.Barrier()
print('Thread ', threadNum, ' analyzes lines ', firstLine, ' to ', lastLine)
DL = approx_DL()
gp = PhotozGP(f_mod, bandCoefAmplitudes, bandCoefPositions, bandCoefWidths,
params['lines_pos'], params['lines_width'],
params['V_C'], params['V_L'],
params['alpha_C'], params['alpha_L'],
redshiftGridGP, use_interpolators=True)
B = numBands
numCol = 3 + B + B*(B+1)//2 + B + f_mod.shape[0]
localData = np.zeros((numLines, numCol))
fmt = '%i ' + '%.12e ' * (localData.shape[1] - 1)
loc = - 1
crossValidate = params['training_crossValidate']
trainingDataIter1 = getDataFromFile(params, firstLine, lastLine,
prefix="training_", getXY=True,
CV=crossValidate)
if crossValidate:
chi2sLocal = None