def run_download(param_file, out_path, data):
"""Helper function for get_ifsar."""
df = pd.read_csv(param_file, sep='\t')
#print(df.head())
if data.lower() == 'ifsar':
url_struct = 'http://dggs.alaska.gov/public_lidar/dds4/ifsar/dtm/'
download_list = list(df.iloc[:, 0])
download_list = [url_struct + str(i) for i in download_list]
else:
download_list = list(df.iloc[:, 0])
#execute downloads in parallel
results_classes = pyParz.foreach(download_list,
get_ifsar,
args=[out_path],
numThreads=8)
def classify(sn,
zhost=1.491,
zhosterr=0.003,
t0_range=None,
zminmax=[1.488, 1.493],
npoints=100,
maxiter=1000,
templateset='SNANA',
excludetemplates=[],
nsteps_pdf=101,
verbose=True):
""" Collect the bayesian evidence for all SN sub-classes.
:param sn:
:param zhost:
:param zhosterr:
:param t0_range:
:param zminmax:
:param npoints:
:param maxiter:
:param verbose:
:return:
"""
tstart = time.time()
if templateset.lower() == 'psnid':
SubClassDict = SubClassDict_PSNID
elif templateset.lower() == 'snana':
SubClassDict = SubClassDict_SNANA
iimodelnames = list(SubClassDict['ii'].keys())
ibcmodelnames = list(SubClassDict['ibc'].keys())
iamodelnames = list(SubClassDict['ia'].keys())
outdict = {}
modelProbs = SubClassDict.copy()
allmodelnames = np.append(np.append(iamodelnames, ibcmodelnames),
iimodelnames)
if excludetemplates:
#Removing templates for simulated data (Pass in CID of SN to exclude template)
theCID = excludetemplates.pop()
excludetemplates.append(getSimTemp(theCID))
if (excludetemplates[0] != 0):
for exmod in excludetemplates:
if exmod in allmodelnames:
allmodelnamelist = allmodelnames.tolist()
allmodelnamelist.remove(exmod)
allmodelnames = np.array(allmodelnamelist)
logpriordict = {
'ia': np.log(0.24 / len(iamodelnames)),
'ibc': np.log(0.19 / len(ibcmodelnames)),
'ii': np.log(0.57 / len(iimodelnames)),
}
logz = {'Ia': [], 'II': [], 'Ibc': []}
bestlogz = -np.inf
sn = inflateUncert(sn)
#-------------------------------------------------------------------------------
'''
#serial code
for modelsource in allmodelnames:
if verbose >1:
dt = time.time() - tstart
print('------------------------------')
print("model: %s dt=%i sec" % (modelsource, dt))
sn, res, fit, priorfn = get_evidence(
sn, modelsource=modelsource, zhost=zhost, zhosterr=zhosterr,
t0_range=t0_range, zminmax=zminmax,
npoints=npoints, maxiter=maxiter, verbose=max(0, verbose - 1))
if nsteps_pdf:
pdf = get_marginal_pdfs(res, nbins=nsteps_pdf,
verbose=max(0, verbose - 1))
else:
pdf = None
outdict[modelsource] = {'sn': sn, 'res': res, 'fit': fit,
'pdf': pdf, 'priorfn': priorfn}
if res.logz>bestlogz :
outdict['bestmodel'] = modelsource
bestlogz = res.logz
# multiply the model evidence by the sub-type prior
if modelsource in iimodelnames:
logprior = logpriordict['ii']
logz['II'].append(logprior + res.logz )
modelProbs['ii'][modelsource] = res.logz
elif modelsource in ibcmodelnames:
logprior = logpriordict['ibc']
logz['Ibc'].append(logprior + res.logz)
modelProbs['ibc'][modelsource] = res.logz
elif modelsource in iamodelnames:
logprior = logpriordict['ia']
logz['Ia'].append(logprior + res.logz)
modelProbs['ia'][modelsource] = res.logz
if(verbose):
import pprint
print(pprint.pprint(modelProbs))
# sum up the evidence from all models for each sn type
logztype = {}
for modelsource in ['II', 'Ibc', 'Ia']:
logztype[modelsource] = logz[modelsource][0]
for i in range(1, len(logz[modelsource])):
logztype[modelsource] = np.logaddexp(
logztype[modelsource], logz[modelsource][i])
'''
#-------------------------------------------------------------------------------
#parallelized code
res = parallelize.foreach(allmodelnames, _parallel, [
verbose, sn, zhost, zhosterr, t0_range, zminmax, npoints, maxiter,
nsteps_pdf, excludetemplates
])
print('------------------------------')
dt = time.time() - tstart
print("dt=%i sec" % dt)
res = {res[i]['key']: res[i] for i in range(len(res))}
for modelsource in allmodelnames:
print(modelsource)
if res[modelsource]['sn'] is None:
continue
outdict[modelsource] = {
'sn': res[modelsource]['sn'],
'res': res[modelsource]['res'],
'pdf': res[modelsource]['pdf']
}
if res[modelsource]['res']['logz'] > bestlogz:
outdict['bestmodel'] = modelsource
bestlogz = res[modelsource]['res']['logz']
# multiply the model evidence by the sub-type prior
if modelsource in iimodelnames:
logprior = logpriordict['ii']
logz['II'].append(logprior + res[modelsource]['res']['logz'])
modelProbs['ii'][modelsource] = res[modelsource]['res']['logz']
elif modelsource in ibcmodelnames:
logprior = logpriordict['ibc']
logz['Ibc'].append(logprior + res[modelsource]['res']['logz'])
modelProbs['ibc'][modelsource] = res[modelsource]['res']['logz']
elif modelsource in iamodelnames:
logprior = logpriordict['ia']
logz['Ia'].append(logprior + res[modelsource]['res']['logz'])
modelProbs['ia'][modelsource] = res[modelsource]['res']['logz']
if (verbose):
import pprint
print(pprint.pprint(modelProbs))
# sum up the evidence from all models for each sn type
logztype = {}
for modelsource in ['II', 'Ibc', 'Ia']:
logztype[modelsource] = logz[modelsource][0]
for i in range(1, len(logz[modelsource])):
logztype[modelsource] = np.logaddexp(logztype[modelsource],
logz[modelsource][i])
#-------------------------------------------------------------------------------
# define the total evidence (final denominator in Bayes theorem) and then
# the classification probabilities
logzall = np.logaddexp(np.logaddexp(logztype['Ia'], logztype['Ibc']),
logztype['II'])
pIa = np.exp(logztype['Ia'] - logzall)
pIbc = np.exp(logztype['Ibc'] - logzall)
pII = np.exp(logztype['II'] - logzall)
outdict['pIa'] = pIa
outdict['pIbc'] = pIbc
outdict['pII'] = pII
outdict['logztype'] = logztype
outdict['logzall'] = logzall
if (verbose):
print("pIa: "),
print(pIa)
print("pIbc: "),
print(pIbc)
print("pII: "),
print(pII)
return outdict
def curveToColor(lc,
colors,
bandFit=None,
snType='II',
bandDict=_filters,
color_bands=_opticalBands,
zpsys='AB',
model=None,
singleBand=False,
verbose=True,
**kwargs):
"""
Function takes a lightcurve file and creates a color table for it.
:param lc: Name of lightcurve file you want to read, or astropy Table containing data
:type lc: str or astropy.Table
:param colors: Colors you want to calculate for the given SN (i.e U-B, r'-J)
:type colors: str or list of strings
:param bandFit: If there is a specific band you would like to fit instead of default
:type bandFit: str,optional
:param snType: Classification of SN
:type snType: str,optional
:param bandDict: sncosmo bandpass for each band used in the fitting/table generation
:type bandDict: dict,optional
:param color_bands: bands making up the known component of chosen colors
:type color_bands: list,optional
:param zpsys: magnitude system (i.e. AB or Vega)
:type zpsys: str,optional
:param model: If there is a specific sncosmo model you would like to fit with, otherwise all models mathcing the SN classification will be tried
:type model: str,optional
:param singleBand: If you would like to only fit the bands in the color
:type singleBand: Boolean,optional
:param verbose: If you would like printing information to be turned on/off
:type verbose: Boolean,optional
:param kwargs: Catches all SNCOSMO fitting parameters here
:returns: colorTable: Astropy Table object containing color information
"""
bands = append([col[0] for col in colors], [col[-1] for col in colors])
for band in _filters:
if band not in bandDict.keys() and band in bands:
bandDict[band] = sncosmo.get_bandpass(_filters[band])
if not isinstance(colors, (tuple, list)):
colors = [colors]
zpMag = sncosmo.get_magsystem(zpsys)
if isinstance(lc, str):
curve = _standardize(sncosmo.read_lc(lc))
else:
try:
curve = _standardize(lc)
except:
raise RuntimeError("Can't understand your lightcurve.")
if _get_default_prop_name('zpsys') not in curve.colnames:
curve[_get_default_prop_name('zpsys')] = zpsys
colorTable = Table(masked=True)
colorTable.add_column(Column(data=[], name=_get_default_prop_name('time')))
for band in bandDict:
if not isinstance(bandDict[band], sncosmo.Bandpass):
bandDict = _bandCheck(bandDict, band)
t0 = None
if verbose:
print('Getting best fit for: ' + ','.join(colors))
args = []
for p in _fittingParams:
args.append(kwargs.get(p, _fittingParams[p]))
if p == 'method':
try:
importlib.import_module(_fittingPackages[args[-1]])
except RuntimeError:
sys.exit()
for color in colors: #start looping through desired colors
if bandDict[color[
0]].wave_eff < _UVrightBound: #then extrapolating into the UV from optical
if not bandFit:
bandFit = color[-1]
if singleBand:
color_bands = [color[-1]]
blue = curve[curve[_get_default_prop_name('band')] ==
color[0]] #curve on the blue side of current color
red = curve[[
x in color_bands for x in curve[_get_default_prop_name('band')]
]] #curve on the red side of current color
else: #must be extrapolating into the IR
if not bandFit:
bandFit = color[0]
if singleBand:
color_bands = [color[0]]
blue = curve[[
x in color_bands for x in curve[_get_default_prop_name('band')]
]]
red = curve[curve[_get_default_prop_name('band')] == color[-1]]
if len(blue) == 0 or len(red) == 0:
if verbose:
print('Asked for color %s but missing necessary band(s)' %
color)
bandFit = None
continue
btemp = [
bandDict[blue[_get_default_prop_name('band')][i]].name
for i in range(len(blue))
]
rtemp = [
bandDict[red[_get_default_prop_name('band')][i]].name
for i in range(len(red))
]
blue.remove_column(_get_default_prop_name('band'))
blue[_get_default_prop_name('band')] = btemp
red.remove_column(_get_default_prop_name('band'))
red[_get_default_prop_name('band')] = rtemp
#now make sure we have zero-points and fluxes for everything
if _get_default_prop_name('zp') not in blue.colnames:
blue[_get_default_prop_name('zp')] = [
zpMag.band_flux_to_mag(1 / sncosmo.constants.HC_ERG_AA,
blue[_get_default_prop_name('band')][i])
for i in range(len(blue))
]
if _get_default_prop_name('zp') not in red.colnames:
red[_get_default_prop_name('zp')] = [
zpMag.band_flux_to_mag(1 / sncosmo.constants.HC_ERG_AA,
red[_get_default_prop_name('band')][i])
for i in range(len(red))
]
if _get_default_prop_name('flux') not in blue.colnames:
blue = mag_to_flux(blue, bandDict, zpsys)
if _get_default_prop_name('flux') not in red.colnames:
red = mag_to_flux(red, bandDict, zpsys)
if not t0: #this just ensures we only run the fitting once
if not model:
if verbose:
print('No model provided, running series of models.')
mod, types = loadtxt(os.path.join(__dir__, 'data', 'sncosmo',
'models.ref'),
dtype='str',
unpack=True)
modDict = {mod[i]: types[i] for i in range(len(mod))}
if snType != 'Ia':
mods = [
x for x in sncosmo.models._SOURCES._loaders.keys()
if x[0] in modDict.keys()
and modDict[x[0]][:len(snType)] == snType
]
elif snType == 'Ia':
mods = [
x for x in sncosmo.models._SOURCES._loaders.keys()
if 'salt2' in x[0]
]
mods = {
x[0] if isinstance(x, (tuple, list)) else x
for x in mods
}
if bandFit == color[0] or len(blue) > len(red):
args[0] = blue
if len(blue) > 60:
fits = []
for mod in mods:
fits.append(_snFit([mod] + args))
else:
fits = pyParz.foreach(mods, _snFit, args)
fitted = blue
notFitted = red
fit = color[0]
elif bandFit == color[-1] or len(blue) < len(red):
args[0] = red
'''
data,temp= sncosmo_fitting.cut_bands(photometric_data(red), sncosmo.Model(tempMod),
z_bounds=all_bounds.get('z', None),
warn=True)
print(data.fluxerr)
cov = diag(data.fluxerr**2) if data.fluxcov is None else data.fluxcov
invcov = linalg.pinv(cov)
args.append(invcov)
sys.exit()
fits=pyParz.foreach(mods,_snFit,args)
args.pop()
'''
if len(red) > 60:
fits = []
for mod in mods:
fits.append(_snFit([mod] + args))
else:
fits = pyParz.foreach(mods, _snFit, args)
fitted = red
notFitted = blue
fit = color[-1]
else:
raise RuntimeError(
'Neither band "%s" nor band "%s" has more points, and you have not specified which to fit.'
% (color[0], color[-1]))
bestChisq = inf
for f in fits:
if f:
res, mod = f
if res.chisq < bestChisq:
bestChisq = res.chisq
bestFit = mod
bestRes = res
if verbose:
print('Best fit model is "%s", with a Chi-squared of %f' %
(bestFit._source.name, bestChisq))
elif bandFit == color[0] or len(blue) > len(red):
args[0] = blue
bestRes, bestFit = _snFit(append(model, args))
fitted = blue
notFitted = red
fit = color[0]
if verbose:
print(
'The model you chose (%s) completed with a Chi-squared of %f'
% (model, bestRes.chisq))
elif bandFit == color[-1] or len(blue) < len(red):
args[0] = red
bestRes, bestFit = _snFit(append(model, args))
fitted = red
notFitted = blue
fit = color[-1]
if verbose:
print(
'The model you chose (%s) completed with a Chi-squared of %f'
% (model, bestRes.chisq))
else:
raise RuntimeError(
'Neither band "%s" nor band "%s" has more points, and you have not specified which to fit.'
% (color[0], color[-1]))
t0 = _getBandMaxTime(bestFit, fitted, bandDict, 'B',
zpMag.band_flux_to_mag(1, bandDict['B']),
zpsys)
if len(t0) == 1:
t0 = t0[0]
else:
raise RuntimeError('Multiple global maxima in best fit.')
else:
if len(blue) > len(red) or bandFit == color[0]:
fitted = blue
notFitted = red
fit = color[0]
elif len(blue) < len(red) or bandFit == color[-1]:
fitted = red
notFitted = blue
fit = color[-1]
else:
raise RuntimeError(
'Neither band "%s" nor band "%s" has more points, and you have not specified which to fit.'
% (color[0], color[-1]))
#return(bestFit,bestRes,t0,fitted,notFitted)
tGrid, bestMag = _snmodel_to_mag(bestFit, fitted, zpsys, bandDict[fit])
ugrid, UMagErr, lgrid, LMagErr = _getErrorFromModel(
append([bestFit._source.name, fitted], args[1:]), zpsys,
bandDict[fit])
tempTable = Table(
[tGrid - t0, bestMag, bestMag * .1],
names=(_get_default_prop_name('time'),
_get_default_prop_name('mag'),
_get_default_prop_name('magerr')
)) #****RIGHT NOW THE ERROR IS JUST SET TO 10%*****
interpFunc = scint.interp1d(
array(tempTable[_get_default_prop_name('time')]),
array(tempTable[_get_default_prop_name('mag')]))
minterp = interpFunc(
array(notFitted[_get_default_prop_name('time')] - t0))
interpFunc = scint.interp1d(ugrid - t0, UMagErr)
uinterp = interpFunc(
array(notFitted[_get_default_prop_name('time')] - t0))
interpFunc = scint.interp1d(lgrid - t0, LMagErr)
linterp = interpFunc(
array(notFitted[_get_default_prop_name('time')] - t0))
magerr = mean([minterp - uinterp, linterp - minterp], axis=0)
for i in range(len(minterp)):
colorTable.add_row(append(
notFitted[_get_default_prop_name('time')][i] - t0,
[1 for j in range(len(colorTable.colnames) - 1)]),
mask=[
True if j > 0 else False
for j in range(len(colorTable.colnames))
])
if fit == color[0]:
colorTable[color] = MaskedColumn(
append([1 for j in range(len(colorTable) - len(minterp))],
minterp -
array(notFitted[_get_default_prop_name('mag')])),
mask=[
True if j < (len(colorTable) - len(minterp)) else False
for j in range(len(colorTable))
])
else:
colorTable[color] = MaskedColumn(
append([1 for j in range(len(colorTable) - len(minterp))],
array(notFitted[_get_default_prop_name('mag')]) -
minterp),
mask=[
True if j < (len(colorTable) - len(minterp)) else False
for j in range(len(colorTable))
])
colorTable[color[0] + color[-1] + '_err'] = MaskedColumn(
append([1 for j in range(len(colorTable) - len(magerr))], magerr +
array(notFitted[_get_default_prop_name('magerr')])),
mask=[
True if j < (len(colorTable) - len(magerr)) else False
for j in range(len(colorTable))
])
#colorTable['V-r']=MaskedColumn(append([1 for j in range(len(colorTable)-len(magerr))],[bestFit.color('bessellv','sdss::r',zpsys,t0) for i in range(len(linterp))]),mask=[True if j<(len(colorTable)-len(magerr)) else False for j in range(len(colorTable))])
tempVRCorr = 0
for name in bestFit.effect_names:
magCorr = _unredden(
color, bandDict,
bestRes.parameters[bestRes.param_names.index(name + 'ebv')],
bestRes.parameters[bestRes.param_names.index(name + 'r_v')])
colorTable[color] -= magCorr
tempVRCorr += _unredden(
'V-R', bandDict,
bestRes.parameters[bestRes.param_names.index(name + 'ebv')],
bestRes.parameters[bestRes.param_names.index(name + 'r_v')])
corr1 = _ccm_extinction(
sncosmo.get_bandpass('besselli').wave_eff,
bestRes.parameters[bestRes.param_names.index(name + 'ebv')],
r_v=3.1)
corr2 = _ccm_extinction(
sncosmo.get_bandpass('bessellr').wave_eff,
bestRes.parameters[bestRes.param_names.index(name + 'ebv')],
r_v=3.1)
corr3 = _ccm_extinction(
sncosmo.get_bandpass('bessellb').wave_eff,
bestRes.parameters[bestRes.param_names.index(name + 'ebv')],
r_v=3.1)
corr4 = _ccm_extinction(
sncosmo.get_bandpass('bessellv').wave_eff,
bestRes.parameters[bestRes.param_names.index(name + 'ebv')],
r_v=3.1)
vr = [
x - tempVRCorr
for x in bestFit.color('bessellv', 'bessellr', zpsys,
arange(t0 - 20, t0 + 100, 1))
]
#vr=(bestFit.bandmag('bessellr',zpsys,arange(t0-20,t0+100,1))-bestFit.bandmag('besselli',zpsys,arange(t0-20,t0+100,1))-(corr2-corr1))/(bestFit.bandmag('bessellb',zpsys,arange(t0-20,t0+100,1))-bestFit.bandmag('bessellv',zpsys,arange(t0-20,t0+100,1))-(corr3-corr4))
bandFit = None
colorTable.sort(_get_default_prop_name('time'))
return (colorTable, vr)
import pyParz
indata = np.arange(0, 100, 1)
def par_func1(x):
time.sleep(.1)
return (x**2)
start = time.time()
res = [par_func1(x) for x in indata]
end = time.time()
print('without par', end - start)
start = time.time()
res = pyParz.foreach(indata, par_func1, args=None)
end = time.time()
print('with par', end - start)
indata2 = 1
def par_func2(args):
x, y = args
time.sleep(.1)
return (x**2 + y)
start = time.time()
res = [par_func2([x, indata2]) for x in indata]
end = time.time()