def do_process(teffs, loggs, ebvs, zs, rvs, index, arr):
output = np.zeros((len(responses) + 1, len(teffs)))
c0 = time.time()
N = len(teffs)
for i, (teff, logg, ebv, z, rv,
ind) in enumerate(zip(teffs, loggs, ebvs, zs, rvs, index)):
if i % 100 == 0:
dt = time.time() - c0
print("ETA", index[0], (N - i) / 100. * dt / 3600., 'hr')
c0 = time.time()
#-- get model SED and absolute luminosity
model.set_defaults(z=z)
wave, flux = model.get_table(teff, logg)
Labs = model.luminosity(wave, flux)
flux_ = reddening.redden(flux,
wave=wave,
ebv=ebv,
rtype='flux',
law=law,
Rv=rv)
#-- calculate synthetic fluxes
output[0, i] = ind
output[1:, i] = model.synthetic_flux(wave,
flux_,
responses,
units=units)
arr.append(output)
def do_process(teffs,loggs,ebvs,zs,rvs,index,arr):
output = np.zeros((len(responses)+1,len(teffs)))
c0 = time.time()
N = len(teffs)
for i,(teff,logg,ebv,z,rv,ind) in enumerate(zip(teffs,loggs,ebvs,zs,rvs,index)):
if i%100==0:
dt = time.time()-c0
print "ETA",index[0],(N-i)/100.*dt/3600.,'hr'
c0 = time.time()
#-- get model SED and absolute luminosity
model.set_defaults(z=z)
wave,flux = model.get_table(teff,logg)
Labs = model.luminosity(wave,flux)
flux_ = reddening.redden(flux,wave=wave,ebv=ebv,rtype='flux',law=law,Rv=rv)
#-- calculate synthetic fluxes
output[0,i] = ind
output[1:,i] = model.synthetic_flux(wave,flux_,responses,units=units)
arr.append(output)
def calc_integrated_grid(threads=1,
ebvs=None,
law='fitzpatrick2004',
Rv=3.1,
units='Flambda',
responses=None,
update=False,
add_spectrophotometry=False,
**kwargs):
"""
Integrate an entire SED grid over all passbands and save to a FITS file.
The output file can be used to fit SEDs more efficiently, since integration
over the passbands has already been carried out.
WARNING: this function can take a loooooong time to compute!
Extra keywords can be used to specify the grid.
@param threads: number of threads
@type threads; integer, 'max', 'half' or 'safe'
@param ebvs: reddening parameters to include
@type ebvs: numpy array
@param law: interstellar reddening law to use
@type law: string (valid law name, see C{reddening.py})
@param Rv: Rv value for reddening law
@type Rv: float
@param units: choose to work in 'Flambda' or 'Fnu'
@type units: str, one of 'Flambda','Fnu'
@param responses: respons curves to add (if None, add all)
@type responses: list of strings
@param update: if true append to existing FITS file, otherwise overwrite
possible existing file.
@type update: boolean
"""
if ebvs is None:
ebvs = np.r_[0:4.01:0.01]
#-- select number of threads
if threads == 'max':
threads = cpu_count()
elif threads == 'half':
threads = cpu_count() / 2
elif threads == 'safe':
threads = cpu_count() - 1
threads = int(threads)
if threads > len(ebvs):
threads = len(ebvs)
logger.info('Threads: %s' % (threads))
#-- set the parameters for the SED grid
model.set_defaults(**kwargs)
#-- get the dimensions of the grid: both the grid points, but also
# the wavelength range
teffs, loggs = model.get_grid_dimensions()
wave, flux = model.get_table(teff=teffs[0], logg=loggs[0])
#-- get the response functions covering the wavelength range of the models
# also get the information on those filters
responses = get_responses(responses=responses,\
add_spectrophotometry=add_spectrophotometry,wave=wave)
#-- definition of one process:
def do_ebv_process(ebvs, arr, responses):
logger.debug('EBV: %s-->%s (%d)' % (ebvs[0], ebvs[-1], len(ebvs)))
for ebv in ebvs:
flux_ = reddening.redden(flux,
wave=wave,
ebv=ebv,
rtype='flux',
law=law,
Rv=Rv)
#-- calculate synthetic fluxes
synflux = model.synthetic_flux(wave, flux_, responses, units=units)
arr.append([np.concatenate(([ebv], synflux))])
logger.debug("Finished EBV process (len(arr)=%d)" % (len(arr)))
#-- do the calculations
c0 = time.time()
output = np.zeros((len(teffs) * len(ebvs), 4 + len(responses)))
start = 0
logger.info('Total number of tables: %i' % (len(teffs)))
exceptions = 0
exceptions_logs = []
for i, (teff, logg) in enumerate(zip(teffs, loggs)):
if i > 0:
logger.info('%s %s %s %s: ET %d seconds' %
(teff, logg, i, len(teffs),
(time.time() - c0) / i * (len(teffs) - i)))
#-- get model SED and absolute luminosity
wave, flux = model.get_table(teff=teff, logg=logg)
Labs = model.luminosity(wave, flux)
#-- threaded calculation over all E(B-V)s
processes = []
manager = Manager()
arr = manager.list([])
all_processes = []
for j in range(threads):
all_processes.append(
Process(target=do_ebv_process,
args=(ebvs[j::threads], arr, responses)))
all_processes[-1].start()
for p in all_processes:
p.join()
try:
#-- collect the results and add them to 'output'
arr = np.vstack([row for row in arr])
sa = np.argsort(arr[:, 0])
arr = arr[sa]
output[start:start + arr.shape[0], :3] = teff, logg, Labs
output[start:start + arr.shape[0], 3:] = arr
start += arr.shape[0]
except:
logger.warning('Exception in calculating Teff=%f, logg=%f' %
(teff, logg))
logger.debug('Exception: %s' % (sys.exc_info()[1]))
exceptions = exceptions + 1
exceptions_logs.append(sys.exc_info()[1])
#-- make FITS columns
gridfile = model.get_file()
if os.path.isfile(os.path.basename(gridfile)):
outfile = os.path.basename(gridfile)
else:
outfile = os.path.join(os.path.dirname(gridfile),
'i{0}'.format(os.path.basename(gridfile)))
outfile = 'i{0}'.format(os.path.basename(gridfile))
outfile = os.path.splitext(outfile)
outfile = outfile[0] + '_law{0}_Rv{1:.2f}'.format(law, Rv) + outfile[1]
logger.info('Precaution: making original grid backup at {0}.backup'.format(
outfile))
if os.path.isfile(outfile):
shutil.copy(outfile, outfile + '.backup')
output = output.T
if not update or not os.path.isfile(outfile):
cols = [
pf.Column(name='teff', format='E', array=output[0]),
pf.Column(name='logg', format='E', array=output[1]),
pf.Column(name='ebv', format='E', array=output[3]),
pf.Column(name='Labs', format='E', array=output[2])
]
for i, photband in enumerate(responses):
cols.append(
pf.Column(name=photband, format='E', array=output[4 + i]))
#-- make FITS columns but copy the existing ones
else:
hdulist = pf.open(outfile, mode='update')
names = hdulist[1].columns.names
cols = [
pf.Column(name=name, format='E', array=hdulist[1].data.field(name))
for name in names
]
for i, photband in enumerate(responses):
cols.append(
pf.Column(name=photband, format='E', array=output[4 + i]))
#-- make FITS extension and write grid/reddening specifications to header
table = pf.new_table(pf.ColDefs(cols))
table.header.update('gridfile', os.path.basename(gridfile))
for key in sorted(model.defaults.keys()):
key_ = (len(key) > 8) and 'HIERARCH ' + key or key
table.header.update(key_, model.defaults[key])
for key in sorted(kwargs.keys()):
key_ = (len(key) > 8) and 'HIERARCH ' + key or key
table.header.update(key_, kwargs[key])
table.header.update('FLUXTYPE', units)
table.header.update('REDLAW', law, 'interstellar reddening law')
table.header.update('RV', Rv, 'interstellar reddening parameter')
#-- make/update complete FITS file
if not update or not os.path.isfile(outfile):
if os.path.isfile(outfile):
os.remove(outfile)
logger.warning('Removed existing file: %s' % (outfile))
hdulist = pf.HDUList([])
hdulist.append(pf.PrimaryHDU(np.array([[0, 0]])))
hdulist.append(table)
hdulist.writeto(outfile)
logger.info("Written output to %s" % (outfile))
else:
hdulist[1] = table
hdulist.flush()
hdulist.close()
logger.info("Appended output to %s" % (outfile))
logger.warning('Encountered %s exceptions!' % (exceptions))
for i in exceptions_logs:
print('ERROR')
print(i)
def calc_integrated_grid(threads=1,ebvs=None,law='fitzpatrick2004',Rv=3.1,
units='Flambda',responses=None,update=False,add_spectrophotometry=False,**kwargs):
"""
Integrate an entire SED grid over all passbands and save to a FITS file.
The output file can be used to fit SEDs more efficiently, since integration
over the passbands has already been carried out.
WARNING: this function can take a loooooong time to compute!
Extra keywords can be used to specify the grid.
@param threads: number of threads
@type threads; integer, 'max', 'half' or 'safe'
@param ebvs: reddening parameters to include
@type ebvs: numpy array
@param law: interstellar reddening law to use
@type law: string (valid law name, see C{reddening.py})
@param Rv: Rv value for reddening law
@type Rv: float
@param units: choose to work in 'Flambda' or 'Fnu'
@type units: str, one of 'Flambda','Fnu'
@param responses: respons curves to add (if None, add all)
@type responses: list of strings
@param update: if true append to existing FITS file, otherwise overwrite
possible existing file.
@type update: boolean
"""
if ebvs is None:
ebvs = np.r_[0:4.01:0.01]
#-- select number of threads
if threads=='max':
threads = cpu_count()
elif threads=='half':
threads = cpu_count()/2
elif threads=='safe':
threads = cpu_count()-1
threads = int(threads)
if threads > len(ebvs):
threads = len(ebvs)
logger.info('Threads: %s'%(threads))
#-- set the parameters for the SED grid
model.set_defaults(**kwargs)
#-- get the dimensions of the grid: both the grid points, but also
# the wavelength range
teffs,loggs = model.get_grid_dimensions()
wave,flux = model.get_table(teff=teffs[0],logg=loggs[0])
#-- get the response functions covering the wavelength range of the models
# also get the information on those filters
responses = get_responses(responses=responses,\
add_spectrophotometry=add_spectrophotometry,wave=wave)
#-- definition of one process:
def do_ebv_process(ebvs,arr,responses):
logger.debug('EBV: %s-->%s (%d)'%(ebvs[0],ebvs[-1],len(ebvs)))
for ebv in ebvs:
flux_ = reddening.redden(flux,wave=wave,ebv=ebv,rtype='flux',law=law,Rv=Rv)
#-- calculate synthetic fluxes
synflux = model.synthetic_flux(wave,flux_,responses,units=units)
arr.append([np.concatenate(([ebv],synflux))])
logger.debug("Finished EBV process (len(arr)=%d)"%(len(arr)))
#-- do the calculations
c0 = time.time()
output = np.zeros((len(teffs)*len(ebvs),4+len(responses)))
start = 0
logger.info('Total number of tables: %i'%(len(teffs)))
exceptions = 0
exceptions_logs = []
for i,(teff,logg) in enumerate(zip(teffs,loggs)):
if i>0:
logger.info('%s %s %s %s: ET %d seconds'%(teff,logg,i,len(teffs),(time.time()-c0)/i*(len(teffs)-i)))
#-- get model SED and absolute luminosity
wave,flux = model.get_table(teff=teff,logg=logg)
Labs = model.luminosity(wave,flux)
#-- threaded calculation over all E(B-V)s
processes = []
manager = Manager()
arr = manager.list([])
all_processes = []
for j in range(threads):
all_processes.append(Process(target=do_ebv_process,args=(ebvs[j::threads],arr,responses)))
all_processes[-1].start()
for p in all_processes:
p.join()
try:
#-- collect the results and add them to 'output'
arr = np.vstack([row for row in arr])
sa = np.argsort(arr[:,0])
arr = arr[sa]
output[start:start+arr.shape[0],:3] = teff,logg,Labs
output[start:start+arr.shape[0],3:] = arr
start += arr.shape[0]
except:
logger.warning('Exception in calculating Teff=%f, logg=%f'%(teff,logg))
logger.debug('Exception: %s'%(sys.exc_info()[1]))
exceptions = exceptions + 1
exceptions_logs.append(sys.exc_info()[1])
#-- make FITS columns
gridfile = model.get_file()
if os.path.isfile(os.path.basename(gridfile)):
outfile = os.path.basename(gridfile)
else:
outfile = os.path.join(os.path.dirname(gridfile),'i{0}'.format(os.path.basename(gridfile)))
outfile = 'i{0}'.format(os.path.basename(gridfile))
outfile = os.path.splitext(outfile)
outfile = outfile[0]+'_law{0}_Rv{1:.2f}'.format(law,Rv)+outfile[1]
logger.info('Precaution: making original grid backup at {0}.backup'.format(outfile))
if os.path.isfile(outfile):
shutil.copy(outfile,outfile+'.backup')
output = output.T
if not update or not os.path.isfile(outfile):
cols = [pf.Column(name='teff',format='E',array=output[0]),
pf.Column(name='logg',format='E',array=output[1]),
pf.Column(name='ebv',format='E',array=output[3]),
pf.Column(name='Labs',format='E',array=output[2])]
for i,photband in enumerate(responses):
cols.append(pf.Column(name=photband,format='E',array=output[4+i]))
#-- make FITS columns but copy the existing ones
else:
hdulist = pf.open(outfile,mode='update')
names = hdulist[1].columns.names
cols = [pf.Column(name=name,format='E',array=hdulist[1].data.field(name)) for name in names]
for i,photband in enumerate(responses):
cols.append(pf.Column(name=photband,format='E',array=output[4+i]))
#-- make FITS extension and write grid/reddening specifications to header
table = pf.new_table(pf.ColDefs(cols))
table.header.update('gridfile',os.path.basename(gridfile))
for key in sorted(model.defaults.keys()):
key_ = (len(key)>8) and 'HIERARCH '+key or key
table.header.update(key_,model.defaults[key])
for key in sorted(kwargs.keys()):
key_ = (len(key)>8) and 'HIERARCH '+key or key
table.header.update(key_,kwargs[key])
table.header.update('FLUXTYPE',units)
table.header.update('REDLAW',law,'interstellar reddening law')
table.header.update('RV',Rv,'interstellar reddening parameter')
#-- make/update complete FITS file
if not update or not os.path.isfile(outfile):
if os.path.isfile(outfile):
os.remove(outfile)
logger.warning('Removed existing file: %s'%(outfile))
hdulist = pf.HDUList([])
hdulist.append(pf.PrimaryHDU(np.array([[0,0]])))
hdulist.append(table)
hdulist.writeto(outfile)
logger.info("Written output to %s"%(outfile))
else:
hdulist[1] = table
hdulist.flush()
hdulist.close()
logger.info("Appended output to %s"%(outfile))
logger.warning('Encountered %s exceptions!'%(exceptions))
for i in exceptions_logs:
print 'ERROR'
print i
