def do_process(teffs, loggs, ebvs, zs, rvs, index, arr): output = np.zeros((len(responses) + 1, len(teffs))) c0 = time.time() N = len(teffs) for i, (teff, logg, ebv, z, rv, ind) in enumerate(zip(teffs, loggs, ebvs, zs, rvs, index)): if i % 100 == 0: dt = time.time() - c0 print("ETA", index[0], (N - i) / 100. * dt / 3600., 'hr') c0 = time.time() #-- get model SED and absolute luminosity model.set_defaults(z=z) wave, flux = model.get_table(teff, logg) Labs = model.luminosity(wave, flux) flux_ = reddening.redden(flux, wave=wave, ebv=ebv, rtype='flux', law=law, Rv=rv) #-- calculate synthetic fluxes output[0, i] = ind output[1:, i] = model.synthetic_flux(wave, flux_, responses, units=units) arr.append(output)
def do_process(teffs,loggs,ebvs,zs,rvs,index,arr): output = np.zeros((len(responses)+1,len(teffs))) c0 = time.time() N = len(teffs) for i,(teff,logg,ebv,z,rv,ind) in enumerate(zip(teffs,loggs,ebvs,zs,rvs,index)): if i%100==0: dt = time.time()-c0 print "ETA",index[0],(N-i)/100.*dt/3600.,'hr' c0 = time.time() #-- get model SED and absolute luminosity model.set_defaults(z=z) wave,flux = model.get_table(teff,logg) Labs = model.luminosity(wave,flux) flux_ = reddening.redden(flux,wave=wave,ebv=ebv,rtype='flux',law=law,Rv=rv) #-- calculate synthetic fluxes output[0,i] = ind output[1:,i] = model.synthetic_flux(wave,flux_,responses,units=units) arr.append(output)
def calc_integrated_grid(threads=1, ebvs=None, law='fitzpatrick2004', Rv=3.1, units='Flambda', responses=None, update=False, add_spectrophotometry=False, **kwargs): """ Integrate an entire SED grid over all passbands and save to a FITS file. The output file can be used to fit SEDs more efficiently, since integration over the passbands has already been carried out. WARNING: this function can take a loooooong time to compute! Extra keywords can be used to specify the grid. @param threads: number of threads @type threads; integer, 'max', 'half' or 'safe' @param ebvs: reddening parameters to include @type ebvs: numpy array @param law: interstellar reddening law to use @type law: string (valid law name, see C{reddening.py}) @param Rv: Rv value for reddening law @type Rv: float @param units: choose to work in 'Flambda' or 'Fnu' @type units: str, one of 'Flambda','Fnu' @param responses: respons curves to add (if None, add all) @type responses: list of strings @param update: if true append to existing FITS file, otherwise overwrite possible existing file. @type update: boolean """ if ebvs is None: ebvs = np.r_[0:4.01:0.01] #-- select number of threads if threads == 'max': threads = cpu_count() elif threads == 'half': threads = cpu_count() / 2 elif threads == 'safe': threads = cpu_count() - 1 threads = int(threads) if threads > len(ebvs): threads = len(ebvs) logger.info('Threads: %s' % (threads)) #-- set the parameters for the SED grid model.set_defaults(**kwargs) #-- get the dimensions of the grid: both the grid points, but also # the wavelength range teffs, loggs = model.get_grid_dimensions() wave, flux = model.get_table(teff=teffs[0], logg=loggs[0]) #-- get the response functions covering the wavelength range of the models # also get the information on those filters responses = get_responses(responses=responses,\ add_spectrophotometry=add_spectrophotometry,wave=wave) #-- definition of one process: def do_ebv_process(ebvs, arr, responses): logger.debug('EBV: %s-->%s (%d)' % (ebvs[0], ebvs[-1], len(ebvs))) for ebv in ebvs: flux_ = reddening.redden(flux, wave=wave, ebv=ebv, rtype='flux', law=law, Rv=Rv) #-- calculate synthetic fluxes synflux = model.synthetic_flux(wave, flux_, responses, units=units) arr.append([np.concatenate(([ebv], synflux))]) logger.debug("Finished EBV process (len(arr)=%d)" % (len(arr))) #-- do the calculations c0 = time.time() output = np.zeros((len(teffs) * len(ebvs), 4 + len(responses))) start = 0 logger.info('Total number of tables: %i' % (len(teffs))) exceptions = 0 exceptions_logs = [] for i, (teff, logg) in enumerate(zip(teffs, loggs)): if i > 0: logger.info('%s %s %s %s: ET %d seconds' % (teff, logg, i, len(teffs), (time.time() - c0) / i * (len(teffs) - i))) #-- get model SED and absolute luminosity wave, flux = model.get_table(teff=teff, logg=logg) Labs = model.luminosity(wave, flux) #-- threaded calculation over all E(B-V)s processes = [] manager = Manager() arr = manager.list([]) all_processes = [] for j in range(threads): all_processes.append( Process(target=do_ebv_process, args=(ebvs[j::threads], arr, responses))) all_processes[-1].start() for p in all_processes: p.join() try: #-- collect the results and add them to 'output' arr = np.vstack([row for row in arr]) sa = np.argsort(arr[:, 0]) arr = arr[sa] output[start:start + arr.shape[0], :3] = teff, logg, Labs output[start:start + arr.shape[0], 3:] = arr start += arr.shape[0] except: logger.warning('Exception in calculating Teff=%f, logg=%f' % (teff, logg)) logger.debug('Exception: %s' % (sys.exc_info()[1])) exceptions = exceptions + 1 exceptions_logs.append(sys.exc_info()[1]) #-- make FITS columns gridfile = model.get_file() if os.path.isfile(os.path.basename(gridfile)): outfile = os.path.basename(gridfile) else: outfile = os.path.join(os.path.dirname(gridfile), 'i{0}'.format(os.path.basename(gridfile))) outfile = 'i{0}'.format(os.path.basename(gridfile)) outfile = os.path.splitext(outfile) outfile = outfile[0] + '_law{0}_Rv{1:.2f}'.format(law, Rv) + outfile[1] logger.info('Precaution: making original grid backup at {0}.backup'.format( outfile)) if os.path.isfile(outfile): shutil.copy(outfile, outfile + '.backup') output = output.T if not update or not os.path.isfile(outfile): cols = [ pf.Column(name='teff', format='E', array=output[0]), pf.Column(name='logg', format='E', array=output[1]), pf.Column(name='ebv', format='E', array=output[3]), pf.Column(name='Labs', format='E', array=output[2]) ] for i, photband in enumerate(responses): cols.append( pf.Column(name=photband, format='E', array=output[4 + i])) #-- make FITS columns but copy the existing ones else: hdulist = pf.open(outfile, mode='update') names = hdulist[1].columns.names cols = [ pf.Column(name=name, format='E', array=hdulist[1].data.field(name)) for name in names ] for i, photband in enumerate(responses): cols.append( pf.Column(name=photband, format='E', array=output[4 + i])) #-- make FITS extension and write grid/reddening specifications to header table = pf.new_table(pf.ColDefs(cols)) table.header.update('gridfile', os.path.basename(gridfile)) for key in sorted(model.defaults.keys()): key_ = (len(key) > 8) and 'HIERARCH ' + key or key table.header.update(key_, model.defaults[key]) for key in sorted(kwargs.keys()): key_ = (len(key) > 8) and 'HIERARCH ' + key or key table.header.update(key_, kwargs[key]) table.header.update('FLUXTYPE', units) table.header.update('REDLAW', law, 'interstellar reddening law') table.header.update('RV', Rv, 'interstellar reddening parameter') #-- make/update complete FITS file if not update or not os.path.isfile(outfile): if os.path.isfile(outfile): os.remove(outfile) logger.warning('Removed existing file: %s' % (outfile)) hdulist = pf.HDUList([]) hdulist.append(pf.PrimaryHDU(np.array([[0, 0]]))) hdulist.append(table) hdulist.writeto(outfile) logger.info("Written output to %s" % (outfile)) else: hdulist[1] = table hdulist.flush() hdulist.close() logger.info("Appended output to %s" % (outfile)) logger.warning('Encountered %s exceptions!' % (exceptions)) for i in exceptions_logs: print('ERROR') print(i)
def calc_integrated_grid(threads=1,ebvs=None,law='fitzpatrick2004',Rv=3.1, units='Flambda',responses=None,update=False,add_spectrophotometry=False,**kwargs): """ Integrate an entire SED grid over all passbands and save to a FITS file. The output file can be used to fit SEDs more efficiently, since integration over the passbands has already been carried out. WARNING: this function can take a loooooong time to compute! Extra keywords can be used to specify the grid. @param threads: number of threads @type threads; integer, 'max', 'half' or 'safe' @param ebvs: reddening parameters to include @type ebvs: numpy array @param law: interstellar reddening law to use @type law: string (valid law name, see C{reddening.py}) @param Rv: Rv value for reddening law @type Rv: float @param units: choose to work in 'Flambda' or 'Fnu' @type units: str, one of 'Flambda','Fnu' @param responses: respons curves to add (if None, add all) @type responses: list of strings @param update: if true append to existing FITS file, otherwise overwrite possible existing file. @type update: boolean """ if ebvs is None: ebvs = np.r_[0:4.01:0.01] #-- select number of threads if threads=='max': threads = cpu_count() elif threads=='half': threads = cpu_count()/2 elif threads=='safe': threads = cpu_count()-1 threads = int(threads) if threads > len(ebvs): threads = len(ebvs) logger.info('Threads: %s'%(threads)) #-- set the parameters for the SED grid model.set_defaults(**kwargs) #-- get the dimensions of the grid: both the grid points, but also # the wavelength range teffs,loggs = model.get_grid_dimensions() wave,flux = model.get_table(teff=teffs[0],logg=loggs[0]) #-- get the response functions covering the wavelength range of the models # also get the information on those filters responses = get_responses(responses=responses,\ add_spectrophotometry=add_spectrophotometry,wave=wave) #-- definition of one process: def do_ebv_process(ebvs,arr,responses): logger.debug('EBV: %s-->%s (%d)'%(ebvs[0],ebvs[-1],len(ebvs))) for ebv in ebvs: flux_ = reddening.redden(flux,wave=wave,ebv=ebv,rtype='flux',law=law,Rv=Rv) #-- calculate synthetic fluxes synflux = model.synthetic_flux(wave,flux_,responses,units=units) arr.append([np.concatenate(([ebv],synflux))]) logger.debug("Finished EBV process (len(arr)=%d)"%(len(arr))) #-- do the calculations c0 = time.time() output = np.zeros((len(teffs)*len(ebvs),4+len(responses))) start = 0 logger.info('Total number of tables: %i'%(len(teffs))) exceptions = 0 exceptions_logs = [] for i,(teff,logg) in enumerate(zip(teffs,loggs)): if i>0: logger.info('%s %s %s %s: ET %d seconds'%(teff,logg,i,len(teffs),(time.time()-c0)/i*(len(teffs)-i))) #-- get model SED and absolute luminosity wave,flux = model.get_table(teff=teff,logg=logg) Labs = model.luminosity(wave,flux) #-- threaded calculation over all E(B-V)s processes = [] manager = Manager() arr = manager.list([]) all_processes = [] for j in range(threads): all_processes.append(Process(target=do_ebv_process,args=(ebvs[j::threads],arr,responses))) all_processes[-1].start() for p in all_processes: p.join() try: #-- collect the results and add them to 'output' arr = np.vstack([row for row in arr]) sa = np.argsort(arr[:,0]) arr = arr[sa] output[start:start+arr.shape[0],:3] = teff,logg,Labs output[start:start+arr.shape[0],3:] = arr start += arr.shape[0] except: logger.warning('Exception in calculating Teff=%f, logg=%f'%(teff,logg)) logger.debug('Exception: %s'%(sys.exc_info()[1])) exceptions = exceptions + 1 exceptions_logs.append(sys.exc_info()[1]) #-- make FITS columns gridfile = model.get_file() if os.path.isfile(os.path.basename(gridfile)): outfile = os.path.basename(gridfile) else: outfile = os.path.join(os.path.dirname(gridfile),'i{0}'.format(os.path.basename(gridfile))) outfile = 'i{0}'.format(os.path.basename(gridfile)) outfile = os.path.splitext(outfile) outfile = outfile[0]+'_law{0}_Rv{1:.2f}'.format(law,Rv)+outfile[1] logger.info('Precaution: making original grid backup at {0}.backup'.format(outfile)) if os.path.isfile(outfile): shutil.copy(outfile,outfile+'.backup') output = output.T if not update or not os.path.isfile(outfile): cols = [pf.Column(name='teff',format='E',array=output[0]), pf.Column(name='logg',format='E',array=output[1]), pf.Column(name='ebv',format='E',array=output[3]), pf.Column(name='Labs',format='E',array=output[2])] for i,photband in enumerate(responses): cols.append(pf.Column(name=photband,format='E',array=output[4+i])) #-- make FITS columns but copy the existing ones else: hdulist = pf.open(outfile,mode='update') names = hdulist[1].columns.names cols = [pf.Column(name=name,format='E',array=hdulist[1].data.field(name)) for name in names] for i,photband in enumerate(responses): cols.append(pf.Column(name=photband,format='E',array=output[4+i])) #-- make FITS extension and write grid/reddening specifications to header table = pf.new_table(pf.ColDefs(cols)) table.header.update('gridfile',os.path.basename(gridfile)) for key in sorted(model.defaults.keys()): key_ = (len(key)>8) and 'HIERARCH '+key or key table.header.update(key_,model.defaults[key]) for key in sorted(kwargs.keys()): key_ = (len(key)>8) and 'HIERARCH '+key or key table.header.update(key_,kwargs[key]) table.header.update('FLUXTYPE',units) table.header.update('REDLAW',law,'interstellar reddening law') table.header.update('RV',Rv,'interstellar reddening parameter') #-- make/update complete FITS file if not update or not os.path.isfile(outfile): if os.path.isfile(outfile): os.remove(outfile) logger.warning('Removed existing file: %s'%(outfile)) hdulist = pf.HDUList([]) hdulist.append(pf.PrimaryHDU(np.array([[0,0]]))) hdulist.append(table) hdulist.writeto(outfile) logger.info("Written output to %s"%(outfile)) else: hdulist[1] = table hdulist.flush() hdulist.close() logger.info("Appended output to %s"%(outfile)) logger.warning('Encountered %s exceptions!'%(exceptions)) for i in exceptions_logs: print 'ERROR' print i