def __init__(self, name, ra=None, dec=None, redshift=None, classification=None, sfd_dir=None, **kwargs): """ """ kwargs = self._load_config_(**kwargs) self.name = name self.redshift = redshift self.classification = classification self.sfd_dir = sfd_dir self.filter_dict = kwargs.pop('filter_dict', _DEFAULT_FILTERS) if ra is not None and dec is not None: self.ra = ra self.dec = dec if _HAS_SFDMAP: if self.sfd_dir is None: self.dustmap = sfdmap.SFDMap() else: self.dustmap = sfdmap.SFDMap(self.sfd_dir) self.mwebv = self.dustmap.ebv(ra, dec) else: self.mwebv = 0.0 r = requests.post('http://skipper.caltech.edu:8080/cgi-bin/growth/print_lc.cgi', auth=(self.user, self.passwd), data={'name': self.name}) r = r.text.split('<table border=0 width=850>')[-1] r = r.replace(' ', '').replace('\n', '') r = '\n'.join(r.split('<br>')) self.table_orig = Table.read(r, format='ascii.csv') self._remove_duplicates_()
def dustmap(self): """Instance of SFD98 dust map""" if self._dustmap is not None: return self._dustmap elif _HAS_SFDMAP: if self.sfd_dir is None: if os.environ.get('SFD_DIR') is not None: self._dustmap = sfdmap.SFDMap() else: return None else: self._dustmap = sfdmap.SFDMap(self.sfd_dir) return self._dustmap else: return None
def calcSpecDF(filenames, grid, gs, data, specobjid): specDF = np.zeros((len(filenames), len(grid))) ebvMap = sfdmap.SFDMap('E:\DB\sfddata-master') for i in xrange(len(filenames)): print('Calculating Spectrum: '), gi = gs.iloc[i] wli = 10**data[i]['loglam'] fluxi = data[i]['flux'] ivari = data[i]['ivar'] assert specobjid[i] == gi[ 'specobjid'], 'Files do not match galaxies dataframe' ra = gi['ra'] dec = gi['dec'] ebvi = ebvMap.ebv(ra, dec) speci = deredden_spectrum(wli, fluxi, ebvi) speci = remove_bad_pixels(speci, ivari) speci = zero_to_nan(speci) wli = de_redshift(wli, gi['z']) speci = same_grid_single(grid, wli, speci) speci = medfilt(speci) speci = norm_spectrum(speci) specDF[i] = speci print(str(round(100 * (i + 1) / float(len(filenames)), 3)) + '%') return specDF
def extinction(ra, dec): dustmap = sfdmap.SFDMap( "/Users/bhagyasubrayan/Desktop/sncosmo/sfddata-master") c = SkyCoord(ra + dec, unit=(u.hourangle, u.deg)) ebv = dustmap.ebv(c) Ar = 2.751 * ebv #1998 prescription Ag = 3.793 * ebv Ai = 2.086 * ebv Az = 1.479 * ebv return Ag, Ar, Ai, Az, ebv
def reduce_mw_extinct(self): ''' 银河消光改正 ''' wav = self.wavelen_list flux = self.flux_list flux_err = self.flux_e_list m = sfdmap.SFDMap('./sfddata-master') ebv = m.ebv(self.get_ra(), self.get_dec()) R_V = 3.1 A_V = ebv * R_V A_lam = ccm89(wav, A_V, R_V) fac = 10**(-0.4 * A_lam) flux_deredden = flux / fac flux_err_deredden = flux_err / fac self.flux_list = flux_deredden self.flux_e_list = flux_err_deredden return self
def add_sfd(): m = sfdmap.SFDMap("sfddata-master") # Must be installed separately x1d = linspace(-pi, pi, 400) y1d = linspace(-pi / 2., pi / 2, 200) xvals, yvals = meshgrid(x1d, y1d) zvals = xvals * 0. for i in range(len(x1d)): for j in range(len(y1d)): zvals[j, i] = m.ebv(x1d[i] * degrad, y1d[j] * degrad) CS = plt.contourf(-xvals, yvals, zvals, levels=[0, 0.01, 0.02, 0.03], colors=[(1, 0, 0), (1, 0.5, 0.5), (1, 0.75, 0.75), (1., 0.9, 0.9)], alpha=0.3)
""" import h5py import numpy as np import pandas as pds import astropy.wcs as awc import astropy.io.ascii as asc import astropy.io.fits as fits import subprocess as subpro from extinction_redden import A_wave from astropy.coordinates import SkyCoord #. dust map with the recalibration by Schlafly & Finkbeiner (2011) import sfdmap E_map = sfdmap.SFDMap('/home/xkchen/module/dust_map/sfddata_maskin') #. dust map of SFD 1998 # from dustmaps.sfd import SFDQuery # sfd = SFDQuery() from mpi4py import MPI commd = MPI.COMM_WORLD rank = commd.Get_rank() cpus = commd.Get_size() #.pipe process from img_resample import resamp_func from BCG_SB_pro_stack import BCG_SB_pros_func, single_img_SB_func from fig_out_module import arr_jack_func from light_measure import light_measure_weit
ObsHistID = ObsHistID.astype(np.int) print(MJDs) assert np.all(np.diff(MJDs) >= 0.) assert np.all(np.diff(ObsHistID) >= 0.) assert np.all(np.diff(_ObsHistID) >= 0.) ## Closing the connection to the database file conn.close() WFDFIELDIDS = [] ## By default, a scaling of 0.86 is applied to the map values to reflect the recalibration by Schlafly & Finkbeiner (2011) ## https://github.com/kbarbary/sfdmap. sfd = sfdmap.SFDMap('/global/homes/m/mjwilson/SFD/') YEAR = 5 _ObsHistID = _ObsHistID[:np.round(YEAR * 0.1 * len(_ObsHistID)).astype(np.int)] for i, x in enumerate(ObsHistID): if x in _ObsHistID: WFDFIELDIDS.append(FieldIDs[i]) print(100. * i / len(ObsHistID)) for i, id in enumerate(_FieldID): if id in WFDFIELDIDS: ## RA, DEC [degrees]. ebv = sfd.ebv(_FieldRa[i], _FieldDec[i])
def do(self): """ Predicts photometric redshifts from RA and DEC points in SDSS An outline of the algorithem is: first pull from SDSS u,g,r,i,z magnitudes from SDSS; should be able to handle a list/array of RA and DEC place u,g,r,i,z into a vector, append the derived information into the data array predict the information from the model return the predictions in the same order to the user inputs: Ra: list or array of len N, right ascensions of target galaxies in decimal degrees Dec: list or array of len N, declination of target galaxies in decimal degrees search: float, arcmin tolerance to search for the object in SDSS Catalogue path_to_model: str, filepath to saved model for prediction Returns: predictions: array of len N, photometric redshift of input galaxy """ try: nowdate = datetime.datetime.utcnow() - datetime.timedelta(1) from django.db.models import Q #HAS To Remain Here, I dunno why print('Entered the photo_z Pan-Starrs CNN cron') #save time b/c the other cron jobs print a time for completion m = sfdmap.SFDMap(mapdir=djangoSettings.STATIC_ROOT+'/sfddata-master') model_filepath = djangoSettings.STATIC_ROOT+'PS_7_01_ultimate.hdf5' #fill this in with one NB_BINS = 270 BATCH_SIZE = 64 ZMIN = 0.0 ZMAX = 0.6 BIN_SIZE = (ZMAX - ZMIN) / NB_BINS range_z = np.linspace(ZMIN, ZMAX, NB_BINS + 1)[:NB_BINS] transients = Transient.objects.filter(Q(host__isnull=False)) my_index = np.array(range(0,len(transients))) #dummy index used in DF, then used to create a mapping from matched galaxies back to these hosts transient_dictionary = dict(zip(my_index,transients)) #another script will place the images into the data model, just pull them out here... #print('Original length of Transients: ',len(transients)) for i,T in enumerate(transients): #get rid of fake entries, or entries not yet classified or given a PS cutout ID = T.name if not(os.path.isfile(djangoSettings.STATIC_ROOT+'/cutouts/cutout_{}_{}.npy'.format(ID,'g'))) or not(os.path.isfile(djangoSettings.STATIC_ROOT+'/cutouts/cutout_{}_{}.npy'.format(ID,'r'))) or not(os.path.isfile(djangoSettings.STATIC_ROOT+'/cutouts/cutout_{}_{}.npy'.format(ID,'i'))) or not(os.path.isfile(djangoSettings.STATIC_ROOT+'/cutouts/cutout_{}_{}.npy'.format(ID,'z'))) or not(os.path.isfile(djangoSettings.STATIC_ROOT+'/cutouts/cutout_{}_{}.npy'.format(ID,'y'))): #check if cutout exists from that transient name in dir... #It would be simpler maybe to just go around the data model, make a dictionary for the images in the cutout folder and grab them from there... transient_dictionary.pop(i) #if I don't have the image in the data model, drop that transient, wait until the other cron does its job. #print('attempting to create model') mymodel = create_model_groundup_decay_ultimate(NB_BINS) #print('attempting to load model') mymodel.load_weights(model_filepath) Number = 1000 #!!! #first do a modulo to find how many 1000's exist in the transient dictionary outer_loop_how_many = len(transient_dictionary)//Number remainder_how_many = len(transient_dictionary)%Number #next take these numbers to make a list of the keys to transient_dictionary, then query parts of that list in turn... whats_left = list(transient_dictionary.keys()) #list holding the keys to transient after removing bad entries... #create a holding array for predictions posterior = np.zeros((len(whats_left),NB_BINS)) #(how many unique hosts,how many bins of discrete redshift) for outer_index in range(outer_loop_how_many): use_these_indices = whats_left[outer_index*Number:(outer_index+1)*Number] #now reset the data array DATA = np.zeros((Number,104,104,5)) #safe because if not 1000 would have //1000 = 0 and range(0) is empty [] #Now reset the ra and dec lists so I can grab more extinctions RA = [] DEC = [] for inner_index,value in enumerate(use_these_indices): T = transient_dictionary[value] ID = T.name for j,F in enumerate(['g','r','i','z','y']): DATA[inner_index,:,:,j] = np.load(djangoSettings.STATIC_ROOT+'/cutouts/cutout_{}_{}.npy'.format(ID,F)) RA.append(T.host.ra) DEC.append(T.host.dec) #Next run model and place into posterior Extinctions = m.ebv(np.array(RA),np.array(DEC)) posterior[outer_index*Number:(outer_index+1)*Number] = mymodel.predict([DATA,Extinctions]) #next use the remainder use_these_indices = whats_left[((outer_loop_how_many)*Number):((outer_loop_how_many)*Number)+remainder_how_many] #Reset Data, only create array large enough to hold whats left DATA = np.zeros((len(use_these_indices),104,104,5)) #reset ra,dec RA = [] DEC = [] for inner_index,value in enumerate(use_these_indices): T = transient_dictionary[value] ID = T.name for j,F in enumerate(['g','r','i','z','y']): DATA[inner_index,:,:,j] = np.load(djangoSettings.STATIC_ROOT+'/cutouts/cutout_{}_{}.npy'.format(ID,F)) RA.append(T.host.ra) DEC.append(T.host.dec) Extinctions = m.ebv(np.array(RA),np.array(DEC)) posterior[((outer_loop_how_many)*Number):((outer_loop_how_many)*Number)+remainder_how_many] = mymodel.predict([DATA,Extinctions]) #Now posterior should be full! #print('done') point_estimates = np.sum(range_z*posterior,1) error = np.ones(len(point_estimates)) for i in range(len(point_estimates)): error[i] = np.std(np.random.choice(a=range_z,size=1000,p=posterior[i,:]/np.sum(posterior[i,:]),replace=True)) #this could be parallized i'm sure for i,value in enumerate(list(transient_dictionary.keys())): T = transient_dictionary[value] if not(T.host.photo_z_PSCNN): T.host.photo_z_PSCNN = point_estimates[i] T.host.photo_z_err_PSCNN = error[i] #T.host.photo_z_posterior_PSCNN = posterior[i] #Gautham suggested we add it to the host model #T.host.photo_z_source = 'YSE internal PS CNN' #this is not neccessary its in the name T.host.save() #takes a long time and then my query needs to be reset which is also a long time except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print("""PS Photo-z cron failed with error %s at line number %s"""%(e,exc_tb.tb_lineno))
from scipy.interpolate import interp1d import numpy from sklearn.preprocessing import Imputer from scipy import signal import sfdmap global m m = sfdmap.SFDMap('sfddata-master/') #wget https://github.com/kbarbary/sfddata/archive/master.tar.gz #tar xzf master.tar.gz def deredden_spectrum(wl, spec, E_bv): """ function dereddens a spectrum based on the given extinction_g value and Fitzpatric99 model IMPORTANT: the spectrum should be in the observer frame (do not correct for redshift) """ # dust model wls = numpy.array([2600, 2700, 4110, 4670, 5470, 6000, 12200, 26500]) a_l = numpy.array([6.591, 6.265, 4.315, 3.806, 3.055, 2.688, 0.829, 0.265]) f_interp = interp1d(wls, a_l, kind="cubic") a_l_all = f_interp(wl) #E_bv = extinction_g / 3.793 A_lambda = E_bv * a_l_all spec_real = spec * 10**(A_lambda / 2.5) return spec_real
def preprocess(DF, PATH='../DATA/sfddata-master/', ebv=True): if ebv: m = sfdmap.SFDMap(PATH) EBV = m.ebv(DF['raMean'].values.astype(np.float32), DF['decMean'].values.astype(np.float32)) DF['ebv'] = EBV else: DF['ebv'] = 0.0 def convert_flux_to_luptitude(f, b, f_0=3631): return -2.5 / np.log(10) * (np.arcsinh( (f / f_0) / (2 * b)) + np.log(b)) b_g = 1.7058474723241624e-09 b_r = 4.65521985283191e-09 b_i = 1.2132217745483221e-08 b_z = 2.013446972858555e-08 b_y = 5.0575501316874416e-08 MEANS = np.array([ 18.70654578, 17.77948707, 17.34226094, 17.1227873, 16.92087669, 19.73947441, 18.89279411, 18.4077393, 18.1311733, 17.64741402, 19.01595669, 18.16447837, 17.73199409, 17.50486095, 17.20389615, 19.07834251, 18.16996592, 17.71492073, 17.44861273, 17.15508793, 18.79100201, 17.89569908, 17.45774026, 17.20338482, 16.93640741, 18.62759241, 17.7453392, 17.31341498, 17.06194499, 16.79030564, 0.02543223 ]) STDS = np.array([ 1.7657395, 1.24853534, 1.08151972, 1.03490545, 0.87252421, 1.32486758, 0.9222839, 0.73701807, 0.65002723, 0.41779001, 1.51554956, 1.05734494, 0.89939638, 0.82754093, 0.63381611, 1.48411417, 1.05425943, 0.89979008, 0.83934385, 0.64990996, 1.54735158, 1.10985163, 0.96460099, 0.90685922, 0.74507053, 1.57813401, 1.14290345, 1.00162105, 0.94634726, 0.80124359, 0.01687839 ]) data_columns = [ 'gFKronFlux', 'rFKronFlux', 'iFKronFlux', 'zFKronFlux', 'yFKronFlux', 'gFPSFFlux', 'rFPSFFlux', 'iFPSFFlux', 'zFPSFFlux', 'yFPSFFlux', 'gFApFlux', 'rFApFlux', 'iFApFlux', 'zFApFlux', 'yFApFlux', 'gFmeanflxR5', 'rFmeanflxR5', 'iFmeanflxR5', 'zFmeanflxR5', 'yFmeanflxR5', 'gFmeanflxR6', 'rFmeanflxR6', 'iFmeanflxR6', 'zFmeanflxR6', 'yFmeanflxR6', 'gFmeanflxR7', 'rFmeanflxR7', 'iFmeanflxR7', 'zFmeanflxR7', 'yFmeanflxR7', 'ebv' ] X = DF[data_columns].values.astype(np.float32) X[:, 0:30:5] = convert_flux_to_luptitude(X[:, 0:30:5], b=b_g) X[:, 1:30:5] = convert_flux_to_luptitude(X[:, 1:30:5], b=b_r) X[:, 2:30:5] = convert_flux_to_luptitude(X[:, 2:30:5], b=b_i) X[:, 3:30:5] = convert_flux_to_luptitude(X[:, 3:30:5], b=b_z) X[:, 4:30:5] = convert_flux_to_luptitude(X[:, 4:30:5], b=b_y) X = (X - MEANS) / STDS X[X > 20] = 20 X[X < -20] = -20 X[np.isnan(X)] = -20 return X