def get_frame_center(filename=None, header=None):
if not header:
header = pyfits.getheader(filename, -1)
w = pywcs.WCS(header=header)
[ra1], [dec1] = w.all_pix2sky(0, 0, 1)
[ra0], [dec0] = w.all_pix2sky(header['NAXIS1'] / 2, header['NAXIS2'] / 2,
1)
sr = coords.sphdist(ra0, dec0, ra1, dec1)[0]
return ra0, dec0, sr
def pixelTasks(parameterListForPixel):
"""Workers will execute this function."""
#unpack the input parameter list that is to say separate them into different arrays
pixelNo, ra, dec, nObs, objID, medRa, medDec, resRa, resDec = parameterListForPixel
#define an index so that all the calculations can be put in place later
#ind =
global searchRadius
#find objects within the searchRadius and whose noOfObs is atleast 3
angSepMask = sphdist(ra, dec, medRa, medDec) <= (searchRadius*60) #& (nObs>=3.0) -- not needed since this has been taken care of while calculating residuals and medians previously
global triand
#separate these entire rows from the original testH5file
#we donot need to check for galaxies now, since the parameters being referred to have already been checked
#searchFile = triand[angSepMask]
objInRadius = objID[angSepMask]
k = triand[triand.get_where_list('obj_id == o') for o in objInRadius]
def check_planets(ra, dec, time, favor2=None):
favor2 = favor2 or Favor2()
for o in [ephem.Moon(), ephem.Venus(), ephem.Mars(), ephem.Jupiter()]:
favor2.obs.date = time
o.compute(favor2.obs)
sr = {
'Moon': 15.0,
'Venus': 0.5,
'Jupiter': 0.5,
'Mars': 0.1
}.get(o.name)
if coords.sphdist(np.rad2deg(o.ra), np.rad2deg(o.dec), ra,
dec)[0] < sr:
return True
return False
def get_frame_center(filename=None, header=None, wcs=None, width=None, height=None):
if not wcs:
if header:
wcs = WCS(header=header)
elif filename:
header = fits.getheader(filename, -1)
wcs = WCS(header=header)
if (not width or not height) and header:
width = header['NAXIS1']
height = header['NAXIS2']
[ra1],[dec1] = wcs.all_pix2world([0], [0], 1)
[ra0],[dec0] = wcs.all_pix2world([width/2], [height/2], 1)
sr = coords.sphdist(ra0, dec0, ra1, dec1)[0]
return ra0, dec0, sr
def close_match_radec(ra1,dec1,ra2,dec2,ep,allow,silent=False,box=False) :
"""
Find the nearest neighbors between two arrays of ra/dec
parameters
----------
ra1,dec1: scalar or array
coordinates of a set of points (degrees). Must be same length.
ra2,dec2: scalar or array
coordinates of a second set of points (degrees). Must be same length.
ep: scalar
maximum match distance between pairs (degrees)
allow: scalar
maximum number of matches in second array to each element in first array.
silent: boolean
make quiet
box: boolean
use square of size ep (default circle):
Original by Dave Johnston, University of Michigan, 1997
Translated from IDL by Eli Rykoff, SLAC
"""
ra1=np.atleast_1d(ra1)
dec1=np.atleast_1d(dec1)
ra2=np.atleast_1d(ra2)
dec2=np.atleast_1d(dec2)
epdec = ep
n1=ra1.size
n2=ra2.size
matcharr=np.zeros([n1,allow],dtype=np.int32)
matcharr.fill(-1)
ind=np.arange(n2,dtype=np.int32)
sor=ra2.argsort()
ra2sort=ra2[sor]
dec2sort=dec2[sor]
ind=ind[sor]
runi=0
endra2=ra2sort[n2-1]
for i in range(n1) :
epra=ep/np.cos(dec1[i]*0.01745329)
ra1minus = ra1[i]-epra
ra1plus = ra1[i]+epra
in1=_binary_search(ra2sort,ra1minus)
if (in1 == -1) :
if (ra1minus < endra2): in1=0
if (in1 != -1):
in2=in1
jj=in2+1
while (jj < n2):
if (ra2sort[in2+1] < ra1plus):
in2+=1
jj+=1
else:
jj=n2
if (n2 == 1):
in2 = 0
if (in1 <= in2) :
dec2check=dec2sort[in1:in2+1]
ra2check=ra2sort[in1:in2+1]
decoffby=np.abs(dec2check-dec1[i])
raoffby=np.abs(ra2check-ra1[i])
good=np.where((decoffby < epdec) & \
(raoffby < epra))[0]+in1
ngood=good.size
if (ngood != 0):
if (not box):
offby = coords.sphdist(ra1[i],dec1[i],\
ra2sort[good],dec2sort[good],\
units=['deg','deg'])
good_offby=np.where(offby <= ep)[0]
ngood = good_offby.size
else :
good_offby = np.arange(ngood)
offby = raoffby
if (ngood != 0) :
good = good[good_offby]
offby=offby[good_offby]
if (ngood > allow) :
good=good[offby.argsort()]
ngood = allow
good=good[0:allow]
matcharr[i,0:ngood]=good
runi=runi+ngood
if (not silent): print "total put in bytarr:",runi
matches=np.where(matcharr != -1)
if (matches[0].size == 0):
if (not silent):print "no matches found"
m1=np.array([-1])
m2=np.array([-1])
return (m1,m2)
m1=matches[0] % n1
m2=matcharr[matches]
m2=ind[m2].flatten()
if (not silent):print m1.size,' matches'
return (m1,m2)
import numpy as np from astropy.time import Time from time import time from esutil.coords import sphdist import healpy as hp noOfYears = 5 pixelWidth = 3 #arcminutes searchRadius = 10 #arcminutes NSIDE = 2**10 # with this NSIDE, I oversample the correction map by a factor of 5 tempArray = np.arange(hp.nside2npix(NSIDE)) theta, phi = hp.pix2ang(NSIDE, tempArray) pixelRa = 180*phi/np.pi pixelDec = 90 - theta*180/np.pi #take galaxies within searchRadius index = sphdist(testFile['ra'], testFile['dec'] <= searchRadius) useThisForUpdate = testFile[index] #count the number of observations #noOfObs = 0 #before the loop #noOfObs +=1 # if it goes to the else part
def process_file(filename, night=None, site=None, fram=None, verbose=False):
if fram is None:
fram = Fram()
if site is None:
# Simple heuristics to derive the site name
for _ in ['auger', 'cta-n', 'cta-s0', 'cta-s1']:
if _ in filename:
site = _
break
header = fits.getheader(filename)
if night is None:
time = parse_iso_time(header['DATE-OBS'])
if header.get('LONGITUD') is not None:
night = get_night(time, lon=header['LONGITUD'])
else:
night = get_night(time, site=site)
if verbose:
print(night, site, header['IMAGETYP'])
# Skip old master calibrations
if header['IMAGETYP'] in ['mdark', 'mflat']:
return None
# Skip frames acquired by rts2-scriptexec
if header.get('TARGET') is None or header.get('CCD_NAME') is None:
return None
image = fits.getdata(filename)
# Original (uncropped) dimensions
width, height = header['NAXIS1'], header['NAXIS2']
image, header = crop_overscans(image, header, subtract=False)
cropped_width, cropped_height = image.shape[1], image.shape[0]
# Clean up the header a bit
header.remove('HISTORY', remove_all=True, ignore_missing=True)
header.remove('COMMENT', remove_all=True, ignore_missing=True)
header.remove('', remove_all=True, ignore_missing=True)
for _ in header.keys():
if _ and _[0] == '_':
header.remove(_, remove_all=True, ignore_missing=True)
type = header.get('IMAGETYP', 'unknown')
if type == 'object' and header.get('CTYPE1'):
wcs = WCS(header)
ra, dec = wcs.all_pix2world([0, image.shape[1], 0.5 * image.shape[1]],
[0, image.shape[0], 0.5 * image.shape[0]],
0)
radius = 0.5 * coords.sphdist(ra[0], dec[0], ra[1], dec[1])[0]
ra0, dec0 = ra[2], dec[2]
# Frame footprint
ra, dec = wcs.all_pix2world([0, 0, image.shape[1], image.shape[1]],
[0, image.shape[0], image.shape[0], 0], 0)
footprint = "(" + ",".join(
["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")"
# Frame footprint at +10 pixels from the edge
ra, dec = wcs.all_pix2world(
[10, 10, image.shape[1] - 10, image.shape[1] - 10],
[10, image.shape[0] - 10, image.shape[0] - 10, 10], 0)
footprint10 = "(" + ",".join(
["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")"
else:
# Should we really discard WCS for non-object frames?
ra0, dec0, radius = 0, 0, 0
footprint, footprint10 = None, None
target = header.get('TARGET')
ccd = header.get('CCD_NAME')
serial = header.get('product_id')
filter = header.get('FILTER', 'unknown')
time = parse_iso_time(header['DATE-OBS'])
exposure = header.get('EXPOSURE')
binning = header.get('BINNING')
mean = np.mean(image)
median = np.median(image)
keywords = dict(header)
fram.query(
'INSERT INTO images (filename,night,time,target,type,filter,ccd,serial,site,ra,dec,radius,exposure,width,height,cropped_width,cropped_height,footprint,footprint10,binning,mean,median,keywords) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (filename) DO NOTHING',
(filename, night, time, target, type, filter, ccd, serial, site, ra0,
dec0, radius, exposure, width, height, cropped_width, cropped_height,
footprint, footprint10, binning, mean, median, keywords))
return {
'filename': filename,
'night': night,
'time': time,
'target': target,
'type': type,
'filter': filter,
'ccd': ccd,
'serial': serial,
'site': site,
'ra0': ra0,
'dec0': dec0,
'radius': radius,
'exposure': exposure,
'width': width,
'height': height,
'binning': binning,
'mean': mean,
'median': median
}
# finding best of everything (for min chi2)
p = period[np.argmin(chi2)]
tID = template[np.argmin(chi2)]
ch2 = chi2[np.argmin(chi2)]
chi2dof = ch2/(N-4)
phi_b = phi[np.argmin(chi2)]
r0_b = r0[np.argmin(chi2)]
F_b = F[np.argmin(chi2)]
if path in data_rrlyr:
# if star is RRLyrae
RRLInd = 1
P_template = true_params['P'][tID]
ang_sep = sphdist(ra, dec, table3['ra'], table3['dec'])*3600
ind = np.where(ang_sep < 1)[0]
rExt = table3['rExt'][ind][0]
d = table3['d'][ind][0]
uF = table3['uF'][ind][0]
gF = table3['gF'][ind][0]
rF = table3['rF'][ind][0]
iF = table3['iF'][ind][0]
zF = table3['zF'][ind][0]
ugmin = table3['ugmin'][ind][0]
ugminErr = table3['ugminErr'][ind][0]
grmin = table3['grmin'][ind][0]
grminErr = table3['grminErr'][ind][0]
if ind != None:
p_true = table2['P'][ind][0]
def pixelTasksCombinedData(parameterList):
#pixelNo is one number
#pixelRa and dec are the ra/dec values of the pixel in consideration
#galID are the object IDs of galaxies -- unique -- from the database
#ra/decFinalGal are final ra/dec values of galaxies obtained from the database
#galIDs -- galaxy IDs corresponding to all detections of galaxies
#ra/decGal -- original ra and dec values of galaxies
#starMjds -- mjd values corresponding to all the detections of stars
pixelNo, pixelRa, pixelDec, galIDfinal, raFinalGal, decFinalGal, galIDs, raGal, decGal, galMjd , starIDs, starMjds, raStar, decStar, pixAllStar, mjdSorted, mjdBreakAt = parameterList
angSepMask = (sphdist(pixelRa, pixelDec, raFinalGal, decFinalGal)) <= (searchRadius/60.0)
#select unique galaxy ids within searchRadius
uniqueGalIDinRadius = galIDfinal[angSepMask]
raFinalGalInRadius = raFinalGal[angSepMask]
#print "raFinalGalInRadius.size",raFinalGalInRadius.size
decFinalGalInRadius = decFinalGal[angSepMask]
a = []
b = []
c = []
d = []
e = []
for i in range(galIDs.size):
if galIDs[i] in uniqueGalIDinRadius:
a.append(galIDs[i])
b.append(i)
c.append(raGal[i])
d.append(decGal[i])
e.append(galMjd[i])
galIDinRadius = np.array(a)
raGalInRadius = np.array(c)
decGalInRadius = np.array(d)
galMjdInRadius = np.array(e)
#match all detections of galaxies with those in radius
#indexInRadius = np.in1d(galIDs,uniqueGalIDinRadius)
#obtain original ra and dec values for galaxies within radius
#raGalInRadius = raGal[indexInRadius]
#decGalInRadius = decGal[indexInRadius]
#galIDinRadius = galIDs[indexInRadius]
#print "galIDinRadius[1171070:1171083]",galIDinRadius[1171070:1171083]
#galMjdInRadius = galMjd[indexInRadius]
currentGalID = galIDinRadius[0]
currentRaGal = [raGalInRadius[0]]
currentDecGal = [decGalInRadius[0]]
galCounter = 0
position = 0
noOfDetectionsOfGal = galIDinRadius.size
print "noOfDetectionsOfGal",noOfDetectionsOfGal
offsetRaArray = np.zeros(noOfDetectionsOfGal)
offsetDecArray = np.zeros(noOfDetectionsOfGal)
#run over all the objects within the searchRadius --- i dont know why we did a -1 previously ! think/ask ! :( -- there is no need and then we donot need to think about the last object separately!
t = time()
for k in np.arange(noOfDetectionsOfGal-1)+1:
#print "on galaxy no", k
ID = galIDinRadius[k]
ra = raGalInRadius[k]
dec = decGalInRadius[k]
#print "ID", ID
if (ID == currentGalID):
currentRaGal.append(ra)
currentDecGal.append(dec)
#print "same object now"
else:
#make them numpy arrays
currentRaGal = np.array(currentRaGal)
currentDecGal = np.array(currentDecGal)
#calculate offsets
#print "k", k
#print "galCounter", galCounter
offsetRa = currentRaGal - raFinalGalInRadius[galCounter]
offsetDec = currentDecGal - decFinalGalInRadius[galCounter]
#print "offsetRa", offsetRa
#print "offsetDec", offsetDec
#store offsets
offsetRaArray[position:k] = offsetRa
offsetDecArray[position:k] = offsetDec
#update counters and variables
galCounter+= 1
position = k
currentGalID = ID
currentRaGal = [raGalInRadius[k]]
currentDecGal = [decGalInRadius[k]]
#print "time taken to calculate offsets of galaxies from their final ra/dec", time()-t
#print "galCounter after loop", galCounter
# select stars inside the pixel
resIndexInPixel = pixAllStar == pixelNo
#uniqueStarIDinPixel = starID[resIndexInPixel]
#match all detections of stars with those in the pixel
#indexInPixel = np.in1d(starIDs, uniqueStarIDinPixel)
#obtain original ra and dec values for stars within pixel
raStarInPixel = raStar[resIndexInPixel]
decStarInPixel = decStar[resIndexInPixel]
starIDInPixel = starIDs[resIndexInPixel]
mjdStarInPixel = starMjds[resIndexInPixel]
medianRaOffsetEpochWise = np.zeros(len(mjdBreakAt)+1)
medianDecOffsetEpochWise = np.zeros(len(mjdBreakAt)+1)
avgMjd = np.zeros(len(mjdBreakAt)+1)
if (len(mjdBreakAt)>= 3.0):
previousVar2 = np.min(mjdSorted)
for var1, var2 in enumerate(mjdBreakAt, start=0):
mjdIndex = (galMjdInRadius < var2) & (previousVar2 <galMjdInRadius)
mjdIndexInPixel = (mjdStarInPixel<var2) & (previousVar2 < mjdStarInPixel)
if any(mjdIndex):
offsetRaValues = offsetRaArray[mjdIndex]
offsetDecValues = offsetDecArray[mjdIndex]
medianRaOffsetEpochWise[var1] = np.median(offsetRaValues)
medianDecOffsetEpochWise[var1] = np.median(offsetDecValues)
#update the ra/dec - replace old values
raStarInPixel[mjdIndexInPixel] -= medianRaOffsetEpochWise[var1]
decStarInPixel[mjdIndexInPixel]-= medianDecOffsetEpochWise[var1]
avgMjd[var1] = (mjdStarInPixel[mjdIndexInPixel].max() - mjdStarInPixel[mjdIndexInPixel].min() )/2
else:
offsetRaValues = offsetRaArray[mjdIndex]
offsetDecValues = offsetDecArray[mjdIndex]
medianRaOffsetEpochWise[var1+1] = np.median(offsetRaValues)
medianDecOffsetEpochWise[var1+1] = np.median(offsetDecValues)
#update the ra/dec - replace old values
raStarInPixel[mjdIndexInPixel] -= medianRaOffsetEpochWise[var1]
decStarInPixel[mjdIndexInPixel]-= medianDecOffsetEpochWise[var1]
avgMjd[var1+1] = (mjdStarInPixel[mjdIndexInPixel].max() - mjdStarInPixel[mjdIndexInPixel].min() )/2
previousVar2 = var2
finalObjIDstar = starIDInPixel
finalRaArrayStar = raStarInPixel
finalDecArrayStar = decStarInPixel
return finalObjIDstar, finalRaArrayStar, finalDecArrayStar
def process_file(filename, night=None, favor2=None, verbose=False):
if favor2 is None:
favor2 = Favor2()
header = fits.getheader(filename, -1)
if night is None:
time = parse_time(header['TIME'])
night = get_night(time)
if verbose:
print(night, header['TYPE'])
# Skip old master calibrations
if header['TYPE'] in ['bgflat', 'superflat', 'flat']:
return None
image = fits.getdata(filename, -1).astype(np.double)
# Frame dimensions
width, height = header['NAXIS1'], header['NAXIS2']
# image,header = crop_overscans(image, header, subtract=False)
if header['TYPE'] not in ['flat', 'masterflat', 'dark', 'masterdark']:
image -= header.get('BASELINE', 100.0)
# Clean up the header a bit
header.remove('HISTORY', remove_all=True, ignore_missing=True)
header.remove('COMMENT', remove_all=True, ignore_missing=True)
header.remove('', remove_all=True, ignore_missing=True)
for _ in header.keys():
if _ and _[0] == '_':
header.remove(_, remove_all=True, ignore_missing=True)
type = header.get('TYPE', 'unknown')
filter = header.get('FILTER', 'unknown')
if filter == 'Custom' or 'Pol' in filter:
print('Broken filter in', filename)
return None
if type not in [
'dark', 'masterdark', 'flat', 'skyflat', 'superflat', 'masterflat'
]:
wcs = WCS(header) if header.get('CTYPE1') else None
if not wcs or not wcs.sip:
print('No WCS information in', filename)
return None
ra, dec = wcs.all_pix2world([0, image.shape[1], 0.5 * image.shape[1]],
[0, image.shape[0], 0.5 * image.shape[0]],
0)
ra1, dec1 = wcs.all_pix2world([image.shape[1], 0], [0, image.shape[0]],
0)
radius = 0.5 * coords.sphdist(ra[0], dec[0], ra[1], dec[1])[0]
radius1 = 0.5 * coords.sphdist(ra1[0], dec1[0], ra1[1], dec1[1])[0]
# Sanity checking with some hard-coded values
if radius < 7.25 or radius > 7.45 or np.abs(1.0 -
radius / radius1) > 0.1:
print('Broken WCS information in', filename)
return None
ra0, dec0 = ra[2], dec[2]
# Frame footprint
ra, dec = wcs.all_pix2world([0, 0, image.shape[1], image.shape[1]],
[0, image.shape[0], image.shape[0], 0], 0)
footprint = "(" + ",".join(
["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")"
# Frame footprint at +10 pixels from the edge
ra, dec = wcs.all_pix2world(
[10, 10, image.shape[1] - 10, image.shape[1] - 10],
[10, image.shape[0] - 10, image.shape[0] - 10, 10], 0)
footprint10 = "(" + ",".join(
["(%g,%g)" % (_, __) for _, __ in zip(ra, dec)]) + ")"
else:
# Should we really discard WCS for non-object frames?
ra0, dec0, radius = None, None, None
footprint, footprint10 = None, None
time = parse_time(header['TIME'])
exposure = header.get('EXPOSURE')
shutter = header.get('SHUTTER')
channel = header.get('CHANNEL ID')
pos0 = header.get('MIRROR_POS0')
pos1 = header.get('MIRROR_POS1')
mean = np.mean(image)
keywords = dict(header)
favor2.query(
'INSERT INTO images (filename,night,time,channel,type,filter,exposure,shutter,pos0,pos1,ra,dec,radius,width,height,mean,footprint,footprint10,keywords) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (filename) DO NOTHING',
(filename, night, time, channel, type, filter, exposure, shutter, pos0,
pos1, ra0, dec0, radius, width, height, mean, footprint, footprint10,
keywords))
return {
'filename': filename,
'night': night,
'time': time,
'channel': channel,
'type': type,
'filter': filter,
'shutter': shutter,
'ra0': ra0,
'dec0': dec0,
'radius': radius,
'exposure': exposure,
'width': width,
'height': height,
'mean': mean,
'keywords': keywords
}
def pixelTasks(parameterListForPixel):
"""Workers will execute this function."""
#unpack the input parameter list (i.e., separate them into different arrays)
#objID -- contains the good galaxy IDs -- unique no of times
#objIDs -- contains all the galaxy IDs corresponding to all the detections of galaxies
#medRa and medDec -- contain good median values
#resRa and resDec -- contain the residual values for each detection of galaxy-- not good
#ra/decAll -- ra/dec values for all the galaxies -- not good
#pixAll -- pixel indices for all the galaxies -- not good
pixelNo, pixRa, pixDec, objID, medRa, medDec, objIDs, MJDs, resRa, resDec, raAll, decAll, pixAll,mjdBreakAt, mjdSorted = parameterListForPixel
#print 'pixelNo', pixelNo
#print 'pixAll',pixAll
#global searchRadius
#global mjdBreakAt
#global mjdSorted
#find objects within the searchRadius
angSepMask = sphdist(pixRa, pixDec, medRa, medDec) <= (searchRadius/60.0)
objInRadius = objID[angSepMask]
#print 'objInRadius',objInRadius
#print "len(objIDs)",len(objIDs)
#print "len(objInRadius)",len(objInRadius)
#this function matches the given arrays, returns a boolean array of the size of first array.
#this index points to rows that contain residuals (and MJDs) of galaxy detections within search radius
'''resIndexInRadius = np.in1d(objIDs, objInRadius)'''
#print "len(resIndexInRadius)", len(resIndexInRadius)
#print 'objIDs[resIndexInRadius]',objIDs[resIndexInRadius]
#print 'len(objIDs[resIndexInRadius])',len(objIDs[resIndexInRadius])
#select unique obj ids in radius
#uniqueObjInRadius = np.unique(objIDs[resIndexInRadius])
#print "len(resRa)", len(resRa)
#print "len(resDec)", len(resDec)
#select the residual values corresponding to the objects in radius
'''resRaValuesInRadius = resRa[resIndexInRadius]
resDecValuesInRadius = resDec[resIndexInRadius]'''
#print 'resRaValuesInRadius', resRaValuesInRadius
#print 'resDecValuesInRadius', resDecValuesInRadius
#print 'len(resRaValuesInRadius)', len(resRaValuesInRadius)
#print 'len(resDecValuesInRadius)', len(resDecValuesInRadius)
#print 'np.count_nonzero(resRaValuesInRadius)', np.count_nonzero(resRaValuesInRadius)
#print 'np.count_nonzero(resDecValuesInRadius)', np.count_nonzero(resDecValuesInRadius)
#select mjds in the radius
'''mjdInRadius = MJDs[resIndexInRadius]'''
#print 'mjdInRadius',mjdInRadius
# r = time()
# a = []
# b = []
# c = []
# for p in range(objIDs.size):
# if objIDs[p] in objInRadius:
# a.append(resRa[p])
# b.append(resDec[p])
# c.append(MJDs[p])
# resRaValuesInRadius = np.array(a)
# resDecValuesInRadius= np.array(b)
# mjdInRadius = np.array(c)
'''instead of the above loop or in1d -- we use PANDAS'''
try:
flag1 = 1
r1 = time()
temp = pd.Index(objIDs)
inde = temp.get_indexer_for(objInRadius)
inde = inde[inde > -1]
inde = np.array(inde, dtype='u8')
#objIDs[inde]
resRaValuesInRadius = resRa[inde]
resDecValuesInRadius = resDec[inde]
mjdInRadius = MJDs[inde]
except IndexError:
return pixelNo, flag1, inde.size
#for storing median values for each epoch
medianResidualRaEpochWise = np.zeros(len(mjdBreakAt)+1)
medianResidualDecEpochWise = np.zeros(len(mjdBreakAt)+1)
#phiForObj = (medRa*np.pi)/180 # should be done outside
#thetaForObj = (90 - medDec)* (np.pi/180) # outside
#pixelIndexForObj = h.ang2pix(nside, thetaForObj, phiForObj) # calc outside, pass here
indexInPixel = pixAll == pixelNo
objIDinPixel = objID[indexInPixel]
#medRaInPixel = medRa[indexInPixel]
#medDecInPixel = medDec[indexInPixel]
#note : objIDinPixel and ra/decInPixel are not the same size, so we need to create another array
'''resIndexInPixel = np.in1d(objIDs, objIDinPixel)'''
#print 'len(objIDs)', len(objIDs)
#print 'len(objIDinPixel)',len(objIDinPixel)
#raInPixel = triand.col('ra')[maskForGal][resIndexInPixel] # access RA and Dec from outside - processing table each time for each pixel is bad
#decInPixel = triand.col('dec')[maskForGal][resIndexInPixel]
''' raInPixel = raAll[resIndexInPixel]
decInPixel = decAll[resIndexInPixel]
objInPixel = objIDs[resIndexInPixel]'''
# print 'raInPixel', raInPixel
# print 'decInPixel', decInPixel
# print 'objInPixel', objInPixel
# print 'len(raInPixel)', len(raInPixel)
# print 'len(decInPixel)', len(decInPixel)
# print 'len(objInPixel)', len(objInPixel)
# d = []
# e = []
# f = []
# g = []
# for o in range(objIDs.size):
# if objIDs[o] in objIDinPixel:
# d.append(raAll[o])
# e.append(decAll[o])
# f.append(objIDs[o])
# g.append(MJDs[o])
# raInPixel = np.array(d)
# decInPixel = np.array(e)
# objInPixel = np.array(f)
# mjdInPixel = np.array(g)
try :
flag2 = 2
temp2 = pd.Index(objIDs)
inde2 = temp2.get_indexer_for(objIDinPixel)
inde2 = inde2[inde2 > -1]
inde2 = np.array(inde2, dtype='u8')
raInPixel = raAll[inde2]
decInPixel = decAll[inde2]
objInPixel = objIDs[inde2]
mjdInPixel = MJDs[inde2]
#print "time taken for pandas indexer", time() - r1
except IndexError:
return pixelNo, flag2, inde2.size
#print "time taken by for loop", time() - r
# sort data in the pixel by objInPixel
#assert((objInPixel[1:] - objInPixel[0:-1]).all >= 0)
'''mjdInPixel = MJDs[resIndexInPixel]'''
#print mjdInPixel
#phiForObj = (triand.col('ra')*np.pi)/180
#thetaForObj = (90 - triand.col('dec'))* (np.pi/180)
#pixelIndexForObj = h.ang2pix(nside, thetaForObj, phiForObj)
#raInRadius = triand.col('ra')[maskForGal][resIndexInRadius]
#decInRadius = triand.col('dec')[maskForGal][resIndexInRadius]
#pixelIndexInRadius = pixelIndexForObj[resIndexInRadius]
#inPixelMask = pixelIndexInRadius == pixelNo
#raInPixel = raInRadius[inPixelMask]
#decInPixel = decInRadius[inPixelMask]
#if there are atleast three epochs present
if (len(mjdBreakAt)>=3.0):
previousVar2 = np.min(mjdSorted)
for var1, var2 in enumerate(mjdBreakAt, start=0):
mjdIndex = (mjdInRadius< var2) & (previousVar2 < mjdInRadius)
mjdIndexInPixel = (mjdInPixel<var2) & (previousVar2 < mjdInPixel)
#print var1
if any(mjdIndex):
#print "in if"
resRaValues = resRaValuesInRadius[mjdIndex]
resDecValues = resDecValuesInRadius[mjdIndex]
print 'resRaValues', resRaValues
print 'resDecValues', resDecValues
#print 'len(resRaValues)',len(resRaValues)
#print 'len(resDecValues)',len(resDecValues)
#print 'np.count_nonzero(resRaValues)', np.count_nonzero(resRaValues)
#print 'np.count_nonzero(resDecValues)', np.count_nonzero(resDecValues)
#resRaValuesTrue = resRaValues[np.where(resRaValues)]
assert(resRaValues.size > 0.0), "resRaValues is empty"
assert(resDecValues.size > 0.0), "resDecValues is empty"
medianResidualRaEpochWise[var1] = np.median(resRaValues)
medianResidualDecEpochWise[var1] = np.median(resDecValues)
print 'medianResidualRaEpochWise[',var1,']', medianResidualRaEpochWise[var1]
print 'medianResidualDecEpochWise[',var1,']', medianResidualDecEpochWise[var1]
#replacing old values with new values
raInPixel[mjdIndexInPixel] -= medianResidualRaEpochWise[var1]
decInPixel[mjdIndexInPixel] -= medianResidualDecEpochWise[var1]
if (var1==(len(mjdBreakAt)-1)):
previousVar2 = var2
var2 = np.max(mjdSorted)
mjdIndex = (mjdInRadius< var2) & (previousVar2 < mjdInRadius)
mjdIndexInPixel = (mjdInPixel<var2) & (previousVar2 < mjdInPixel)
if any(mjdIndex):
resRaValues = resRaValuesInRadius[mjdIndex]
resDecValues = resDecValuesInRadius[mjdIndex]
print 'resRaValues', resRaValues
print 'resDecValues', resDecValues
assert(resRaValues.size > 0.0), "resRaValues is empty"
assert(resDecValues.size > 0.0), "resDecValues is empty"
medianResidualRaEpochWise[var1+1] = np.median(resRaValues)
medianResidualDecEpochWise[var1+1] = np.median(resDecValues)
print 'medianResidualRaEpochWise[',var1+1,']', medianResidualRaEpochWise[var1+1]
print 'medianResidualDecEpochWise[',var1+1,']', medianResidualDecEpochWise[var1+1]
#replacing old values with new values
raInPixel[mjdIndexInPixel] -= medianResidualRaEpochWise[var1+1]
decInPixel[mjdIndexInPixel] -= medianResidualDecEpochWise[var1+1]
previousVar2 = var2
#else:
# print "in else"
# print "resRaValuesInRadius",resRaValuesInRadius
# resRaValues = resRaValuesInRadius[mjdIndex]
# resDecValues = resDecValuesInRadius[mjdIndex]
# assert(resRaValues.size > 0.0), "resRaValues is empty"
# assert(resDecValues.size > 0.0), "resDecValues is empty"
# medianResidualRaEpochWise[var1+1] = np.median(resRaValues)
# medianResidualDecEpochWise[var1+1] = np.median(resDecValues)
# print 'medianResidualRaEpochWise[',var1+1,']', medianResidualRaEpochWise[var1+1]
# print 'medianResidualDecEpochWise[',var1+1,']', medianResidualDecEpochWise[var1+1]
# #replacing old values with new values
# raInPixel[mjdIndexInPixel] -= medianResidualRaEpochWise[var1+1]
# decInPixel[mjdIndexInPixel] -= medianResidualDecEpochWise[var1+1]
# previousVar2 = var2
#calculate final ra/dec as the median of newRa/newDec values
finalRaArray = np.zeros(objIDinPixel.size)
finalDecArray = np.zeros(objIDinPixel.size)
medianRaError = np.zeros(objIDinPixel.size)
medianDecError = np.zeros(objIDinPixel.size)
#maskForStar = table.get_where_list('gal == 0')
#total number of *detections* of stars
#noOfDetOfStar = np.where(maskForStar)[0].size
#arrays to store deltas for stars
#deltaRaArray = np.zeros(noOfDetOfStar)
#deltaDecArray = np.zeros(noOfDetOfStar)
#if(pixelNo==74):
# print "Its pixel no 74"
# print "objInPixel:", objInPixel
# print "raInPixel:", raInPixel
# print "decInPixel:", decInPixel
try:
#take the current values separately
obj = objInPixel[0]
raObj = [raInPixel[0]]
decObj = [decInPixel[0]]
mjdObj = [mjdInPixel[0]]
#set variables
counter = 0
pos2 = 0
t2 = time()
for i in np.arange(objInPixel.size-1)+1:#np.arange(10)+1: #
#print 'on the row number',i
objID = objInPixel[i]
ra = raInPixel[i]
dec = decInPixel[i]
if(objID == obj):
raObj.append(ra)
decObj.append(dec)
else:
raObj = np.array(raObj)
decObj= np.array(decObj)
#print 'obj', obj
#print 'raObj', raObj
#print 'decObj', decObj
#find the final ra and dec values'
#assert(raObj.size > 0.0), "raObj is empty"
#assert(decObj.size > 0.0), "decObj is empty"
finalRaArray[counter] = np.median(raObj)
finalDecArray[counter] = np.median(decObj)
#calculate rms values for ra/dec
rmsRa = 0.741*(np.percentile(raObj, 75) - np.percentile(raObj, 25))
rmsDec = 0.741*(np.percentile(decObj, 75) - np.percentile(decObj, 25))
#calculate uncertainity in median coordinates
#print 'rmsRa', rmsRa
#print 'rmsDec', rmsDec
#print 'rmsRa.size = ', rmsRa.size
#print 'rmsDec.size=', rmsDec.size
#print 'raObj.size', raObj.size
#print 'decObj.size', decObj.size
medianRaError[counter] = np.sqrt((np.pi/2)/(raObj.size-1))*rmsRa
medianDecError[counter] = np.sqrt((np.pi/2)/(decObj.size-1))*rmsDec
# print finalRaArray[counter]
#print i
#print 'counter', counter
#calculate how much the galaxy moved in each epoch
#recall : the final ra and dec values were obtained after removing offsets epochwise and obj_id wise
#thus implying that these values make the galaxies static as required
#deltaRa = raObj2 - finalRaArray[counter]
#deltaDec = decObj2 - finalDecArray[counter]
#deltaRaArray[pos2:i] = deltaRa
#deltaDecArray[pos2:i] = deltaDec
#move to the next obj
obj = objID
pos2 = i
counter +=1
#make them lists again
raObj =[raInPixel[i]]
decObj =[decInPixel[i]]
#raObj2=[objInPixelFile2['ra'][i]]
#decObj2=[objInPixelFile2['dec'][i]]
#processing the last object now
raObj = np.array(raObj)
decObj = np.array(decObj)
#assert(raObj.size > 0.0), "raObj is empty"
#assert(decObj.size > 0.0), "decObj is empty"
finalRaArray[counter] = np.median(raObj)
finalDecArray[counter] = np.median(decObj)
#calculate rms values for ra/dec
rmsRa = 0.741*(np.percentile(raObj, 75) - np.percentile(raObj, 25))
rmsDec = 0.741*(np.percentile(decObj, 75) - np.percentile(decObj, 25))
medianRaError[counter] = np.sqrt((np.pi/2)/(raObj.size-1))*rmsRa
medianDecError[counter] = np.sqrt((np.pi/2)/(decObj.size-1))*rmsDec
#t3 = time()
#print 'time taken for calculating ra and dec values for a pixel is ', t3-t2
return objIDinPixel, finalRaArray, finalDecArray, medianRaError, medianDecError
except IndexError:
#pass
print "index error"
print pixelNo
#return objIDinPixel, finalRaArray, finalDecArray, medianRaError, medianDecError
return np.array(objIDinPixel), np.array(finalRaArray), np.array(finalDecArray), np.array(medianRaError), np.array(medianDecError)
def pixelTasks(parameterListForPixel):
"""Workers will execute this function."""
#unpack the input parameter list that is to say separate them into different arrays
pixelNo, ra, dec, nObs, medRa, medDec, resRa, resDec = parameterListForPixel
#define an index so that all the calculations can be put in place later
#ind =
global searchRadius
#find objects within the searchRadius and whose noOfObs is atleast 3
angSepMask = (sphdist(ra, dec, medRa, medDec) <= (searchRadius*60)) & (nObs>=3.0)
global testFile
#separate these entire rows from the original testFile
searchFile = testFile[angSepMask]
#putting unique object IDs in this file
uniqueSearchFile = np.unique(searchFile['obj_id'])
#file containing data for objects inside pixel
objInPixelFile = testFile[pixelIndexForObj == pixelNo]
#file containing unique objIDs of the pixel
uniqueObjInPixelFile = np.unique(objInPixelFile['obj_id'])
global mjdBreakAt
global mjdSorted
medianResidualRaEpochWise = np.zeros(len(mjdBreakAt)+1)
medianResidualDecEpochWise = np.zeros(len(mjdBreakAt)+1)
#if there are atleast three epochs present
if (len(mjdBreakAt)>=3.0):
previousVar2 = np.min(mjdSorted)
for var1, var2 in enumerate(mjdBreakAt, start=0):
mjdIndex = (searchFile['mjd']< var2) & (previousVar2 < searchFile['mjd'])
mjdIndexInPixel = (objInPixelFile['mjd']<var2) & (previousVar2 < objInPixelFile['mjd'])
#print var1
if any(mjdIndex):
medianResidualRaEpochWise[var1] = np.median(resRa[mjdIndex])
medianResidualDecEpochWise[var1] = np.median(resDec[mjdIndex])
#replacing old values with new values
objInPixelFile['ra'][mjdIndexInPixel] -= medianResidualRaEpochWise[var1]
objInPixelFile['dec'][mjdIndexInPixel] -= medianResidualDecEpochWise[var1]
else:
medianResidualRaEpochWise[var1+1] = np.median(resRa[mjdIndex])
medianResidualDecEpochWise[var1+1] = np.median(resDec[mjdIndex])
#replacing old values with new values
objInPixelFile['ra'][mjdIndexInPixel] -= medianResidualRaEpochWise[var1+1]
objInPixelFile['dec'][mjdIndexInPixel] -= medianResidualDecEpochWise[var1+1]
previousVar2 = var2
#calculate final ra/dec as the median of newRa/newDec values
finalRaArray = np.zeros(uniqueObjInPixelFile.size)
finalDecArray = np.zeros(uniqueObjInPixelFile.size)
#take the current values separately
currObj = objInPixelFile['obj_id'][0]
raObj = [objInPixelFile['ra'][0]]
decObj = [objInPixelFile['dec'][0]]
mjdObj = [objInPixelFile['mjd'][0]]
#set variables
counter = 0
#t2 = time()
for i in np.arange(objInPixelFile.size-1)+1:#np.arange(10)+1: #
objID = objInPixelFile['obj_id'][i]
ra = objInPixelFile['ra'][i]
dec = objInPixelFile['dec'][i]
if(objID == currObj):
raObj.append(ra)
decObj.append(dec)
else:
raObj=np.array(raObj)
decObj=np.array(decObj)
finalRaArray[counter] = np.median(raObj)
finalDecArray[counter] = np.median(decObj)
# print finalRaArray[counter]
#print i
#print counter
currObj=objID
raObj=[objInPixelFile['ra'][i]]
decObj=[objInPixelFile['dec'][i]]
counter +=1
#t3 = time()
#print 'time taken for calculating ra and dec values for a pixel is ', t3-t2
return finalRaArray, finalDecArray
#where can you
tBreakAt=np.where(condition)
#inputArray will be mjds and the binArray will be mjd plus minus deltaT
indexArray = np.digitize(inputArray, binArray)
#pixelisation
NSIDE = 2**10
#nside2npix give the number of pixels for a given NSIDE
m = np.arange(hp.nside2npix(NSIDE))
#pix2ang gives the angular coordinates
theta, phi = hp.pix2ang(NSIDE, m)
#pixelcenters
raCenter = 180*phi/np.pi
decCenter = 90 - theta*180/np.pi
#raCenter and decCenter are pixel centers
angularSeparation=sphdist(raCenter, decCenter, testFile['ra'], testFile['dec'])
temp = angularSeparation*60 < searchRadius
searchThisArrayIDs = np.unique(testFile['obj_id'][temp])
# sphericalDistArray = []*len(rBar_k)
# for row in rBar_k:
# #rBar_k[1] gives the ra values
# sphericalDistArray.append(callingFunction(rBar_k[1], raCenter))
#......................................................................
#obtain galaxies within a searchRadius of three times the pixelWidth - the updateArray
#you would need to take distance between the galaxies on the sky, so convert ra dec values to spherical coordinates - obtain the spherical distance - check if this is less than the searchRadius
#.......................................................................
searchThisArrayTemp = []
for i in testFile:
angSep = sphdist(raCenter, decCenter, i['ra'],i['dec'])
#print 'angSept is ', angSep
if ((angSep*60)<3000):
#if((angSep*60)<10):
#print 'i within 10 is', i
#break
searchThisArrayTemp.append(i)
#print searchThisArrayTemp
#break
searchThisArray = np.unique(searchThisArrayTemp[0])
#contains all the angular separations
angSep = sphdist(raCenter, decCenter, testFile['ra'],testFile['dec'])
#dtype=boolean
phiForObj = (testFile['ra'] * np.pi)/180 thetaForObj = (90 - testFile['dec'])* (np.pi/180) pixelIndexForObj= hp.ang2pix(nside,thetaForObj, phiForObj) #pick a pixel from the testFile pickPixelNo = pixelIndexForObj[len(pixelIndexForObj)/2] #find objects within the searchRadius and whose noOfObs is atleast 3 #-------------------- #VERSION 1 #-------------------- #take those objects whose atleast one value of ra/dec is inside the searchRadius angSepMask1 = sphdist( pixelRa[pickPixelNo], pixelDec[pickPixelNo], testFile['ra'], testFile['dec'] ) <= (searchRadius*60) #separate these entire rows from the original testFile searchFile = testFile[angSepMask1] #putting unique object IDs in this file uniqueSearchFile = np.unique(searchFile['obj_id']) #file containing data for objects inside pixel objInPixelFile = testFile[pixelIndexForObj == pickPixelNo] #file containing unique objIDs of the pixel uniqueObjInPixelFile = np.unique(objInPixelFile['obj_id']) #for containing no of observations for each object noOfObs = np.ones(len(uniqueSearchFile))