def growtrees(graph, d):
good = []
for CC in d:
CCgood = []
for trip in d[CC]:
blobsnow = [trip]
while blobsnow:
for blob in blobsnow:
successors = graph.successors(blob.dets[-1])
if not successors:
blobsnow.remove(blob)
worthadding(blob, CCgood)
continue
for next in successors:
trydets = blob.dets[:]
trydets.append(next)
tryblob = Triplet(trydets)
elements, errs = tryblob.calcOrbit()
if (elements['a'] > 2 and elements['e'] < 1):
if blob in blobsnow:
blobsnow.remove(blob)
blobsnow.append(tryblob)
worthadding(blob, CCgood)
if blob in blobsnow:
blobsnow.remove(blob)
good.extend(CCgood)
return good
def getInitTrips(triplets, savename):
trackID = 1
trackDict = {}
print('assigning trackids')
trips = []
for trip in triplets:
trips.append(trip)
trip.trackid = trackID
trackDict[trackID] = trip.dets
trackID += 1
# This loop makes a trackDict with trip.dets indexed starting at 1
triplets = trips
finalList = []
goodIDs, badIDs = siftChecks(trackDict, savename)
for i, chi in goodIDs:
trip = Triplet(trackDict[i])
trip.chiSq = chi
trip.trackid = i
finalList.append(trip)
for i, chi in badIDs:
trip = Triplet(trackDict[i])
trip.trackid = i
trip.chiSq = chi
print('bad triplet:')
print(trip)
finalList.append(trip)
return finalList
def generate_fake_preds(trip, mjd1, mjd2):
fakeDets = []
for det in trip.dets:
if (det.lookAhead != -1):
fakeDets.append(det)
#fakeDets = [x for x in trip.dets if x.fakeid == fakeid]
fakeTrip = Triplet(fakeDets)
coords = []
for days in range(int(mjd1), int(mjd2), 1):
coordinate = fakeTrip.predictPos(days)
coor = list(coordinate[0])
if coor[0] > 180:
coor[0] = coor[0] - 360
# Ensures ra is between -180 and 180.
coords.append(coor)
return coords
def generalCombine(arr, similarity, keep=False, progress=False):
"""
arr - numpy array of NLetRegion
similarity - float between 0 and 1
keep - set it to True to keep nlets in inactive list of NLetRegion
progress - set it to True for showing progress
"""
combined = resizableArray()
unchanged = resizableArray()
minRA = arr[0, 0].raLo
minDec = arr[0, 0].decLo
stepRA = arr[0, 0].raHi - minRA
stepDec = arr[0, 0].decHi - minDec
if progress:
count = 0
time0 = time.time()
uniqueNLets = {}
for i in arr:
for j in i:
for nlet in j:
nlet.sortByMjd()
uniqueNlets[tuple([x.objid for x in nlet.dets])] = nlet
size = len(uniqueNlets.values())
for x in arr:
for y in arr:
for nlet in y:
potential = []
objid = set([x.objid for x in nlet.dets])
target = int(similarity * len(nlet.dets))
othernlets, coord = getSameRegNLet(nlet, arr, minRA, minDec,
stepRA, stepDec)
for other in othernlets:
otherID = set([x.objid for x in others.dets])
if len(objid.intersection(otherID)) >= target:
detgroup = [x for x in nlet.dets]
potential.append(
Triplet(
list(set([x for x in other.dets] + detgroup))))
if len(potential) != 0:
for pot in potential:
combined.append(pot)
else:
unchanged.append(nlet)
if progress:
count += 1
printPercentage(count, size, time.time() - time0)
for radec in coord:
i = int((radec[0] - minRA) / stepRA)
j = int((radec[1] - minDec) / stepDec)
if keep:
arr[i, j].inactive.append(nlet)
arr[i, j].nlets.remove(nlet)
return combined, unchanged
def combine(arr, numdet, progress=False):
# we want the merged one to contain n+1 detections
target = numdet + 1
#combined = []
#unchanged = []
combined = resizableArray()
unchanged = resizableArray()
minRA = arr[0, 0].raLo
minDec = arr[0, 0].decLo
stepRA = arr[0, 0].raHi - minRA
stepDec = arr[0, 0].decHi - minDec
newobjid = -1 # to be updated for the extra detection to be added
if progress:
count = 0
time0 = time.time()
size = 0
for i in arr:
for j in i:
size += len(j.nlets)
size /= numdet
for x in arr:
for y in x:
# y is one of TripRegion
for nlet1 in y.nlets:
potential = []
objid = [i.objid for i in nlet1.dets]
addedId = []
othernlets, coord = getSameRegNLet(nlet1, arr, minRA, minDec,
stepRA, stepDec)
for nlet2 in othernlets:
detgroup = [x for x in nlet1.dets]
for det in nlet2.dets:
if det.objid not in objid and det.objid not in addedId:
detgroup.append(det)
newobjid = det.objid
# check that there are n+1 objects
if len(detgroup) == target:
potential.append(Triplet(detgroup))
addedId.append(newobjid)
if len(potential) == 0:
unchanged.append(nlet1)
else:
for pot in potential:
combined.append(pot)
if progress:
count += 1
printPercentage(count, size, time.time() - time0)
for radec in coord:
i = int((radec[0] - minRA) / stepRA)
j = int((radec[1] - minDec) / stepDec)
# if you want to keep the nlet, uncomment the line below
# arr[i,j].inactive.append(nlet1)
arr[i, j].nlets.remove(nlet1)
return combined.toList(), unchanged.toList()
def mergeFakes(triplets):
fakeDict = {}
for trip in triplets:
fakeid = trip.dets[0].fakeid
if (fakeid in fakeDict):
fakeDict[fakeid].extend(trip.dets)
else:
fakeDict[trip.dets[0].fakeid] = trip.dets
trips = []
for trip in fakeDict.values():
trips.append(Triplet(trip))
return trips
def checkDetection(nlet, predDet, newdet, chiThresh):
debugCounter = 0
timeWrap = 0
timeChiSq = 0
timeA = time.time()
inEllipse = withinEllipse(newdet, predDet)
timeB = time.time()
if (inEllipse):
print('\nra:' + str(predDet.ra) + ' dec:' + str(predDet.dec))
print('erra:' + str(predDet.erra) + ' errb:' + str(predDet.errb) +
' pa:' + str(predDet.pa))
print(newdet.toStr())
debugCounter += 1
newTrip = Triplet(nlet.dets[:])
newTrip.addDetection(newdet)
timeB = time.time()
chisq = newTrip.getChiSq()
oldsize = len(nlet.dets)
newsize = len(newTrip.dets)
##
timeC = time.time()
timeWrap = timeB - timeA
timeChiSq = timeC - timeB
#print('timewrap' + str(timeWrap))
#print('timechisq' + str(timeChiSq))
if (chisq < chiThresh and newTrip.elements['a'] > 2
and newsize > oldsize):
print('\nfound new detection')
print(newdet.toStr())
print(newTrip.toStr())
return newTrip, debugCounter, (timeWrap, timeChiSq)
timeWrap = timeB - timeA
return nlet, debugCounter, (timeWrap, timeChiSq)
def siftTrips(fitsName, trackDict, chiThresh=50):
print('only keeping orbtis with chisq under ' + str(chiThresh))
orbits = Table.read(fitsName, format='fits')
# Records the initial time.
time1 = time.time()
mask_chisq = (orbits['CHISQ'] < chiThresh) & (orbits['FLAGS'] == 0)
orbitIDs = orbits[mask_chisq]['ORBITID']
# orbitIDs is the list of orbits with ChiSq under 50.
tripList = []
for row in orbitIDs:
trackid = row
objids = trackDict[trackid]
# trackDict is a dictionary of orbits; trackDict[trackid] is a single orbit.
del trackDict[trackid]
trip = Triplet(objids)
# Finds the triplet for the orbit.
trip.trackid = trackid
# Creates an ID for each triplet (called trackid).
tripList.append(trip)
# Creates tripList, a list of all triplets.
# Records the final time.
time2 = time.time()
'''
#version 2
tripList2 = []
for row in orbits:
if row['CHISQ'] < chiThresh:
trackid = '{0:06d}'.format(row['ORBITID'])
objids = trackDict[trackid]
tripList2.append(Triplet(objids))
time3 = time.time()
'''
print('done after ' + str(time2 - time1) + ' seconds')
#print('v2: ' + str(time3-time2))
return tripList
def formTriplets(args):
detPairs, queue = args
#queue = 0 if not multiprocessing
tripList = []
time0 = time.time()
counter = 0
for det in detPairs:
if (queue == 0):
counter += 1
printPercentage(counter, len(detPairs), time.time() - time0)
for link in det.linkedList:
for trip in link.linkedList:
triplet = Triplet([det, link, trip])
tripList.append(triplet)
if (queue != 0):
queue.put(tripList)
return tripList
def generalCombine(arr, similarity, chisq):
combined = []
unchanged = []
minRA = arr[0, 0].raLo
minDec = arr[0, 0].decLo
stepRA = arr[0, 0].raHi - minRA
stepDec = arr[0, 0].decHi - minDec
for x in arr:
for y in x:
# y is one of TripRegion
for nlet1 in y.nlets:
potential = []
objid = {}
for det1 in nlet1.dets:
objid[det1.objid] = det1
# doesn't assume that all nlets have the same number of detections
# if all nlets do have the same number of detections this could be
# moved outside of for-loop
numobjid = len(objid)
othernlets, coord = getSameRegNLet(nlet1, arr, minRA, minDec,
stepRA, stepDec)
for nlet2 in othernlets:
newdict = objid.copy()
for det2 in nlet2.dets:
newdict[det2.objid] = det2
numadded = len(newdict) - numobjid
if numadded >= similarity:
potential.append(Triplet(newdict.values()))
potential = checkGoodOrbit(potential, chisq)
if len(potential) > 0:
combined = combined + potential
else:
unchanged.append(nlet1)
for radec in coord:
i = int((radec[0] - minRA) / stepRA)
j = int((radec[1] - minDec) / stepDec)
arr[i, j].inactive.append(nlet1)
arr[i, j].nlets.remove(nlet1)
return combined, unchanged
def addToNlet(nlet, expDict):
time1 = time.time()
expLst = sorted([x.expnum for x in nlet.dets])
####
time2 = time.time()
firstExp = expLst[len(expLst) / 2]
lastExp = firstExp
print('first exp: ' + str(firstExp) + ' last exp: ' + str(lastExp))
chiThresh = 100
counter = 0
####
size = len(expDict)
print('number of exposures to check: ' + str(size))
tempLet = Triplet(nlet.dets)
#only look at exposures after last and before first
time4 = time.time()
sortedDict = sorted(expDict.iteritems())
greater = [x for x in sortedDict if x[0] > lastExp]
lesser = [x for x in sortedDict if x[0] < firstExp]
lesser.reverse()
time0 = time.time()
#dontworryaboutwhatthisdoes(greater)
#dontworryaboutwhatthisdoes(lesser)
timeList = time2 - time1
timeInit = time4 - time2
timeSort = time0 - time4
timePrint = 0
timePred = 0
timeWrap = 0
timeCheck = 0
timeCheckWrap = 0
timeCheckChi = 0
totalCount = 0
#checking every exposure
for key, value in greater:
counter += 1
timeA = time.time()
print '\nchecking exposure ' + str(key) + ' with ' + str(
len(value)) + ' items',
###
#printPercentage(counter, size, time.time()-time0)
###
timeB = time.time()
mjd = value[0].mjd
coord, erra, errb, pa = tempLet.predictPos(mjd)
#print('pa' + str(pa))
###
timeC = time.time()
det = Detection(coord[0], coord[1], mjd, 1, 0, key, 0, 0, 0, 0)
det.erra = erra
det.errb = errb
det.pa = pa
print 'erra: ' + str(erra) + ' errb: ' + str(errb),
###
timeD = time.time()
#checks every detection in the exposure
debugCounter = 0
for pdet in value:
tempLet, debugCount, (timeF, timeG) = checkDetection(
tempLet, det, pdet, chiThresh)
debugCounter += debugCount
timeCheckWrap += timeF
timeCheckChi += timeG
timeE = time.time()
print 'found ' + str(debugCounter) + ' within error ellipse',
totalCount += debugCounter
timePrint += timeB - timeA
timePred += timeC - timeB
timeWrap += timeD - timeC
timeCheck += timeE - timeD
for key, value in lesser:
counter += 1
timeA = time.time()
print '\nchecking exposure ' + str(key) + ' with ' + str(
len(value)) + ' items',
#printPercentage(counter, size, time.time()-time0)
##
timeB = time.time()
mjd = value[0].mjd
coord, erra, errb, pa = tempLet.predictPos(mjd)
timeC = time.time()
det = Detection(coord[0], coord[1], mjd, 1, 0, key, 0, 0, 0, 0)
det.erra = erra
det.errb = errb
det.pa = pa
print 'erra: ' + str(erra) + ' errb: ' + str(errb),
##
timeD = time.time()
#checks every detection in the exposure
debugCounter = 0
for pdet in value:
tempLet, debugCount, (timeF, timeG) = checkDetection(
tempLet, det, pdet, chiThresh)
debugCounter += debugCount
timeCheckWrap += timeF
timeCheckChi += timeG
totalCount += debugCounter
timeE = time.time()
print 'found ' + str(debugCounter) + ' within error ellipse',
timePrint += timeB - timeA
timePred += timeC - timeB
timeWrap += timeD - timeC
timeCheck += timeE - timeD
print('timeList: ' + str(timeList))
print('timeInit: ' + str(timeInit))
print('timeSort: ' + str(timeSort))
print('timePrint: ' + str(timePrint))
print('timePred: ' + str(timePred))
print('timeWrap: ' + str(timeWrap))
print('timeCheck: ' + str(timeCheck))
print('timeCheckWrap: ' + str(timeCheckWrap))
print('timeCheckChi: ' + str(timeCheckChi))
print('total detections checked: ' + str(totalCount))
return tempLet
def newMergeTrips(triplets, savename, thresh=10):
finalList = []
totalSize = len(triplets)
print('\nnumber to merge: ' + str(totalSize))
checkList = {}
rmList = {}
trackid = 0
time0 = time.time()
trackList = {}
objidTrack = {}
tempTrack = 0
for trip in triplets:
#overwriting all the trackid's of the triplets to a unique id
trip.trackid = tempTrack
trackList[tempTrack] = trip
tempTrack += 1
for det in trip.dets:
if (det.objid in objidTrack):
objidTrack[det.objid].append(trip.trackid)
else:
objidTrack[det.objid] = [trip.trackid]
while (len(trackList) > 0):
LL.printPercentage(totalSize - len(trackList), totalSize,
time.time() - time0)
track, thisTrip = trackList.popitem()
update = False
for det in thisTrip.dets:
for tempid in objidTrack[det.objid]:
if (tempid not in trackList):
continue
trip = trackList[tempid]
if (thisTrip.shareM(
trip,
max(len(thisTrip.dets), len(trip.dets)) / 2 + 1)
or thisTrip.isSubset(trip) or trip.isSubset(thisTrip)):
# Once this if statement is triggered, the first return statement can't be triggered
# and the function will eventually go to the last return statement
# also the code will escape the outer for loop to be within the while loop.
newDets = thisTrip.merge(trip)
checkList[trackid] = newDets
rmList[trackid] = (thisTrip, trip)
trackList.pop(tempid)
trackid += 1
update = True
break
if (update):
break
if (not update):
finalList.append(thisTrip)
if (len(checkList) == 0):
return finalList
# Exits the function, returning finaList.
goodMergeIds, badMergeIds = siftChecks(checkList, savename, thresh)
for i, chi in goodMergeIds:
trip = Triplet(checkList[i])
trip.chiSq = chi
finalList.append(trip)
for i, chi in badMergeIds:
trip1, trip2 = rmList[i]
if (trip1.chiSq > 0 and trip2.chiSq > 0):
if (trip1.chiSq / len(trip1.dets) > trip2.chiSq / len(trip2.dets)):
finalList.append(trip2)
else:
finalList.append(trip1)
else:
if (trip1.realLength() > trip2.realLength()):
finalList.append(trip1)
else:
finalList.append(trip2)
return newMergeTrips(finalList, savename, thresh)