def main():
args = argparse.ArgumentParser()
args.add_argument('triplets',
nargs=1,
help='path to pickle file; ' +
'file has format triplet+SNOBS_SEASON###_ML0#.pickle')
args.add_argument('numChunks',
nargs='?',
default=100,
help='number of chunks')
args = args.parse_args()
print('open pickle file')
pickleFile = open(args.triplets[0], 'rb')
print('loading pickle file')
triplets = pickle.load(pickleFile)
tripChunks = splitList(triplets, int(args.numChunks))
#save triplets
saveName = args.triplets[0].split('+')[-1].split('.')[0]
saveName2 = args.triplets[0].split('+')[-2]
if '/' in saveName2:
saveName2 = saveName2.split('/')[-1]
print(saveName2)
for x in range(len(tripChunks)):
writeTriplets(
tripChunks[x],
'chunk' + str(x) + '+' + saveName2 + '+' + saveName + '.txt',
False)
pickleTriplets(
tripChunks[x],
'chunk' + str(x) + '+' + saveName2 + '+' + saveName + '.pickle')
def main():
args = argparse.ArgumentParser()
args.add_argument('triplets', nargs=1, help='path to pickle file')
args = args.parse_args()
triplets = pickle.load(open(args.triplets[0], 'rb'))
saveName = args.triplets[0].split('+')[-1].split('.')[0]
# merges all triplets onto a graph
G, dict = connectGraph(triplets)
# know the tripelts inside each connected components
d = organizetriplets(G, triplets)
good = growtrees(G, d)
#saves every merged triplets
saveGoodName = 'goodMergedTriplets3+' + saveName
#saveBadName = 'badTriplets+' + saveName
writeTriplets(good, saveGoodName + '.txt')
#writeTriplets(bad, saveBadName + '.txt')
pickleTriplets(good, saveGoodName + '.pickle')
#pickleTriplets(bad, saveBadName + '.pickle')
# graphs original merged graph
#graphLinks(G, dict, saveName)
G, dict = connectGraph(good)
graphLinks(G, dict, saveName)
#, [3.4,5.2,-1,0])
'''
def main():
args = argparse.ArgumentParser()
args.add_argument(
'linkedPairs',
nargs=1,
help='path to pickle file; ' +
'file has format detectionLinks+SNOBS_SEASON###_ML0#.pickle')
args.add_argument('numChunks',
nargs='?',
default=1,
help='number of chunks')
args.add_argument('ncpus', nargs='?', default=1, help='number of cpus')
args = args.parse_args()
#load the pickle file with list of detections and links
print('load pickle file')
detPairs = pickle.load(open(args.linkedPairs[0], 'rb'))
#makes a list of potential triplets
Ncpu = int(args.ncpus)
triplets = formTriplets((detPairs, 0))
numChunks = args.numChunks
tripChunks = splitList(triplets, numChunks)
#triplets = multiProcessPairs(detPairs, Ncpu)
saveName = args.linkedPairs[0].split('+')[-1].split('.')[0]
for x in range(len(tripChunks)):
writeTriplets(tripChunks[x],
'chunk{0:04d}'.format(x) + '+' + saveName + '.txt',
False)
pickleTriplets(tripChunks[x],
'chunk{0:04d}'.format(x) + '+' + saveName + '.pickle')
def main():
args = argparse.ArgumentParser()
args.add_argument('triplets', nargs='+', help='list of triplets to merge')
args.add_argument('-f',
'--fake',
action='store_true',
help='whether to look at fakes')
args = args.parse_args()
savename1 = args.triplets[0].split('/')[-1].split('.')[0]
savename = 'merged+' + savename1
# Sets the name for the save (which begins with "merged")
triplets = []
print('loading triplets...')
time0 = time.time()
# Records an inital time
for trips in args.triplets:
triplets.extend(pickle.load(open(trips, 'rb')))
# Opens the file
print('done loading after ' + str(time.time() - time0) + ' seconds')
if (args.fake):
mergedTrips = mergeFakes(triplets)
else:
siftedTrips = getInitTrips(triplets, savename1)
mergedTrips = newMergeTrips(siftedTrips, savename1)
print('\nsize of final list = ' + str(len(mergedTrips)))
finalList = []
for trip in mergedTrips:
if (trip.realLength() > 4):
finalList.append(trip)
# Keeps triplets with five or more detections.
print('size after reducing: ' + str(len(finalList)))
writeTriplets(finalList, savename + '.txt')
pickleTriplets(finalList, savename + '.pickle')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('nlets', nargs=1, help='path to nlet pickle file')
parser.add_argument('chisq',
nargs='?',
default=5,
help='chisq threshold for good orbit')
args = parser.parse_args()
print("Loading pickle file")
nlets = pickle.load(open(args.nlets[0], 'rb'))
print("Done loading")
savename = args.nlets[0].split('+')[-1].split('.')[0]
print('Recursively merging...')
while len(nlets) != 0:
t0 = time.time()
numdet = len(nlets[0].dets)
print('Processing ' + str(len(nlets)) + ' ' + str(numdet) + '-lets')
minRA, maxRA, minDec, maxDec = getBounds(nlets)
arr = partition(nlets, minRA, maxRA, minDec, maxDec, 700, 700)
nPlusOne, unchanged = combine(arr, numdet, True)
# remove duplicates
nPlusOne = removeDuplicates(nPlusOne)
unchanged = removeDuplicates(unchanged)
# remove same exposure
nPlusOne = removeSameExposure(nPlusOne)
unchange = removeSameExposure(unchanged)
print("\nChecking good orbits for " + str(len(nPlusOne)) + " n+1 lets")
# check for good orbits
nPlusOne = checkGoodOrbit(nPlusOne, float(args.chisq), progress=True)
print("Checking good orbits for " + str(len(unchanged)) + " nlets")
unchanged = checkGoodOrbit(unchanged, float(args.chisq), progress=True)
num_u = len(unchanged)
if num_u != 0:
print('Saving ' + str(num_u) + ' unchanged ' + str(numdet) +
'-lets')
saveas_u = 'good' + str(numdet) + 'letU' + str(
args.chisq) + '+' + savename
writeTriplets(unchanged, saveas_u + '.txt', False)
pickleTriplets(unchanged, saveas_u + '.pickle')
nlets = nPlusOne
print('Time taken this cycle: ' + str(time.time() - t0))
print('Done')
def main():
args = argparse.ArgumentParser()
args.add_argument('triplets',
help='path to pickle file; file has format ' +
'chunk###+SNOBS_S\EASON###_ML0#.pickle')
args.add_argument('-d',
'--detFile',
help='path to csv file with detections')
args.add_argument('-o',
'--orbit',
action='store_true',
help='produce orbitfile only')
args.add_argument('-x',
'--suppress',
action='store_true',
help='do not run fitter')
args = args.parse_args()
print('\nopen pickle file ' + args.triplets)
with open(args.triplets, 'rb') as pickleFile:
print('load pickle file')
triplets = pickle.load(pickleFile)
print('done loading')
saveName = args.triplets.split('+')[-1].split('.')[0]
chunkName = args.triplets.split('/')[-1].split('+')[0]
if (args.suppress):
lets = removeBadChisq(triplets)
else:
if (args.detFile is None):
detDict = {}
else:
detDict = LL.objidDictionary(args.detFile)
try:
triplets = list(set(triplets))
except TypeError:
triplets = triplets
outname, trackIDs, lets = writeProcessingFile(triplets, detDict,
chunkName, saveName)
fitsFile = callBulkFit(outname)
if (args.orbit):
callBulkEle(fitsFile)
else:
lets = siftTrips(fitsFile, trackIDs)
# Saving and pickling triplets to chunkName + '+goodtriplets+' + saveName + '.txt'
writeTriplets(lets, chunkName + '+goodtriplets+' + saveName + '.txt',
args.orbit)
pickleTriplets(lets,
chunkName + '+goodtriplets+' + saveName + '.pickle',
rmunbound=False)
def main():
args = argparse.ArgumentParser()
args.add_argument('triplets',
nargs=1,
help='path to pickle file; file has format ' +
'chunk###+SNOBS_SEASON###_ML0#.pickle')
#args.add_argument('chisq', nargs='?', default=5,
# help='chisq threshold to be considered a good triplet')
args.add_argument('ncpus', nargs='?', default=1, help='number of cpus')
args = args.parse_args()
print('open pickle file ' + args.triplets[0])
pickleFile = open(args.triplets[0], 'rb')
print('load pickle file')
triplets = pickle.load(pickleFile)
Ncpu = int(args.ncpus)
#checks orbits for good fit
goodTrips, missTrips, badTrips, p9Trips = potentialTriplets((triplets, 0))
#multiProcessTriplets(triplets, Ncpu)
#saving triplets
saveName = args.triplets[0].split('+')[-1].split('.')[0]
chunkName = args.triplets[0].split('/')[-1].split('+')[0]
saveName = chunkName + '+' + saveName
writeTriplets(goodTrips, 'goodtriplets+' + saveName + '.txt', False)
writeTriplets(missTrips, 'misstriplets+' + saveName + '.txt', False)
writeTriplets(badTrips, 'badtriplets+' + saveName + '.txt', False)
writeTriplets(p9Trips, 'p9triplets+' + saveName + '.txt', False)
pickleTriplets(goodTrips, 'goodtriplets+' + saveName + '.pickle')
pickleTriplets(missTrips, 'misstriplets+' + saveName + '.pickle')
pickleTriplets(badTrips, 'badtriplets+' + saveName + '.pickle')
pickleTriplets(p9Trips, 'p9triplets+' + saveName + '.pickle')
def main():
args = argparse.ArgumentParser()
args.add_argument('linkedPairs', help='path to pickle file; ' + 'file has format detectionLinks+SNOBS_SEASON###_ML0#.pickle')
args.add_argument('-n', '--chunkSize', help='size of chunks, leave empty for only one chunk')
args.add_argument('-c', '--cont', help='last chunk processed')
args = args.parse_args()
# Three arguments, of which the last two are optional.
tripletStart = [-1,-1,-1]
chunkStart = 1
if(args.cont):
print('loading last chunk: ' + args.cont)
fname = args.cont.split('+')[0]
if('/' in fname):
fname = fname.split('/')[-1]
chunkStart = int(fname[-6:]) + 1
with open(args.cont, 'rb') as f:
lastChunk = pickle.load(f)
lastTrip = lastChunk[-1]
tripletStart = lastTrip[1]
print('last triplet: ' + str(tripletStart))
print('starting from chunk: ' + str(chunkStart))
# load the pickle file with list of detections and links
print('load pickle file')
detPairs = pickle.load(open(args.linkedPairs, 'rb'))
print('done loading')
# makes a list of potential triplets
chunkSize = -1
if(args.chunkSize):
chunkSize = int(args.chunkSize)
saveName = args.linkedPairs.split('+')[-1].split('.')[0]
print('forming triplets')
triplets = formTriplets(detPairs, chunkSize, saveName, tripletStart, chunkStart)
# tripChunks = splitList(triplets, numChunks, buffered, saveName, printP)
# triplets = multiProcessPairs(detPairs, Ncpu)
# we save it earlier if argument is buffered
x = 0
pickleTriplets(triplets, 'chunk{0:06d}'.format(x) + '+' + saveName + '.pickle', False)
writeTriplets(triplets, 'chunk{0:06d}'.format(x) + '+' + saveName + '.txt', False, False)
def main():
args = argparse.ArgumentParser()
args.add_argument('detList',
nargs=1,
help='path to csv list of detections')
args.add_argument('potentialNlets',
nargs=1,
help='Nlets we want to add on to')
args = args.parse_args()
print('\nloading nlets ' + args.potentialNlets[0])
nlets = pickle.load(open(args.potentialNlets[0], 'rb'))
nameList = args.potentialNlets[0].split('+')
saveName = nameList[-1].split('.')[0]
saveName2 = nameList[-2]
if '/' in saveName2:
saveName2 = saveName2.split('/')[-1]
print(saveName2)
saveName3 = ''
if (len(nameList) >= 3):
saveName3 = nameList[-3]
if '/' in saveName3:
saveName3 = saveName3.split('/')[-1]
print('\nwrapping detections: ' + args.detList[0])
detList = wrapDets(args.detList[0])
print('\nmaking dictionary')
expDict = expDictionary(detList)
size = len(nlets)
counter = 0
time0 = time.time()
newNletList = []
for nlet in nlets:
counter += 1
print('\n*****adding to Nlet #' + str(counter) + ' of ' + str(size) +
' *****')
print('OLD TRIPLET:')
print(nlet.toStr())
newLet = addToNlet(nlet, expDict)
print('\nNEW TRIPLET:')
print(newLet.toStr())
newNletList.append(newLet)
name = 'crossCampaignTriplets+' + saveName3 + '+' + saveName2 + '+' + saveName
writeTriplets(newNletList, name + '.txt', True)
pickleTriplets(newNletList, name + '.pickle')
def formTriplets(detPairs, chunkSize, saveName, tripletStart=[-1,-1,-1], chunkStart = 1):
# detPairs, queue = args
linkDict = {}
for pair in detPairs:
linkDict[pair[0]] = pair[1]
tripList = []
time0 = time.time()
counter = 0
x = chunkStart
trackCount = 0
print('size of each chunk: ' +str(chunkSize))
notFound = True
for det in detPairs:
counter += 1
links = det[1]
if counter%1000==0:
printPercentage(counter, len(detPairs), time.time()-time0)
if det[0] != tripletStart[0] and tripletStart[0] != -1 and notFound:
continue
for link in links:
if link != tripletStart[1] and tripletStart[1] != -1 and notFound :
continue
links2 = linkDict[link]
for trip in links2:
if trip != tripletStart[2] and tripletStart[2] != -1 and notFound:
continue
elif(det[0] == tripletStart[0] and link == tripletStart[1] and trip == tripletStart[2]):
notFound = False
trackid = (trackCount%chunkSize)
triplet = (trackid, [det[0], link, trip])
trackCount += 1
tripList.append(triplet)
if chunkSize > 0 and len(tripList) >= chunkSize:
pickleTriplets(tripList, 'chunk{0:06d}'.format(x) + '+' + saveName + '.pickle', False)
# writeTriplets(tripList, 'chunk{0:06d}'.format(x) +
# '+' + saveName + '.txt', False, False)
x += 1
trackCount = 0
tripList = []
return tripList
def main():
args = argparse.ArgumentParser()
args.add_argument('detList',
nargs=1,
help='path to csv list of detections')
args.add_argument('predList',
nargs=1,
help='path to list of predicted detections')
args = args.parse_args()
saveName = args.predList[0].split('+')[-1].split('.')[0]
# print(args.detList[0].split('+'))
saveName2 = args.predList[0].split('+')[-2]
print(saveName2)
print('wrapping detections: ' + args.detList[0])
detList = wrapDets(args.detList[0])
print('making dictionary')
expDict = expDictionary(detList)
print('loading prediction list: ' + args.predList[0])
predList = pickle.load(open(args.predList[0]))
newTripList = []
size = len(predList)
counter = 0
for tup in predList:
counter += 1
print('*****adding to Nlet #' + str(counter) + ' of ' + str(size) +
' *****')
print('OLD TRIPLET:')
print(tup[0].toStr())
newTriplet = addDetections(tup[0], tup[1], expDict)
print(newTriplet.toStr())
newTripList.append(newTriplet)
name = 'crossCampaignTriplets+' + saveName2 + '+' + saveName
writeTriplets(newTripList, name + '.txt', True)
pickleTriplets(newTripList, name + '.pickle')
def main():
args = argparse.ArgumentParser()
args.add_argument('folder', nargs=1, help='path to folder of chunks')
args = args.parse_args()
files = os.listdir(args.folder[0])
tripList = []
saveName = ''
print('opening files in: ' + args.folder[0])
time0 = time.time()
counter = 0
for f in files:
counter += 1
printPercentage(counter, len(files), time.time() - time0)
if (f.split('+')[0] == 'goodtriplets'
and f.split('.')[-1] == 'pickle'):
print('\nopening: ' + args.folder[0] + f)
trips = pickle.load(open(args.folder[0] + f, 'rb'))
saveName = f.split('+')[-1].split('.')[0]
for t in trips:
tripList.append(t)
writeTriplets(tripList, 'goodTriplets+' + saveName + '.txt', True)
pickleTriplets(tripList, 'goodTriplets+' + saveName + '.pickle')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('nlets', nargs=1, help='path to nlet pickle file')
parser.add_argument('chisq',
nargs='?',
default=5,
help='chisq threshold for good orbit')
args = parser.parse_args()
nlets = pickle.load(open(args.nlets[0], 'rb'))
savename = args.nlets[0].split('+')[-1].split('.')[0]
chisq = float(args.chisq)
merged = []
print('Recursively merging...')
minRA, maxRA, minDec, maxDec = getBounds(nlets)
while len(nlets) != 0:
t0 = time.time()
numdet = len(nlets[0].dets)
print('Processing ' + str(len(nlets)) + ' ' + str(numdet) + '-lets')
arr = partition(nlets, minRA, maxRA, minDec, maxDec, 700, 700)
nPlusOne, unchanged = combine(arr, numdet)
# remove duplicates
nPlusOne = removeDuplicates(nPlusOne)
unchanged = removeDuplicates(unchanged)
# remove same exposure
nPlusOne = removeSameExposure(nPlusOne)
unchange = removeSameExposure(unchanged)
# check for good orbits
nPlusOne = checkGoodOrbit(nPlusOne, float(args.chisq))
unchanged = checkGoodOrbit(unchanged, float(args.chisq))
merged.append(unchanged)
nlets = nPlusOne
print('Time taken this cycle: ' + str(time.time() - t0))
print('Done N-1 Merging')
print('Starting Half Merge...')
stepRA = arr[0][0].raHi - arr[0][0].raLo
stepDec = arr[0][0].decHi - arr[0][0].decLo
# start with nlet with most number of detections
merged.reverse()
remaining = []
leftover = merged[0]
for i in range(len(merged) - 1):
if len(leftover) == 0:
print('ERROR: No nlets left in this cycle')
continue
arr = partition(leftover, minRA, maxRA, minDec, maxDec, 700, 700)
numdet = len(leftover[0].dets)
print("Self merging " + str(len(leftover)) + ' ' + str(numdet) +
"-let")
sameCombined, sameUnchanged = generalCombine(arr, (numdet + 1) / 2,
chisq)
renewArray(arr)
for oneless in merged[i + 1]:
insertNLet(oneless, arr, minRA, minDec, stepRA, stepDec)
print("Second stage merging with " + str(numdet - 1) + "-let")
mixCombined, mixUnchanged = generalCombine(
arr, (len(merged[i + 1][0].dets) + 1) / 2, chisq)
remaining = remaining + sameCombined + sameUnchanged + mixCombined
leftover = mixUnchanged
print('Removing duplicates and same exposures')
# remove duplicates and same exposure
remaining = removeDuplicates(remaining)
remaining = removeSameExposure(remaining)
remaining = compartmentalize(remaining)
savename = args.nlets[0].split('+')[-1].split('.')[0]
for rem in remaining:
saveas = 'good' + str(len(
rem[0].dets)) + "letHM" + str(chisq) + '+' + savename
writeTriplets(rem, saveas + '.txt', False)
pickleTriplets(rem, saveas + '.pickle')
print('Done')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('nlets', nargs=1, help='path to nlet pickle file')
parser.add_argument('chisq',
nargs='?',
default=5,
help='chi-square threshold for orbit fitter')
args = parser.parse_args()
nlets = pickle.load(open(args.nlets[0], 'rb'))
# identify the number of detections in each Triplet object
objcount = len(nlets[0].dets)
t0 = time.time()
# First get bounds of season
# Get the array of partitioned season
# Each cell of the array holds Triplet object with detections that fall in that region
# if Triplet object have 5 detections (quint) then it will belong to up to 5 cells
# Finally combine function merges nlet to form (n+1)let
minRA, maxRA, minDec, maxDec = getBounds(nlets)
arr = partition(nlets, minRA, maxRA, minDec, maxDec, 800, 800)
nPlusOne, unchanged = combine(arr, objcount)
# remove duplicates
nPlusOne = removeDuplicates(nPlusOne)
unchanged = removeDuplicates(unchanged)
# remove same exposure
nPlusOne = removeSameExposure(nPlusOne)
unchanged = removeSameExposure(unchanged)
# check for good orbits
print('Orbit fit test in progress')
goodNPlusOne = checkGoodOrbit(nPlusOne, float(args.chisq))
goodUnchanged = checkGoodOrbit(unchanged, float(args.chisq))
t1 = time.time() - t0
print('Time taken: ' + str(t1))
# dictionary for savename
numobj = {
3: 'triplet',
4: 'quad',
5: 'quint',
6: 'hex',
7: 'sept',
8: 'oct'
}
# Now saving (n+1)let
names = {
3: 'goodtriplet',
4: 'goodquad',
5: 'goodquint',
6: 'goodhex',
7: 'goodsept',
8: 'goodoct'
}
# some calculations to get stats
total = len(goodNPlusOne) + len(goodUnchanged)
correctUnchanged = [x for x in goodUnchanged if x.sameFake()]
correctNPlusOne = [x for x in goodNPlusOne if x.sameFake()]
notsame = total - len(correctUnchanged) - len(correctNPlusOne)
inputFakeid = list(set([y.fakeid for y in x.dets for x in nlets]))
unchangedFakeid = list(set([x.dets[0].fakeid for x in correctUnchanged]))
nPlusOneFakeid = list(set([x.dets[0].fakeid for x in correctNPlusOne]))
'''
A fake with fakeid X is considered recoverable if there exists an nlet composed of
detections only of fakeid X (i.e. all detections have fakeid X)
'''
print('Number of input ' + numobj[objcount] + ': ' + str(len(nlets)))
print('Number of input fakeids ' + str(len(inputFakeid)))
print('Number of good fits remaining: ' + str(total))
print('Number of good remaining ' + numobj[objcount] + ': ' +
str(len(correctUnchanged)))
print('Number of good remaining ' + numobj[objcount + 1] + ': ' +
str(len(correctNPlusOne)))
print('Number of recoverable fakes in remaining ' + numobj[objcount] +
': ' + str(len(unchangedFakeid)))
print('Number of recoverable fakes in remaining ' + numobj[objcount + 1] +
': ' + str(len(nPlusOneFakeid)))
print('Number of recoverable fakes remaining: ' +
str(len(set(unchangedFakeid + nPlusOneFakeid))))
print('\n Checking duplicates:')
print('Input:')
checkDuplicates(nlets, numobj[objcount])
print('Processed:')
checkDuplicates(goodNPlusOne, numobj[objcount + 1])
checkDuplicates(goodUnchanged, numobj[objcount])
#print('Number of duplicates in ' + numobj[objcount+1] + ' :' + str(numdup_np1))
savename = args.nlets[0].split('+')[-1].split('.')[0]
saveas_n1 = names[objcount + 1] + str(args.chisq) + '+' + savename
saveas_u = names[objcount] + 'U' + str(args.chisq) + '+' + savename
# save unchanged triplet
writeTriplets(goodUnchanged, saveas_u + '.txt', False)
pickleTriplets(goodUnchanged, saveas_u + '.pickle')
writeTriplets(goodNPlusOne, saveas_n1 + '.txt', False)
pickleTriplets(goodNPlusOne, saveas_n1 + '.pickle')
