def main(): args = argparse.ArgumentParser() args.add_argument('triplets', nargs=1, help='path to pickle file; ' + 'file has format triplet+SNOBS_SEASON###_ML0#.pickle') args.add_argument('numChunks', nargs='?', default=100, help='number of chunks') args = args.parse_args() print('open pickle file') pickleFile = open(args.triplets[0], 'rb') print('loading pickle file') triplets = pickle.load(pickleFile) tripChunks = splitList(triplets, int(args.numChunks)) #save triplets saveName = args.triplets[0].split('+')[-1].split('.')[0] saveName2 = args.triplets[0].split('+')[-2] if '/' in saveName2: saveName2 = saveName2.split('/')[-1] print(saveName2) for x in range(len(tripChunks)): writeTriplets( tripChunks[x], 'chunk' + str(x) + '+' + saveName2 + '+' + saveName + '.txt', False) pickleTriplets( tripChunks[x], 'chunk' + str(x) + '+' + saveName2 + '+' + saveName + '.pickle')
def main(): args = argparse.ArgumentParser() args.add_argument('triplets', nargs=1, help='path to pickle file') args = args.parse_args() triplets = pickle.load(open(args.triplets[0], 'rb')) saveName = args.triplets[0].split('+')[-1].split('.')[0] # merges all triplets onto a graph G, dict = connectGraph(triplets) # know the tripelts inside each connected components d = organizetriplets(G, triplets) good = growtrees(G, d) #saves every merged triplets saveGoodName = 'goodMergedTriplets3+' + saveName #saveBadName = 'badTriplets+' + saveName writeTriplets(good, saveGoodName + '.txt') #writeTriplets(bad, saveBadName + '.txt') pickleTriplets(good, saveGoodName + '.pickle') #pickleTriplets(bad, saveBadName + '.pickle') # graphs original merged graph #graphLinks(G, dict, saveName) G, dict = connectGraph(good) graphLinks(G, dict, saveName) #, [3.4,5.2,-1,0]) '''
def main(): args = argparse.ArgumentParser() args.add_argument( 'linkedPairs', nargs=1, help='path to pickle file; ' + 'file has format detectionLinks+SNOBS_SEASON###_ML0#.pickle') args.add_argument('numChunks', nargs='?', default=1, help='number of chunks') args.add_argument('ncpus', nargs='?', default=1, help='number of cpus') args = args.parse_args() #load the pickle file with list of detections and links print('load pickle file') detPairs = pickle.load(open(args.linkedPairs[0], 'rb')) #makes a list of potential triplets Ncpu = int(args.ncpus) triplets = formTriplets((detPairs, 0)) numChunks = args.numChunks tripChunks = splitList(triplets, numChunks) #triplets = multiProcessPairs(detPairs, Ncpu) saveName = args.linkedPairs[0].split('+')[-1].split('.')[0] for x in range(len(tripChunks)): writeTriplets(tripChunks[x], 'chunk{0:04d}'.format(x) + '+' + saveName + '.txt', False) pickleTriplets(tripChunks[x], 'chunk{0:04d}'.format(x) + '+' + saveName + '.pickle')
def main(): args = argparse.ArgumentParser() args.add_argument('triplets', nargs='+', help='list of triplets to merge') args.add_argument('-f', '--fake', action='store_true', help='whether to look at fakes') args = args.parse_args() savename1 = args.triplets[0].split('/')[-1].split('.')[0] savename = 'merged+' + savename1 # Sets the name for the save (which begins with "merged") triplets = [] print('loading triplets...') time0 = time.time() # Records an inital time for trips in args.triplets: triplets.extend(pickle.load(open(trips, 'rb'))) # Opens the file print('done loading after ' + str(time.time() - time0) + ' seconds') if (args.fake): mergedTrips = mergeFakes(triplets) else: siftedTrips = getInitTrips(triplets, savename1) mergedTrips = newMergeTrips(siftedTrips, savename1) print('\nsize of final list = ' + str(len(mergedTrips))) finalList = [] for trip in mergedTrips: if (trip.realLength() > 4): finalList.append(trip) # Keeps triplets with five or more detections. print('size after reducing: ' + str(len(finalList))) writeTriplets(finalList, savename + '.txt') pickleTriplets(finalList, savename + '.pickle')
def main(): parser = argparse.ArgumentParser() parser.add_argument('nlets', nargs=1, help='path to nlet pickle file') parser.add_argument('chisq', nargs='?', default=5, help='chisq threshold for good orbit') args = parser.parse_args() print("Loading pickle file") nlets = pickle.load(open(args.nlets[0], 'rb')) print("Done loading") savename = args.nlets[0].split('+')[-1].split('.')[0] print('Recursively merging...') while len(nlets) != 0: t0 = time.time() numdet = len(nlets[0].dets) print('Processing ' + str(len(nlets)) + ' ' + str(numdet) + '-lets') minRA, maxRA, minDec, maxDec = getBounds(nlets) arr = partition(nlets, minRA, maxRA, minDec, maxDec, 700, 700) nPlusOne, unchanged = combine(arr, numdet, True) # remove duplicates nPlusOne = removeDuplicates(nPlusOne) unchanged = removeDuplicates(unchanged) # remove same exposure nPlusOne = removeSameExposure(nPlusOne) unchange = removeSameExposure(unchanged) print("\nChecking good orbits for " + str(len(nPlusOne)) + " n+1 lets") # check for good orbits nPlusOne = checkGoodOrbit(nPlusOne, float(args.chisq), progress=True) print("Checking good orbits for " + str(len(unchanged)) + " nlets") unchanged = checkGoodOrbit(unchanged, float(args.chisq), progress=True) num_u = len(unchanged) if num_u != 0: print('Saving ' + str(num_u) + ' unchanged ' + str(numdet) + '-lets') saveas_u = 'good' + str(numdet) + 'letU' + str( args.chisq) + '+' + savename writeTriplets(unchanged, saveas_u + '.txt', False) pickleTriplets(unchanged, saveas_u + '.pickle') nlets = nPlusOne print('Time taken this cycle: ' + str(time.time() - t0)) print('Done')
def main(): args = argparse.ArgumentParser() args.add_argument('triplets', help='path to pickle file; file has format ' + 'chunk###+SNOBS_S\EASON###_ML0#.pickle') args.add_argument('-d', '--detFile', help='path to csv file with detections') args.add_argument('-o', '--orbit', action='store_true', help='produce orbitfile only') args.add_argument('-x', '--suppress', action='store_true', help='do not run fitter') args = args.parse_args() print('\nopen pickle file ' + args.triplets) with open(args.triplets, 'rb') as pickleFile: print('load pickle file') triplets = pickle.load(pickleFile) print('done loading') saveName = args.triplets.split('+')[-1].split('.')[0] chunkName = args.triplets.split('/')[-1].split('+')[0] if (args.suppress): lets = removeBadChisq(triplets) else: if (args.detFile is None): detDict = {} else: detDict = LL.objidDictionary(args.detFile) try: triplets = list(set(triplets)) except TypeError: triplets = triplets outname, trackIDs, lets = writeProcessingFile(triplets, detDict, chunkName, saveName) fitsFile = callBulkFit(outname) if (args.orbit): callBulkEle(fitsFile) else: lets = siftTrips(fitsFile, trackIDs) # Saving and pickling triplets to chunkName + '+goodtriplets+' + saveName + '.txt' writeTriplets(lets, chunkName + '+goodtriplets+' + saveName + '.txt', args.orbit) pickleTriplets(lets, chunkName + '+goodtriplets+' + saveName + '.pickle', rmunbound=False)
def main(): args = argparse.ArgumentParser() args.add_argument('triplets', nargs=1, help='path to pickle file; file has format ' + 'chunk###+SNOBS_SEASON###_ML0#.pickle') #args.add_argument('chisq', nargs='?', default=5, # help='chisq threshold to be considered a good triplet') args.add_argument('ncpus', nargs='?', default=1, help='number of cpus') args = args.parse_args() print('open pickle file ' + args.triplets[0]) pickleFile = open(args.triplets[0], 'rb') print('load pickle file') triplets = pickle.load(pickleFile) Ncpu = int(args.ncpus) #checks orbits for good fit goodTrips, missTrips, badTrips, p9Trips = potentialTriplets((triplets, 0)) #multiProcessTriplets(triplets, Ncpu) #saving triplets saveName = args.triplets[0].split('+')[-1].split('.')[0] chunkName = args.triplets[0].split('/')[-1].split('+')[0] saveName = chunkName + '+' + saveName writeTriplets(goodTrips, 'goodtriplets+' + saveName + '.txt', False) writeTriplets(missTrips, 'misstriplets+' + saveName + '.txt', False) writeTriplets(badTrips, 'badtriplets+' + saveName + '.txt', False) writeTriplets(p9Trips, 'p9triplets+' + saveName + '.txt', False) pickleTriplets(goodTrips, 'goodtriplets+' + saveName + '.pickle') pickleTriplets(missTrips, 'misstriplets+' + saveName + '.pickle') pickleTriplets(badTrips, 'badtriplets+' + saveName + '.pickle') pickleTriplets(p9Trips, 'p9triplets+' + saveName + '.pickle')
def main(): args = argparse.ArgumentParser() args.add_argument('linkedPairs', help='path to pickle file; ' + 'file has format detectionLinks+SNOBS_SEASON###_ML0#.pickle') args.add_argument('-n', '--chunkSize', help='size of chunks, leave empty for only one chunk') args.add_argument('-c', '--cont', help='last chunk processed') args = args.parse_args() # Three arguments, of which the last two are optional. tripletStart = [-1,-1,-1] chunkStart = 1 if(args.cont): print('loading last chunk: ' + args.cont) fname = args.cont.split('+')[0] if('/' in fname): fname = fname.split('/')[-1] chunkStart = int(fname[-6:]) + 1 with open(args.cont, 'rb') as f: lastChunk = pickle.load(f) lastTrip = lastChunk[-1] tripletStart = lastTrip[1] print('last triplet: ' + str(tripletStart)) print('starting from chunk: ' + str(chunkStart)) # load the pickle file with list of detections and links print('load pickle file') detPairs = pickle.load(open(args.linkedPairs, 'rb')) print('done loading') # makes a list of potential triplets chunkSize = -1 if(args.chunkSize): chunkSize = int(args.chunkSize) saveName = args.linkedPairs.split('+')[-1].split('.')[0] print('forming triplets') triplets = formTriplets(detPairs, chunkSize, saveName, tripletStart, chunkStart) # tripChunks = splitList(triplets, numChunks, buffered, saveName, printP) # triplets = multiProcessPairs(detPairs, Ncpu) # we save it earlier if argument is buffered x = 0 pickleTriplets(triplets, 'chunk{0:06d}'.format(x) + '+' + saveName + '.pickle', False) writeTriplets(triplets, 'chunk{0:06d}'.format(x) + '+' + saveName + '.txt', False, False)
def main(): args = argparse.ArgumentParser() args.add_argument('detList', nargs=1, help='path to csv list of detections') args.add_argument('potentialNlets', nargs=1, help='Nlets we want to add on to') args = args.parse_args() print('\nloading nlets ' + args.potentialNlets[0]) nlets = pickle.load(open(args.potentialNlets[0], 'rb')) nameList = args.potentialNlets[0].split('+') saveName = nameList[-1].split('.')[0] saveName2 = nameList[-2] if '/' in saveName2: saveName2 = saveName2.split('/')[-1] print(saveName2) saveName3 = '' if (len(nameList) >= 3): saveName3 = nameList[-3] if '/' in saveName3: saveName3 = saveName3.split('/')[-1] print('\nwrapping detections: ' + args.detList[0]) detList = wrapDets(args.detList[0]) print('\nmaking dictionary') expDict = expDictionary(detList) size = len(nlets) counter = 0 time0 = time.time() newNletList = [] for nlet in nlets: counter += 1 print('\n*****adding to Nlet #' + str(counter) + ' of ' + str(size) + ' *****') print('OLD TRIPLET:') print(nlet.toStr()) newLet = addToNlet(nlet, expDict) print('\nNEW TRIPLET:') print(newLet.toStr()) newNletList.append(newLet) name = 'crossCampaignTriplets+' + saveName3 + '+' + saveName2 + '+' + saveName writeTriplets(newNletList, name + '.txt', True) pickleTriplets(newNletList, name + '.pickle')
def main(): args = argparse.ArgumentParser() args.add_argument('detList', nargs=1, help='path to csv list of detections') args.add_argument('predList', nargs=1, help='path to list of predicted detections') args = args.parse_args() saveName = args.predList[0].split('+')[-1].split('.')[0] # print(args.detList[0].split('+')) saveName2 = args.predList[0].split('+')[-2] print(saveName2) print('wrapping detections: ' + args.detList[0]) detList = wrapDets(args.detList[0]) print('making dictionary') expDict = expDictionary(detList) print('loading prediction list: ' + args.predList[0]) predList = pickle.load(open(args.predList[0])) newTripList = [] size = len(predList) counter = 0 for tup in predList: counter += 1 print('*****adding to Nlet #' + str(counter) + ' of ' + str(size) + ' *****') print('OLD TRIPLET:') print(tup[0].toStr()) newTriplet = addDetections(tup[0], tup[1], expDict) print(newTriplet.toStr()) newTripList.append(newTriplet) name = 'crossCampaignTriplets+' + saveName2 + '+' + saveName writeTriplets(newTripList, name + '.txt', True) pickleTriplets(newTripList, name + '.pickle')
def main(): args = argparse.ArgumentParser() args.add_argument('folder', nargs=1, help='path to folder of chunks') args = args.parse_args() files = os.listdir(args.folder[0]) tripList = [] saveName = '' print('opening files in: ' + args.folder[0]) time0 = time.time() counter = 0 for f in files: counter += 1 printPercentage(counter, len(files), time.time() - time0) if (f.split('+')[0] == 'goodtriplets' and f.split('.')[-1] == 'pickle'): print('\nopening: ' + args.folder[0] + f) trips = pickle.load(open(args.folder[0] + f, 'rb')) saveName = f.split('+')[-1].split('.')[0] for t in trips: tripList.append(t) writeTriplets(tripList, 'goodTriplets+' + saveName + '.txt', True) pickleTriplets(tripList, 'goodTriplets+' + saveName + '.pickle')
def main(): parser = argparse.ArgumentParser() parser.add_argument('nlets', nargs=1, help='path to nlet pickle file') parser.add_argument('chisq', nargs='?', default=5, help='chisq threshold for good orbit') args = parser.parse_args() nlets = pickle.load(open(args.nlets[0], 'rb')) savename = args.nlets[0].split('+')[-1].split('.')[0] chisq = float(args.chisq) merged = [] print('Recursively merging...') minRA, maxRA, minDec, maxDec = getBounds(nlets) while len(nlets) != 0: t0 = time.time() numdet = len(nlets[0].dets) print('Processing ' + str(len(nlets)) + ' ' + str(numdet) + '-lets') arr = partition(nlets, minRA, maxRA, minDec, maxDec, 700, 700) nPlusOne, unchanged = combine(arr, numdet) # remove duplicates nPlusOne = removeDuplicates(nPlusOne) unchanged = removeDuplicates(unchanged) # remove same exposure nPlusOne = removeSameExposure(nPlusOne) unchange = removeSameExposure(unchanged) # check for good orbits nPlusOne = checkGoodOrbit(nPlusOne, float(args.chisq)) unchanged = checkGoodOrbit(unchanged, float(args.chisq)) merged.append(unchanged) nlets = nPlusOne print('Time taken this cycle: ' + str(time.time() - t0)) print('Done N-1 Merging') print('Starting Half Merge...') stepRA = arr[0][0].raHi - arr[0][0].raLo stepDec = arr[0][0].decHi - arr[0][0].decLo # start with nlet with most number of detections merged.reverse() remaining = [] leftover = merged[0] for i in range(len(merged) - 1): if len(leftover) == 0: print('ERROR: No nlets left in this cycle') continue arr = partition(leftover, minRA, maxRA, minDec, maxDec, 700, 700) numdet = len(leftover[0].dets) print("Self merging " + str(len(leftover)) + ' ' + str(numdet) + "-let") sameCombined, sameUnchanged = generalCombine(arr, (numdet + 1) / 2, chisq) renewArray(arr) for oneless in merged[i + 1]: insertNLet(oneless, arr, minRA, minDec, stepRA, stepDec) print("Second stage merging with " + str(numdet - 1) + "-let") mixCombined, mixUnchanged = generalCombine( arr, (len(merged[i + 1][0].dets) + 1) / 2, chisq) remaining = remaining + sameCombined + sameUnchanged + mixCombined leftover = mixUnchanged print('Removing duplicates and same exposures') # remove duplicates and same exposure remaining = removeDuplicates(remaining) remaining = removeSameExposure(remaining) remaining = compartmentalize(remaining) savename = args.nlets[0].split('+')[-1].split('.')[0] for rem in remaining: saveas = 'good' + str(len( rem[0].dets)) + "letHM" + str(chisq) + '+' + savename writeTriplets(rem, saveas + '.txt', False) pickleTriplets(rem, saveas + '.pickle') print('Done')
def main(): parser = argparse.ArgumentParser() parser.add_argument('nlets', nargs=1, help='path to nlet pickle file') parser.add_argument('chisq', nargs='?', default=5, help='chi-square threshold for orbit fitter') args = parser.parse_args() nlets = pickle.load(open(args.nlets[0], 'rb')) # identify the number of detections in each Triplet object objcount = len(nlets[0].dets) t0 = time.time() # First get bounds of season # Get the array of partitioned season # Each cell of the array holds Triplet object with detections that fall in that region # if Triplet object have 5 detections (quint) then it will belong to up to 5 cells # Finally combine function merges nlet to form (n+1)let minRA, maxRA, minDec, maxDec = getBounds(nlets) arr = partition(nlets, minRA, maxRA, minDec, maxDec, 800, 800) nPlusOne, unchanged = combine(arr, objcount) # remove duplicates nPlusOne = removeDuplicates(nPlusOne) unchanged = removeDuplicates(unchanged) # remove same exposure nPlusOne = removeSameExposure(nPlusOne) unchanged = removeSameExposure(unchanged) # check for good orbits print('Orbit fit test in progress') goodNPlusOne = checkGoodOrbit(nPlusOne, float(args.chisq)) goodUnchanged = checkGoodOrbit(unchanged, float(args.chisq)) t1 = time.time() - t0 print('Time taken: ' + str(t1)) # dictionary for savename numobj = { 3: 'triplet', 4: 'quad', 5: 'quint', 6: 'hex', 7: 'sept', 8: 'oct' } # Now saving (n+1)let names = { 3: 'goodtriplet', 4: 'goodquad', 5: 'goodquint', 6: 'goodhex', 7: 'goodsept', 8: 'goodoct' } # some calculations to get stats total = len(goodNPlusOne) + len(goodUnchanged) correctUnchanged = [x for x in goodUnchanged if x.sameFake()] correctNPlusOne = [x for x in goodNPlusOne if x.sameFake()] notsame = total - len(correctUnchanged) - len(correctNPlusOne) inputFakeid = list(set([y.fakeid for y in x.dets for x in nlets])) unchangedFakeid = list(set([x.dets[0].fakeid for x in correctUnchanged])) nPlusOneFakeid = list(set([x.dets[0].fakeid for x in correctNPlusOne])) ''' A fake with fakeid X is considered recoverable if there exists an nlet composed of detections only of fakeid X (i.e. all detections have fakeid X) ''' print('Number of input ' + numobj[objcount] + ': ' + str(len(nlets))) print('Number of input fakeids ' + str(len(inputFakeid))) print('Number of good fits remaining: ' + str(total)) print('Number of good remaining ' + numobj[objcount] + ': ' + str(len(correctUnchanged))) print('Number of good remaining ' + numobj[objcount + 1] + ': ' + str(len(correctNPlusOne))) print('Number of recoverable fakes in remaining ' + numobj[objcount] + ': ' + str(len(unchangedFakeid))) print('Number of recoverable fakes in remaining ' + numobj[objcount + 1] + ': ' + str(len(nPlusOneFakeid))) print('Number of recoverable fakes remaining: ' + str(len(set(unchangedFakeid + nPlusOneFakeid)))) print('\n Checking duplicates:') print('Input:') checkDuplicates(nlets, numobj[objcount]) print('Processed:') checkDuplicates(goodNPlusOne, numobj[objcount + 1]) checkDuplicates(goodUnchanged, numobj[objcount]) #print('Number of duplicates in ' + numobj[objcount+1] + ' :' + str(numdup_np1)) savename = args.nlets[0].split('+')[-1].split('.')[0] saveas_n1 = names[objcount + 1] + str(args.chisq) + '+' + savename saveas_u = names[objcount] + 'U' + str(args.chisq) + '+' + savename # save unchanged triplet writeTriplets(goodUnchanged, saveas_u + '.txt', False) pickleTriplets(goodUnchanged, saveas_u + '.pickle') writeTriplets(goodNPlusOne, saveas_n1 + '.txt', False) pickleTriplets(goodNPlusOne, saveas_n1 + '.pickle')