def gtdiffrsp_mp(bins, SCFile, EVFile, OutFile, SaveTemp, SrcModel, IRF): '''This function looks at the start and stop times in an event file and splits the time into chunks. It then submits jobs based upon those start and stop times.''' print "Opening event file to determine break points..." hdulist = pyfits.open(EVFile) tstart = hdulist[0].header['TSTART'] tstop = hdulist[0].header['TSTOP'] hdulist.close() starts, step = np.linspace(tstart, tstop, bins, endpoint=False, retstep=True) stops = starts + step scfiles = [SCFile for st in starts] evfiles = [EVFile for st in starts] srcmdls = [SrcModel for st in starts] irfs = [IRF for st in starts] pool = Pool(processes=bins) times = np.array([starts, stops, scfiles, evfiles, srcmdls, irfs]) print "Spawning {} jobs...".format(bins) tempfilenames = pool.map(diffrsp, times.transpose()) print "Combining temporary files..." eventsum(tempfilenames, OutFile, SaveTemp)
def expsum(filenames, Outfile, SaveTemp): '''This function takes a list of exposure maps and adds them together. If there is only one file to be added, it just copies it to the outfile. If there is more than one, it uses pyfits to open them all up, sum the first hdus (the data) and then replaces the first file's primary hdu with this summed data and writes it to the output file.''' if len(filenames) <= 1: subprocess.call(["cp", filenames[0], Outfile]) else: expmap_files = [pyfits.open(filename) for filename in filenames] summed_expmap_hdu = (np.array([expmap_file[0].data for expmap_file in expmap_files])).sum(axis=0) expmap_files[0][0].data = summed_expmap_hdu expmap_files[0][0].update_header() expmap_files[0].writeto(Outfile, clobber='True') for expmap_file in expmap_files: expmap_file.close() if SaveTemp: print "Did not delete the following temporary files:" print filenames else: print "Deleting temporary files..." for filename in filenames: os.remove(filename)
def gtdiffrsp_mp(bins, SCFile, EVFile, OutFile, SaveTemp, SrcModel,IRF): '''This function looks at the start and stop times in an event file and splits the time into chunks. It then submits jobs based upon those start and stop times.''' print "Opening event file to determine break points..." hdulist = pyfits.open(EVFile) tstart = hdulist[0].header['TSTART'] tstop = hdulist[0].header['TSTOP'] hdulist.close() starts, step = np.linspace(tstart,tstop,bins,endpoint=False, retstep=True) stops = starts + step scfiles = [SCFile for st in starts] evfiles = [EVFile for st in starts] srcmdls = [SrcModel for st in starts] irfs = [IRF for st in starts] pool = Pool(processes=bins) times = np.array([starts,stops,scfiles,evfiles,srcmdls,irfs]) print "Spawning {} jobs...".format(bins) tempfilenames = pool.map(diffrsp,times.transpose()) print "Combining temporary files..." eventsum(tempfilenames, OutFile, SaveTemp)
def srcmapssum(results, ref_hdu, outfile, savetemp): '''This function takes a list of source maps and joins them together. If there is only one file to be joined, it just copies it to the outfile. If there is more than one, it uses pyfits to open them all up, add the energy slices together and then outputs this. The first element in the list is the index.''' results.sort() if len(results) <= 1: subprocess.call(["cp", results[0][1], outfile]) else: SrcMaps = [pyfits.open(result[1]) for result in results] for idx,sourceHDU in enumerate(SrcMaps[0][3:]): newHDU = SrcMaps[0][idx+3].copy() newHDU.data = np.delete(SrcMaps[0][idx+3].data,-1,0) for SrcMap in SrcMaps[1:]: newHDU.data = np.append(newHDU.data,SrcMap[idx+3].data[:-1],axis=0) newHDU.data = np.append(newHDU.data,[SrcMaps[-1][idx+3].data[-1]],axis=0) #newHDU.add_checksum() ref_hdu.append(newHDU) ref_hdu.writeto(outfile,clobber='yes') if savetemp: print "Did not delete the following temporary files:" print results else: print "Deleting temporary files..." for result in results: os.remove(result[1])
def expsum(filenames, Outfile, SaveTemp): '''This function takes a list of exposure maps and adds them together. If there is only one file to be added, it just copies it to the outfile. If there is more than one, it uses pyfits to open them all up, sum the first hdus (the data) and then replaces the first file's primary hdu with this summed data and writes it to the output file.''' if len(filenames) <= 1: subprocess.call(["cp", filenames[0], Outfile]) else: expmap_files = [pyfits.open(filename) for filename in filenames] summed_expmap_hdu = (np.array( [expmap_file[0].data for expmap_file in expmap_files])).sum(axis=0) expmap_files[0][0].data = summed_expmap_hdu expmap_files[0][0].update_header() expmap_files[0].writeto(Outfile, clobber='True') for expmap_file in expmap_files: expmap_file.close() if SaveTemp: print "Did not delete the following temporary files:" print filenames else: print "Deleting temporary files..." for filename in filenames: os.remove(filename)
def gtsrcmaps_mp(nxpix, nypix, binsz, scfile, evfile, expcube, ccube, coordsys, proj, xref, yref, axisrot, ebinalg, emin, emax, enumbins,jobs, srcmdl, bexpmap, irfs, outfile, savetmp): '''This function opens up the input counts map and divides up the energy bins into seperate jobs which are sent to the pool to be run.''' ref_hdu = pyfits.open(ccube) energies = np.array(ref_hdu[1].data) #There's an error when you only use one bin so you need to #make sure and compute more than one bin per job. while len(energies)/float(jobs) < 3: print "Too many jobs ({}), reducing by 1".format(jobs) jobs -= 1 print "Jobs is now {}".format(jobs) energy_arrays = np.array_split(energies,jobs) options = {'nxpix': nxpix, 'nypix': nypix, 'binsz': binsz, 'scfile': scfile, 'evfile': evfile, 'coordsys': coordsys, 'xref': xref, 'yref': yref, 'axisrot': axisrot, 'proj': proj, 'ebinalg': ebinalg, 'emin': emin, 'emax': emax, 'enumbins': enumbins, 'expcube': expcube, 'srcmdl': srcmdl, 'bexpmap': bexpmap, 'irfs': irfs} SQ = [(array,options) for array in energy_arrays] pool = Pool(processes=jobs) print "Spawning {} jobs...".format(jobs) results = pool.map(run_gtsrcmaps,SQ) print "Combining temporary files..." srcmapssum(results, ref_hdu, outfile, savetmp)
def gtltcube_mp(bins, SCFile, EVFile, OutFile, SaveTemp, zmax, tmin, tmax): '''This functions looks at a spacecraft file and splits the time into chunks that match the bin edges in the spacecraft file. It then submits jobs based upon those start and stop times. This is to make the resulting files as close to the original as possible. Note that this assumes you are using the full time period in your spacecraft file.''' verbose = False if EVFile != "": evfile = pyfits.open(EVFile, mode='readonly') gti_data = evfile[2].data if tmin == 0: print "Determining start and stop times from the event file..." tstart = evfile[0].header['TSTART'] tstop = evfile[0].header['TSTOP'] else: print "Using user defined tmin and tmax..." tstart = tmin tstop = tmax print "Opening SC file to determine break points..." hdulist = pyfits.open(SCFile, mode='readonly') scdata = hdulist[1].data hdulist.close() scstart = scdata.field('START') scstop = scdata.field('STOP') time_filter = (tstart <= scstart) & (scstop <= tstop) redo = True if EVFile != "": print "Checking for good times in the event file..." while redo: redo = False scstartssplit = np.array_split(scstart[time_filter], int(bins)) scstopssplit = np.array_split(scstop[time_filter], bins) #Explicitly set the first and last point to the values in the evfile header scstartssplit[0][0] = tstart scstopssplit[-1][-1] = tstop starts = [st[0] for st in scstartssplit] stops = [st[-1] for st in scstopssplit] if EVFile != "": for interval in zip(starts, stops): if verbose: print "Looking at interval", interval[0], "to", interval[1] good_times = False #grrrr. some bug in pyfits doesn't let me do this the python way... for gti_i in range(len(gti_data)): if (not good_times): if verbose: print " Checking gti", gti_data[gti_i][ 'START'], "to", gti_data[gti_i]['STOP'] gti_starts = interval[0] <= gti_data[gti_i][ 'START'] <= interval[1] gti_stops = interval[0] <= gti_data[gti_i][ 'STOP'] <= interval[1] if verbose: print " Does this gti start inside this interval? ", gti_starts if verbose: print " Does this gti stop inside this interval? ", gti_stops good_times = gti_starts or gti_stops if verbose: print if verbose: print " Are there good times inside this interval? ", good_times if not good_times: redo = True if verbose: print if redo: if bins <= 1: print "No good time intervals found. Bailing..." sys.exit(1) print "One (or more) of the slices doesn't have a GTI." print "Reducing the number of threads from ", bins, "to", bins - 1 bins -= 1 scfiles = [SCFile for st in scstartssplit] evfiles = [EVFile for st in scstartssplit] print "EVFiles:", evfiles zmaxes = [zmax for st in scstartssplit] pool = Pool(processes=bins) times = np.array([starts, stops, scfiles, evfiles, zmaxes]) print "Spawning {} jobs...".format(bins) tempfilenames = pool.map(ltcube, times.transpose()) print "Combining temporary files..." ltsum(tempfilenames, OutFile, SaveTemp)
def gtltcube_mp(bins, SCFile, EVFile, OutFile, SaveTemp, zmax, tmin, tmax): '''This functions looks at a spacecraft file and splits the time into chunks that match the bin edges in the spacecraft file. It then submits jobs based upon those start and stop times. This is to make the resulting files as close to the original as possible. Note that this assumes you are using the full time period in your spacecraft file.''' verbose = False if EVFile != "": evfile = pyfits.open(EVFile, mode='readonly') gti_data = evfile[2].data if tmin == 0: print "Determining start and stop times from the event file..." tstart = evfile[0].header['TSTART'] tstop = evfile[0].header['TSTOP'] else: print "Using user defined tmin and tmax..." tstart = tmin tstop = tmax print "Opening SC file to determine break points..." hdulist = pyfits.open(SCFile, mode='readonly') scdata = hdulist[1].data hdulist.close() scstart = scdata.field('START') scstop = scdata.field('STOP') time_filter = (tstart <= scstart) & (scstop <= tstop) redo = True if EVFile !="": print "Checking for good times in the event file..." while redo: redo = False scstartssplit = np.array_split(scstart[time_filter],int(bins)) scstopssplit = np.array_split(scstop[time_filter],bins) #Explicitly set the first and last point to the values in the evfile header scstartssplit[0][0] = tstart scstopssplit[-1][-1] = tstop starts = [st[0] for st in scstartssplit] stops = [st[-1] for st in scstopssplit] if EVFile != "": for interval in zip(starts,stops): if verbose: print "Looking at interval",interval[0],"to",interval[1] good_times = False #grrrr. some bug in pyfits doesn't let me do this the python way... for gti_i in range(len(gti_data)): if(not good_times): if verbose: print " Checking gti",gti_data[gti_i]['START'],"to",gti_data[gti_i]['STOP'] gti_starts = interval[0] <= gti_data[gti_i]['START'] <= interval[1] gti_stops = interval[0] <= gti_data[gti_i]['STOP'] <= interval[1] if verbose: print " Does this gti start inside this interval? ", gti_starts if verbose: print " Does this gti stop inside this interval? ", gti_stops good_times = gti_starts or gti_stops if verbose: print if verbose: print " Are there good times inside this interval? ", good_times if not good_times: redo = True if verbose: print if redo: if bins <= 1: print "No good time intervals found. Bailing..." sys.exit(1) print "One (or more) of the slices doesn't have a GTI." print "Reducing the number of threads from ",bins,"to",bins-1 bins -= 1 scfiles = [SCFile for st in scstartssplit] evfiles = [EVFile for st in scstartssplit] print "EVFiles:",evfiles zmaxes = [zmax for st in scstartssplit] pool = Pool(processes=bins) times = np.array([starts,stops,scfiles,evfiles,zmaxes]) print "Spawning {} jobs...".format(bins) tempfilenames = pool.map(ltcube,times.transpose()) print "Combining temporary files..." ltsum(tempfilenames, OutFile, SaveTemp)