def __init__(self, nprocs, njobs, progress=True, log=True): """ Initialize the input/output queues and make the workers """ # set up the results queue self.results = Queue() # set up the tasks queue self.tasks = Queue() # hold the dequeued results self.deqd_results = [] self.nprocs = nprocs self.log = log if self.log: # redirect stderr to a file self.temp_stderr = tempfile.TemporaryFile() sys.stderr = self.temp_stderr # make a unique file name for std out fileName, extension = os.path.splitext(os.path.basename(sys.argv[0])) time_stamp = utilities.get_timestamp() self.stdout = open(os.getcwd() + os.sep + "%s.%s.log" \ %(fileName, time_stamp), 'w') sys.stdout = self.stdout # set up the logger to log to sys.stderr self.logger = multiprocessing.log_to_stderr() self.logger.setLevel(logging.INFO) # if we want a progress bar if progress and progressLoaded: self.bar = initializeProgressBar(njobs, fd=sys.__stderr__) else: self.bar = None # create an exception event self.exception = multiprocessing.Event()
def fourierSpaceCorrFromRandoms(paramsToUpdate, valsToUpdate, nProcs, corrType="cross"): """ @brief compute the real space correlation between a true data map and random map, for the nMaps pairs of maps @param paramsToUpdate: the parameters to change for each pair of maps that we are correlating (dict) @param valsToUpdate: the numbers to insert into the paramsToUpdate values (list) @param nProcs: the number of processes to use (int) @keyword corrType: type of spectrum to compute ('cross' or 'auto') """ num_jobs = len(valsToUpdate) # the number of correlations to do # read the base parameter file params = flipperDict.flipperDict() try: params.readFromFile("global.dict") except: raise originalStdOut = sys.stdout # might not have this package try: bar = initializeProgressBar(num_jobs) except: pass def worker(job_queue, results_queue): """ @brief worker function for multiprocessing """ # pull tasks until there are none left while True: # dequeue the next job next_task = job_queue.get() # task=None means this worker is finished if next_task is None: # make sure we tell the queue we finished the task job_queue.task_done() break else: # tasks are tuples of params oldParams, num = next_task # try to update the progress bar try: bar.update(num + 1) except: pass initialPath = os.getcwd() # do the work # make new directory and cd there if not os.path.exists("tmp_%d" % num): os.makedirs("tmp_%d" % num) os.chdir("tmp_%d" % num) # update the parameters newParams = xcorrUtils.updateParams(paramsToUpdate, oldParams, valsToUpdate[num]) # hide output from the speck(Cross) output sys.stdout = open(os.devnull, "w") # compute the fourier space correlation, given the parameters if corrType == "cross": err = runSpeckCross(newParams) if corrType == "auto": err = runSpeck(newParams) # go back to old directory and delete temporary directory os.chdir(initialPath) if os.path.exists("tmp_%d" % num): os.system("rm -rf ./tmp_%d" % num) # restore stdout to original value sys.stdout = originalStdOut # store the results results_queue.put((num)) # make sure we tell the queue we finished the task job_queue.task_done() return 0 # store the system time sys_time = time.time() # establish communication queues that contain all jobs job_numbers = mp.JoinableQueue() results = mp.Queue() # create a process for each cpu available or up to the limit specified by user if nProcs <= mp.cpu_count(): num_workers = nProcs else: num_workers = mp.cpu_count() print "Creating %d workers" % num_workers procs = [mp.Process(target=worker, args=(job_numbers, results)) for i in xrange(num_workers)] # start the processes for proc in procs: proc.start() # enqueue the positions for i in xrange(num_jobs): job_numbers.put((params, i)) # Add a poison pill (=None) for each worker for i in xrange(num_workers): job_numbers.put(None) # wait for all of the jobs to finish job_numbers.join() return 0
def realSpaceCorr(params): """ @brief compute the real space correlation between two maps and output the relevant files @param params: dictionary containing the following: totalMapPairs: the total pairs of maps to compute correlation over (int) root_A: the file tag for map #1 (str) root_B: the file tag for map #2 (str) thetaExtent: the angular extent to compute correlation to, in degrees (float) files_A: array containing filenames of map A (can be more than 1 non overlapping regions) files_B: array containing filenames of map B (can be more than 1 non overlapping regions) makePlots: whether to plot the results (bool) """ totalMapPairs = params['totalMapPairs'] tag_A = params['root_A'] tag_B = params['root_B'] print "cross correlating %s and %s..." %(tag_A, tag_B) # extent of correlation to calculate dimDegree = params['thetaExtent'] dimDegree *= np.pi/180. # loop over all map pairs for i in range(totalMapPairs): print 'correlating map pairs #%d...' %i # read in maps to cross correlate map_A = liteMap.liteMapFromFits(params['files_A'][i]) map_B = liteMap.liteMapFromFits(params['files_B'][i]) map_A.data = np.nan_to_num(map_A.data) map_B.data = np.nan_to_num(map_B.data) # set outliers to the median value inds = np.where(map_A.data != 0.0) dev = np.std(map_A.data[inds]) inds2 = np.where(abs(map_A.data) > 10.*dev) map_A.data[inds2] = np.median(map_A.data[inds]) inds = np.where(map_B.data != 0.0) dev = np.std(map_B.data[inds]) inds2 = np.where(abs(map_B.data) > 10.*dev) map_B.data[inds2] = np.median(map_B.data[inds]) # this map will store mapA_shifted*mapB values map_AB = map_A.copy() map_AB.data[:] = 0. # make map that will be use for shifting map_A_shifted = map_A.copy() map_A_shifted.data[:] = 0. # only do this first time around if i == 0: # num of pixels in this theta extent Ndim = np.int(dimDegree/map_A.pixScaleX) if np.mod(Ndim,2) != 0: Ndim += 1 # initialize correlation and theta matrices corr = np.zeros((Ndim+1,Ndim+1), dtype=float) theta = np.zeros((Ndim+1,Ndim+1), dtype=float) # this will be in arcminutes weight = np.zeros((Ndim+1,Ndim+1), dtype=float) # weight array for combining multiple maps # might not have this package try: bar = initializeProgressBar((Ndim+1.0)**2) except: pass # n shifts map in x-direction, m shifts map in y directions iter = 0 for m in xrange(-Ndim/2, Ndim/2+1, 1): for n in xrange(-Ndim/2, Ndim/2+1, 1): try: bar.update(iter + 1) except: pass iter += 1 # shift map A and then multiply shifted map A by map B map_A_shifted.data = MapShiftFunc(map_A.data, n, m) map_AB.data[:] = map_A_shifted.data[:]*map_B.data[:] inds = np.where(map_AB.data != 0.) w = 1.0*len(inds[0]) # number of nonzero values in mean # due weighted sum of this corr value and any past corr values corr[m+Ndim/2,n+Ndim/2] = corr[m+Ndim/2,n+Ndim/2]*weight[m+Ndim/2,n+Ndim/2] + w*(map_AB.data[inds]).mean() # update the nonzero elements at this array element weight[m+Ndim/2,n+Ndim/2] += w # divide by the total weight corr[m+Ndim/2,n+Ndim/2] /= weight[m+Ndim/2,n+Ndim/2] # store the theta value theta[m+Ndim/2,n+Ndim/2] = np.sqrt(n**2*map_A.pixScaleX**2+m**2*map_A.pixScaleY**2)*180.*60/np.pi # plot and save the 2D correlation figure arcmin = 180.*60/np.pi # make sure output directories exist if not os.path.exists('./output'): os.makedirs('./output') if params['makePlots']: # make sure figs directory exists if not os.path.exists('./output/figs'): os.makedirs('./output/figs') plt.imshow(corr,origin='down',cmap = cm.gray,extent=[-Ndim/2*map_A.pixScaleX*arcmin,(Ndim/2+1)*map_A.pixScaleX*arcmin,\ -Ndim/2*map_A.pixScaleY*arcmin,(Ndim/2+1)*map_A.pixScaleY*arcmin]) plt.colorbar() plt.savefig('./output/figs/corr_%s_%s.png'%(tag_A,tag_B)) # plot the 1D correlation and save fig, r, mean, std = plot1DCorr(None, data=(theta, corr)) fig.savefig('./output/figs/corr_1d_%s_%s.png'%(tag_A,tag_B)) # make sure data directory exists if not os.path.exists('./output/data'): os.makedirs('./output/data') # save the 2D correlation and theta as a pickle pickle.dump([theta, corr],open('./output/data/corr_%s_%s.pkl' %(tag_A,tag_B), 'w')) plt.close() # save the 2D correlation as a fits file hdu = pyfits.PrimaryHDU(corr) hdu.writeto('./output/data/corr_%s_%s.fits'%(tag_A,tag_B),clobber=True) return 0