def __init__(self, nprocs, njobs, progress=True, log=True):
"""
Initialize the input/output queues and make the workers
"""
# set up the results queue
self.results = Queue()
# set up the tasks queue
self.tasks = Queue()
# hold the dequeued results
self.deqd_results = []
self.nprocs = nprocs
self.log = log
if self.log:
# redirect stderr to a file
self.temp_stderr = tempfile.TemporaryFile()
sys.stderr = self.temp_stderr
# make a unique file name for std out
fileName, extension = os.path.splitext(os.path.basename(sys.argv[0]))
time_stamp = utilities.get_timestamp()
self.stdout = open(os.getcwd() + os.sep + "%s.%s.log" \
%(fileName, time_stamp), 'w')
sys.stdout = self.stdout
# set up the logger to log to sys.stderr
self.logger = multiprocessing.log_to_stderr()
self.logger.setLevel(logging.INFO)
# if we want a progress bar
if progress and progressLoaded:
self.bar = initializeProgressBar(njobs, fd=sys.__stderr__)
else:
self.bar = None
# create an exception event
self.exception = multiprocessing.Event()
def fourierSpaceCorrFromRandoms(paramsToUpdate, valsToUpdate, nProcs, corrType="cross"):
"""
@brief compute the real space correlation between a true data map
and random map, for the nMaps pairs of maps
@param paramsToUpdate: the parameters to change for each pair of maps
that we are correlating (dict)
@param valsToUpdate: the numbers to insert into the paramsToUpdate values (list)
@param nProcs: the number of processes to use (int)
@keyword corrType: type of spectrum to compute ('cross' or 'auto')
"""
num_jobs = len(valsToUpdate) # the number of correlations to do
# read the base parameter file
params = flipperDict.flipperDict()
try:
params.readFromFile("global.dict")
except:
raise
originalStdOut = sys.stdout
# might not have this package
try:
bar = initializeProgressBar(num_jobs)
except:
pass
def worker(job_queue, results_queue):
"""
@brief worker function for multiprocessing
"""
# pull tasks until there are none left
while True:
# dequeue the next job
next_task = job_queue.get()
# task=None means this worker is finished
if next_task is None:
# make sure we tell the queue we finished the task
job_queue.task_done()
break
else:
# tasks are tuples of params
oldParams, num = next_task
# try to update the progress bar
try:
bar.update(num + 1)
except:
pass
initialPath = os.getcwd()
# do the work
# make new directory and cd there
if not os.path.exists("tmp_%d" % num):
os.makedirs("tmp_%d" % num)
os.chdir("tmp_%d" % num)
# update the parameters
newParams = xcorrUtils.updateParams(paramsToUpdate, oldParams, valsToUpdate[num])
# hide output from the speck(Cross) output
sys.stdout = open(os.devnull, "w")
# compute the fourier space correlation, given the parameters
if corrType == "cross":
err = runSpeckCross(newParams)
if corrType == "auto":
err = runSpeck(newParams)
# go back to old directory and delete temporary directory
os.chdir(initialPath)
if os.path.exists("tmp_%d" % num):
os.system("rm -rf ./tmp_%d" % num)
# restore stdout to original value
sys.stdout = originalStdOut
# store the results
results_queue.put((num))
# make sure we tell the queue we finished the task
job_queue.task_done()
return 0
# store the system time
sys_time = time.time()
# establish communication queues that contain all jobs
job_numbers = mp.JoinableQueue()
results = mp.Queue()
# create a process for each cpu available or up to the limit specified by user
if nProcs <= mp.cpu_count():
num_workers = nProcs
else:
num_workers = mp.cpu_count()
print "Creating %d workers" % num_workers
procs = [mp.Process(target=worker, args=(job_numbers, results)) for i in xrange(num_workers)]
# start the processes
for proc in procs:
proc.start()
# enqueue the positions
for i in xrange(num_jobs):
job_numbers.put((params, i))
# Add a poison pill (=None) for each worker
for i in xrange(num_workers):
job_numbers.put(None)
# wait for all of the jobs to finish
job_numbers.join()
return 0
def realSpaceCorr(params):
"""
@brief compute the real space correlation between two maps
and output the relevant files
@param params: dictionary containing the following:
totalMapPairs: the total pairs of maps to compute correlation over (int)
root_A: the file tag for map #1 (str)
root_B: the file tag for map #2 (str)
thetaExtent: the angular extent to compute correlation to, in degrees (float)
files_A: array containing filenames of map A (can be more than 1 non overlapping regions)
files_B: array containing filenames of map B (can be more than 1 non overlapping regions)
makePlots: whether to plot the results (bool)
"""
totalMapPairs = params['totalMapPairs']
tag_A = params['root_A']
tag_B = params['root_B']
print "cross correlating %s and %s..." %(tag_A, tag_B)
# extent of correlation to calculate
dimDegree = params['thetaExtent']
dimDegree *= np.pi/180.
# loop over all map pairs
for i in range(totalMapPairs):
print 'correlating map pairs #%d...' %i
# read in maps to cross correlate
map_A = liteMap.liteMapFromFits(params['files_A'][i])
map_B = liteMap.liteMapFromFits(params['files_B'][i])
map_A.data = np.nan_to_num(map_A.data)
map_B.data = np.nan_to_num(map_B.data)
# set outliers to the median value
inds = np.where(map_A.data != 0.0)
dev = np.std(map_A.data[inds])
inds2 = np.where(abs(map_A.data) > 10.*dev)
map_A.data[inds2] = np.median(map_A.data[inds])
inds = np.where(map_B.data != 0.0)
dev = np.std(map_B.data[inds])
inds2 = np.where(abs(map_B.data) > 10.*dev)
map_B.data[inds2] = np.median(map_B.data[inds])
# this map will store mapA_shifted*mapB values
map_AB = map_A.copy()
map_AB.data[:] = 0.
# make map that will be use for shifting
map_A_shifted = map_A.copy()
map_A_shifted.data[:] = 0.
# only do this first time around
if i == 0:
# num of pixels in this theta extent
Ndim = np.int(dimDegree/map_A.pixScaleX)
if np.mod(Ndim,2) != 0:
Ndim += 1
# initialize correlation and theta matrices
corr = np.zeros((Ndim+1,Ndim+1), dtype=float)
theta = np.zeros((Ndim+1,Ndim+1), dtype=float) # this will be in arcminutes
weight = np.zeros((Ndim+1,Ndim+1), dtype=float) # weight array for combining multiple maps
# might not have this package
try:
bar = initializeProgressBar((Ndim+1.0)**2)
except:
pass
# n shifts map in x-direction, m shifts map in y directions
iter = 0
for m in xrange(-Ndim/2, Ndim/2+1, 1):
for n in xrange(-Ndim/2, Ndim/2+1, 1):
try:
bar.update(iter + 1)
except:
pass
iter += 1
# shift map A and then multiply shifted map A by map B
map_A_shifted.data = MapShiftFunc(map_A.data, n, m)
map_AB.data[:] = map_A_shifted.data[:]*map_B.data[:]
inds = np.where(map_AB.data != 0.)
w = 1.0*len(inds[0]) # number of nonzero values in mean
# due weighted sum of this corr value and any past corr values
corr[m+Ndim/2,n+Ndim/2] = corr[m+Ndim/2,n+Ndim/2]*weight[m+Ndim/2,n+Ndim/2] + w*(map_AB.data[inds]).mean()
# update the nonzero elements at this array element
weight[m+Ndim/2,n+Ndim/2] += w
# divide by the total weight
corr[m+Ndim/2,n+Ndim/2] /= weight[m+Ndim/2,n+Ndim/2]
# store the theta value
theta[m+Ndim/2,n+Ndim/2] = np.sqrt(n**2*map_A.pixScaleX**2+m**2*map_A.pixScaleY**2)*180.*60/np.pi
# plot and save the 2D correlation figure
arcmin = 180.*60/np.pi
# make sure output directories exist
if not os.path.exists('./output'):
os.makedirs('./output')
if params['makePlots']:
# make sure figs directory exists
if not os.path.exists('./output/figs'):
os.makedirs('./output/figs')
plt.imshow(corr,origin='down',cmap = cm.gray,extent=[-Ndim/2*map_A.pixScaleX*arcmin,(Ndim/2+1)*map_A.pixScaleX*arcmin,\
-Ndim/2*map_A.pixScaleY*arcmin,(Ndim/2+1)*map_A.pixScaleY*arcmin])
plt.colorbar()
plt.savefig('./output/figs/corr_%s_%s.png'%(tag_A,tag_B))
# plot the 1D correlation and save
fig, r, mean, std = plot1DCorr(None, data=(theta, corr))
fig.savefig('./output/figs/corr_1d_%s_%s.png'%(tag_A,tag_B))
# make sure data directory exists
if not os.path.exists('./output/data'):
os.makedirs('./output/data')
# save the 2D correlation and theta as a pickle
pickle.dump([theta, corr],open('./output/data/corr_%s_%s.pkl' %(tag_A,tag_B), 'w'))
plt.close()
# save the 2D correlation as a fits file
hdu = pyfits.PrimaryHDU(corr)
hdu.writeto('./output/data/corr_%s_%s.fits'%(tag_A,tag_B),clobber=True)
return 0
