def run_mpi(): print "Rank :", rank, "started" npix = hp.nside2npix(config.nside_out) dim, ind_elements = cov_ut.get_dim(config.pol_type) bolo_list = [None, None, None] inv_cov_matrix_local = np.zeros((npix, ind_elements), dtype=np.float) b_matrix_local = np.zeros((npix, dim), dtype=np.float) hitmap_local = np.zeros(npix, dtype=np.float) bolo_segment_dict = get_local_bolo_segment_list(rank, size, config.bolo_list, config.segment_list) time.sleep(0.1*rank) print "Rank :", rank, ", Bolos and Segments :", bolo_segment_dict comm.Barrier() recon_dir = get_recon_dir() if rank == 0: make_data_dirs() for bolo_name in bolo_segment_dict.keys(): print "Rank :", rank, "Bolos class being generated" bolo_list = initialise_bolo(bolo_name, bolo_list) for segment in bolo_segment_dict[bolo_name]: prompter.prompt("Rank : %d doing Bolo : %s and segment : %d" % (rank, bolo_name, segment)) if config.take_diff_signal: signal, v, pol_ang = acquire_difference_signal(bolo_a, bolo_b, segment) else: signal, v, pol_ang = acquire_signal(bolo, segment) if config.subtract_template: signal_TEMPLATE = bolo_TEMPLATE.read_timestream(segment, read_list=["signal"])["signal"] signal -= estimated_y*signal_TEMPLATE print "Rank :", rank, "Bolos signal read" hitpix = hp.vec2pix(config.nside_out, v[...,0], v[...,1], v[...,2]) del v cov_ut.get_inv_cov_matrix(hitpix, pol_ang, signal, inv_cov_matrix_local, b_matrix_local, hitmap_local, npix, config.pol_type) print "Rank :", rank, "Inverse covariance matrix generated" if config.subtract_template: del signal_TEMPLATE del signal del pol_ang del hitpix inv_cov_matrix_local_segment = distribute_matrix(inv_cov_matrix_local, "cov_matrix") del inv_cov_matrix_local b_matrix_local_segment = distribute_matrix(b_matrix_local, "b_matrix") del b_matrix_local hitmap_local_segment = distribute_matrix(hitmap_local, "hitmap") del hitmap_local cov_matrix_local_segment = cov_ut.get_covariance_matrix(inv_cov_matrix_local_segment, hitmap_local_segment, config.pol_type) sky_map_local_segment = cov_ut.get_sky_map(cov_matrix_local_segment, b_matrix_local_segment, hitmap_local_segment, config.pol_type) write_segments(hitmap_local_segment, "hitmap", recon_dir) write_segments(inv_cov_matrix_local_segment, "inverse_covariance_matrix", recon_dir) write_segments(cov_matrix_local_segment, "covariance_matrix", recon_dir) write_segments(sky_map_local_segment, "sky_map", recon_dir)
def distribute_matrix(local_full_matrix, matrix_type): npix = hp.nside2npix(config.nside_out) dim, ind_elements = cov_ut.get_dim(config.pol_type) segment_length = npix/size if matrix_type == "hitmap": local_segmented_matrix = np.zeros(segment_length) else: inner_dim = {"cov_matrix" : ind_elements, "b_matrix" : dim} local_segmented_matrix = np.zeros((segment_length, inner_dim[matrix_type])) for i in range(size): start = i*segment_length stop = (i+1)*segment_length comm.Reduce(local_full_matrix[start:stop], local_segmented_matrix, MPI.SUM, root=i) return local_segmented_matrix
def accumulate_segments(size): recon_dir = get_recon_dir() map_segment_dir, hitmap_segment_dir, cov_matrix_segment_dir, inv_cov_matrix_segment_dir = get_segment_dirs() npix = hp.nside2npix(config.nside_out) npix_segment = npix/size dim, ind_elements = cov_ut.get_dim(config.pol_type) sky_map = np.empty((dim, npix)) for i in range(size): start = i*npix_segment stop = (i+1)*npix_segment print os.path.join(recon_dir, map_segment_dir, str(i).zfill(4) + '.npy'), os.path.exists(os.path.join(recon_dir, map_segment_dir, str(i).zfill(4) + '.npy')) sky_map_segment = np.load(os.path.join(recon_dir, map_segment_dir, str(i).zfill(4) + '.npy')) sky_map[..., start:stop] = sky_map_segment hp.write_map(os.path.join(recon_dir, "sky_map.fits"), sky_map) del sky_map del sky_map_segment shutil.rmtree(os.path.join(recon_dir, map_segment_dir)) hitmap = np.empty(npix) for i in range(size): start = i*npix_segment stop = (i+1)*npix_segment hitmap_segment = np.load(os.path.join(recon_dir, hitmap_segment_dir, str(i).zfill(4) + '.npy')) hitmap[..., start:stop] = hitmap_segment hp.write_map(os.path.join(recon_dir, "hitmap.fits"), hitmap) del hitmap del hitmap_segment shutil.rmtree(os.path.join(recon_dir, hitmap_segment_dir)) inverse_cov_matrix = np.empty((npix, ind_elements)) for i in range(size): start = i*npix_segment stop = (i+1)*npix_segment inverse_cov_matrix_segment = np.load(os.path.join(recon_dir, inv_cov_matrix_segment_dir, str(i).zfill(4) + '.npy')) inverse_cov_matrix[start:stop] = inverse_cov_matrix_segment hp.write_map(os.path.join(recon_dir, "inverse_covariance_maps.fits"), inverse_cov_matrix.T) del inverse_cov_matrix del inverse_cov_matrix_segment shutil.rmtree(os.path.join(recon_dir, inv_cov_matrix_segment_dir)) cov_matrix = np.empty((npix, ind_elements)) for i in range(size): start = i*npix_segment stop = (i+1)*npix_segment cov_matrix_segment = np.load(os.path.join(recon_dir, cov_matrix_segment_dir, str(i).zfill(4) + '.npy')) if config.pol_type == "TQU": cov_matrix_segment = cov_matrix_segment.reshape((npix_segment, dim**2))[..., np.array([0,1,2,4,5,8])] elif config.pol_type =="QU": cov_matrix_segment = cov_matrix_segment.reshape((npix_segment, dim**2))[..., np.array([0,1,3])] else: cov_matrix_segment = cov_matrix_segment.reshape((npix_segment, dim**2)) cov_matrix[start:stop] = cov_matrix_segment hp.write_map(os.path.join(recon_dir, "covariance_maps.fits"), cov_matrix.T) del cov_matrix del cov_matrix_segment shutil.rmtree(os.path.join(recon_dir, cov_matrix_segment_dir))