run_sim2smv(prefix=file_prefix, crystal=crystal, spectra=iterator, rotation=rand_ori, quick=quick, rank=rank) if __name__ == "__main__": from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() import os, omptbx workaround_nt = int(os.environ.get("OMP_NUM_THREADS", 1)) omptbx.omp_set_num_threads(workaround_nt) N_total = 100000 # number of items to simulate N_stride = size # total number of worker tasks print("hello from rank %d of %d" % (rank, size), "with omp_threads=", omp_get_num_procs()) import datetime start_elapse = time() if rank == 0: print("Rank 0 time", datetime.datetime.now()) from LS49.spectra.generate_spectra import spectra_simulation from LS49.sim.step5_pad import microcrystal print("hello2 from rank %d of %d" % (rank, size)) SS = spectra_simulation() C = microcrystal( Deff_A=4000, length_um=4., beam_diameter_um=1.0) # assume smaller than 10 um crystals
def run_detail(show_plot, save_plot, use_dummy_data=False): file_name = sys.argv[1] from xfel.clustering.singleframe import CellOnlyFrame from cctbx import crystal cells = [] for line in open(file_name, "r").xreadlines(): tokens = line.strip().split() unit_cell = tuple(float(x) for x in tokens[0:6]) space_group_symbol = tokens[6] crystal_symmetry = crystal.symmetry( unit_cell=unit_cell, space_group_symbol=space_group_symbol) cells.append(CellOnlyFrame(crystal_symmetry, path=None)) MM = [c.mm for c in cells] # get all metrical matrices from scitbx.array_family import flex MM_double = flex.double() for i in range(len(MM)): Tup = MM[i] for j in range(6): MM_double.append(Tup[j]) print("There are %d cells" % (len(MM))) if show_plot or save_plot: import matplotlib if not show_plot: # http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear matplotlib.use('Agg') # use a non-interactive backend from matplotlib import pyplot as plt plt.figure(1) plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells], "k.", markersize=3.) plt.axes().set_aspect("equal") if save_plot: plt.savefig(plot_name, size_inches=(10, 10), dpi=300, bbox_inches='tight') if show_plot: plt.show() print("Now constructing a Dij matrix.") NN = len(MM) import omptbx omptbx.omp_set_num_threads(64) from cctbx.uctbx.determine_unit_cell import NCDist_flatten if use_dummy_data: ''' Generate blob data using sklearn. See example here. http://scikit-learn.org/stable/auto_examples/cluster/plot_cluster_comparison.html ''' try: from sklearn import datasets except ImportError: print( "Module sklearn not available. Needed to generate dummy data.") import numpy as np NN = 100 blobs = datasets.make_blobs(n_samples=NN, random_state=22) xx = [] yy = [] Dij = np.zeros([NN, NN]) Dij = flex.double(Dij) for x, y in blobs[0]: xx.append(x) yy.append(y) # Get Dij matrix for i in range(len(xx)): for j in range(len(xx)): dij2 = (xx[i] - xx[j]) * (xx[i] - xx[j]) + (yy[i] - yy[j]) * ( yy[i] - yy[j]) dij = np.sqrt(dij2) Dij[i * len(xx) + j] = dij if show_plot: import matplotlib.pyplot as plt #plt.figure() plt.scatter(xx, yy) plt.show() else: Dij = NCDist_flatten(MM_double) # loop is flattened plot_with_dimensional_embedding(Dij / flex.max(Dij), show_plot=show_plot)
def run_batch_job(test_without_mpi=False): params,options = parse_input() if params.log.by_rank: import io, sys if params.log.rank_profile: import cProfile pr = cProfile.Profile() pr.enable() # workaround for getting master nexus os.environ["NXMX_LOCAL_DATA"] = params.nxmx_local_data if test_without_mpi or params.test_without_mpi: from LS49.adse13_196.mock_mpi import mpiEmulator MPI = mpiEmulator() else: from libtbx.mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() import omptbx workaround_nt = int(os.environ.get("OMP_NUM_THREADS",1)) omptbx.omp_set_num_threads(workaround_nt) N_stride = size # total number of worker tasks print("hello from rank %d of %d"%(rank,size),"with omp_threads=",omp_get_num_procs()) import datetime start_comp = time() print(rank, time(), "finished with the calculation of channels, now construct single broadcast") if rank == 0: print("Rank 0 time", datetime.datetime.now()) spectrum_dict = {} from iotbx.reflection_file_reader import any_reflection_file from LS49 import ls49_big_data merge_file = os.path.join(ls49_big_data,"adse13_228","cyto_init_merge.mtz") Fmerge = any_reflection_file(merge_file).as_miller_arrays()[0].as_amplitude_array() print("Fmerge min/max = %f / %f" % (min(Fmerge.data()), max(Fmerge.data()))) transmitted_info = dict(spectra = spectrum_dict, amplitudes = Fmerge, ) else: transmitted_info = None transmitted_info = comm.bcast(transmitted_info, root = 0) comm.barrier() parcels = list(range(rank,params.N_total,N_stride)) print(rank, time(), "finished with single broadcast, now set up the rank logger") if params.log.by_rank: expand_dir = os.path.expandvars(params.log.outdir) os.makedirs(expand_dir, exist_ok=True) log_path = os.path.join(expand_dir,"rank_%d.log"%rank) error_path = os.path.join(expand_dir,"rank_%d.err"%rank) #print("Rank %d redirecting stdout/stderr to"%rank, log_path, error_path) sys.stdout = io.TextIOWrapper(open(log_path,'ab', 0), write_through=True) sys.stderr = io.TextIOWrapper(open(error_path,'ab', 0), write_through=True) print(rank, time(), "finished with the rank logger, now construct the GPU cache container") try: from simtbx.gpu import gpu_energy_channels gpu_channels_singleton = gpu_energy_channels ( deviceId = rank % params.devices_per_node ) # singleton will instantiate, regardless of cuda, device count, or exascale API except ImportError: gpu_channels_singleton = None comm.barrier() import random while len(parcels)>0: idx = random.choice(parcels) cache_time = time() print("idx------start-------->",idx,"rank",rank,time()) # if rank==0: os.system("nvidia-smi") tst_one(i_exp=idx,spectra=transmitted_info["spectra"], Fmerge=transmitted_info["amplitudes"], gpu_channels_singleton=gpu_channels_singleton, rank=rank,params=params ) parcels.remove(idx) print("idx------finis-------->",idx,"rank",rank,time(),"elapsed",time()-cache_time) comm.barrier() print("Overall rank",rank,"at",datetime.datetime.now(), "seconds elapsed after srun startup %.3f"%(time()-start_elapse)) print("Overall rank",rank,"at",datetime.datetime.now(), "seconds elapsed after Python imports %.3f"%(time()-start_comp)) if params.log.rank_profile: pr.disable() pr.dump_stats("cpu_%d.prof"%rank)
def run_LY99_batch(test_without_mpi=False): params, options = parse_input() log_by_rank = bool(int(os.environ.get("LOG_BY_RANK", 0))) rank_profile = bool(int(os.environ.get("RANK_PROFILE", 1))) if log_by_rank: import io, sys if rank_profile: import cProfile pr = cProfile.Profile() pr.enable() if test_without_mpi: from LS49.adse13_196.mock_mpi import mpiEmulator MPI = mpiEmulator() else: from libtbx.mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() import omptbx workaround_nt = int(os.environ.get("OMP_NUM_THREADS", 1)) omptbx.omp_set_num_threads(workaround_nt) N_total = int(os.environ["N_SIM"]) # number of items to simulate N_stride = size # total number of worker tasks print("hello from rank %d of %d" % (rank, size), "with omp_threads=", omp_get_num_procs()) import datetime start_comp = time() # now inside the Python imports, begin energy channel calculation wavelength_A = 1.74 # general ballpark X-ray wavelength in Angstroms wavlen = flex.double([12398.425 / (7070.5 + w) for w in range(100)]) direct_algo_res_limit = 1.7 local_data = data() # later put this through broadcast GF = gen_fmodel(resolution=direct_algo_res_limit, pdb_text=local_data.get("pdb_lines"), algorithm="fft", wavelength=wavelength_A) GF.set_k_sol(0.435) GF.make_P1_primitive() # Generating sf for my wavelengths sfall_channels = {} for x in range(len(wavlen)): if rank > len(wavlen): break if x % size != rank: continue GF.reset_wavelength(wavlen[x]) GF.reset_specific_at_wavelength( label_has="FE1", tables=local_data.get("Fe_oxidized_model"), newvalue=wavlen[x]) GF.reset_specific_at_wavelength( label_has="FE2", tables=local_data.get("Fe_reduced_model"), newvalue=wavlen[x]) sfall_channels[x] = GF.get_amplitudes() reports = comm.gather(sfall_channels, root=0) if rank == 0: sfall_channels = {} for report in reports: sfall_channels.update(report) comm.barrier() print( rank, time(), "finished with the calculation of channels, now construct single broadcast" ) if rank == 0: print("Rank 0 time", datetime.datetime.now()) from LS49.spectra.generate_spectra import spectra_simulation from LS49.adse13_196.revapi.LY99_pad import microcrystal print("hello2 from rank %d of %d" % (rank, size)) SS = spectra_simulation() C = microcrystal( Deff_A=4000, length_um=4., beam_diameter_um=1.0) # assume smaller than 10 um crystals from LS49 import legacy_random_orientations random_orientations = legacy_random_orientations(N_total) transmitted_info = dict(spectra=SS, crystal=C, sfall_info=sfall_channels, random_orientations=random_orientations) else: transmitted_info = None transmitted_info = comm.bcast(transmitted_info, root=0) comm.barrier() parcels = list(range(rank, N_total, N_stride)) print(rank, time(), "finished with single broadcast, now set up the rank logger") if log_by_rank: expand_dir = os.path.expandvars(params.logger.outdir) log_path = os.path.join(expand_dir, "rank_%d.log" % rank) error_path = os.path.join(expand_dir, "rank_%d.err" % rank) #print("Rank %d redirecting stdout/stderr to"%rank, log_path, error_path) sys.stdout = io.TextIOWrapper(open(log_path, 'ab', 0), write_through=True) sys.stderr = io.TextIOWrapper(open(error_path, 'ab', 0), write_through=True) print( rank, time(), "finished with the rank logger, now construct the GPU cache container") import random gpu_instance = get_exascale("gpu_instance", params.context) gpu_energy_channels = get_exascale("gpu_energy_channels", params.context) gpu_run = gpu_instance(deviceId=rank % int(os.environ.get("DEVICES_PER_NODE", 1))) gpu_channels_singleton = gpu_energy_channels( deviceId=gpu_run.get_deviceID()) # singleton will instantiate, regardless of gpu, device count, or exascale API comm.barrier() while len(parcels) > 0: idx = random.choice(parcels) cache_time = time() print("idx------start-------->", idx, "rank", rank, time()) # if rank==0: os.system("nvidia-smi") tst_one( image=idx, spectra=transmitted_info["spectra"], crystal=transmitted_info["crystal"], random_orientation=transmitted_info["random_orientations"][idx], sfall_channels=transmitted_info["sfall_info"], gpu_channels_singleton=gpu_channels_singleton, rank=rank, params=params) parcels.remove(idx) print("idx------finis-------->", idx, "rank", rank, time(), "elapsed", time() - cache_time) comm.barrier() del gpu_channels_singleton # avoid Kokkos allocation "device_Fhkl" being deallocated after Kokkos::finalize was called print("Overall rank", rank, "at", datetime.datetime.now(), "seconds elapsed after srun startup %.3f" % (time() - start_elapse)) print("Overall rank", rank, "at", datetime.datetime.now(), "seconds elapsed after Python imports %.3f" % (time() - start_comp)) if rank_profile: pr.disable() pr.dump_stats("cpu_%d.prof" % rank)
def find_peaks_clean(self): import omptbx # doesn't seem to be any benefit to using more than say 8 threads num_threads = min(8, omptbx.omp_get_num_procs(), self.params.nproc) omptbx.omp_set_num_threads(num_threads) d_min = self.params.fft3d.reciprocal_space_grid.d_min rlgrid = 2 / (d_min * self.gridding[0]) frame_number = self.reflections['xyzobs.px.value'].parts()[2] scan_range_min = max( int(math.floor(flex.min(frame_number))), self.imagesets[0].get_array_range()[0]) # XXX what about multiple imagesets? scan_range_max = min( int(math.ceil(flex.max(frame_number))), self.imagesets[0].get_array_range()[1]) # XXX what about multiple imagesets? scan_range = self.params.scan_range if not len(scan_range): scan_range = [[scan_range_min, scan_range_max]] scan = self.imagesets[0].get_scan() # XXX what about multiple imagesets? angle_ranges = [ [scan.get_angle_from_array_index(i, deg=False) for i in range_] for range_ in scan_range] grid = flex.double(flex.grid(self.gridding), 0) sampling_volume_map(grid, flex.vec2_double(angle_ranges), self.imagesets[0].get_beam().get_s0(), self.imagesets[0].get_goniometer().get_rotation_axis(), rlgrid, d_min, self.params.b_iso) fft = fftpack.complex_to_complex_3d(self.gridding) grid_complex = flex.complex_double( reals=grid, imags=flex.double(grid.size(), 0)) grid_transformed = fft.forward(grid_complex) grid_real = flex.pow2(flex.real(grid_transformed)) gamma = 1 peaks = flex.vec3_double() #n_peaks = 200 n_peaks = 100 # XXX how many do we need? dirty_beam = grid_real dirty_map = self.grid_real.deep_copy() import time t0 = time.time() peaks = clean_3d(dirty_beam, dirty_map, n_peaks, gamma=gamma) t1 = time.time() #print "clean_3d took %.2f s" %(t1-t0) reciprocal_lattice_points = self.reflections['rlp'].select( self.reflections_used_for_indexing) peaks = self.optimise_peaks(peaks, reciprocal_lattice_points) peaks_frac = flex.vec3_double() for p in peaks: peaks_frac.append((p[0]/self.gridding[0], p[1]/self.gridding[1], p[2]/self.gridding[2])) #print p, peaks_frac[-1] if self.params.debug: self.debug_write_ccp4_map(grid, "sampling_volume.map") self.debug_write_ccp4_map(grid_real, "sampling_volume_FFT.map") self.debug_write_ccp4_map(dirty_map, "clean.map") self.sites = peaks_frac # we don't really know the "volume"" of the peaks, but this method should # find the peaks in order of their intensity (strongest first) self.volumes = flex.double(range(len(self.sites), 0, -1)) return
def run(self): self.parse_input() N_total = 100000 # self.params.N_total # number of items to simulate, nominally 100000 logical_rank = self.mpi_helper.rank logical_size = self.mpi_helper.size if self.mpi_helper.rank == 0 and self.mpi_helper.size == 1: # special case of testing it try: logical_rank = self.params.tester.rank logical_size = self.params.tester.size except Exception: pass N_stride = int(math.ceil( N_total / logical_size)) # total number of tasks per rank print("hello from rank %d of %d with stride %d" % (logical_rank, logical_size, N_stride)) #from scitbx.lbfgs.tst_mpi_split_evaluator import mpi_split_evaluator_run #from scitbx.lbfgs.tst_mpi_split_evaluator import run_mpi as simple_tester #simple_tester() if self.params.starting_model.algorithm == "to_file": if self.mpi_helper.rank == 0: HKL_lookup, static_fcalcs = get_static_fcalcs_with_HKL_lookup() from LS49.work2_for_aca_lsq.remake_range_intensities_with_complex \ import get_intensity_structure model_intensities = get_intensity_structure( static_fcalcs, FE1_model=Fe_oxidized_model, FE2_model=Fe_reduced_model) with (open(self.params.starting_model.filename, "wb")) as out: pickle.dump(HKL_lookup, out, pickle.HIGHEST_PROTOCOL) pickle.dump(static_fcalcs, out, pickle.HIGHEST_PROTOCOL) pickle.dump(model_intensities, out, pickle.HIGHEST_PROTOCOL) return else: if self.mpi_helper.rank == 0: with (open(self.params.starting_model.filename, "rb")) as inp: print("the starting model (used for channel weighting) is", self.params.starting_model.filename) HKL_lookup = pickle.load(inp) static_fcalcs = pickle.load(inp) model_intensities = pickle.load(inp) from LS49.spectra.generate_spectra import spectra_simulation from LS49.sim.step5_pad import microcrystal shuffA = list(range(N_total)) import random random.shuffle(shuffA) transmitted_info = dict( HKL_lookup=HKL_lookup, static_fcalcs=static_fcalcs, model_intensities=model_intensities, spectra_simulation=spectra_simulation(), crystal=microcrystal(Deff_A=4000, length_um=4., beam_diameter_um=1.0), shuffA=shuffA) else: transmitted_info = None print("before braodcast with ", self.mpi_helper.rank, self.mpi_helper.size) transmitted_info = self.mpi_helper.comm.bcast(transmitted_info, root=0) self.mpi_helper.comm.barrier() print("after barrier") # ----------------------------------------------------------------------- if self.mpi_helper.rank == 0: print("Finding initial G and abc factors") per_rank_items = [] per_rank_keys = [] per_rank_G = [] min_spots = 3 N_input = 0 import os, omptbx # cori workaround, which does not get OMP_NUM_THREADS from environment workaround_nt = int(os.environ.get("OMP_NUM_THREADS", 1)) omptbx.omp_set_num_threads(workaround_nt) for item, key in get_items(logical_rank, N_total, N_stride, transmitted_info["shuffA"], self.params.cohort): N_input += 1 if len(item) >= min_spots: try: FOI = fit_one_image_multispot( key=key, list_of_images=item, HKL_lookup=transmitted_info["HKL_lookup"], model_intensities=transmitted_info[ "model_intensities"], spectra=transmitted_info["spectra_simulation"], crystal=transmitted_info["crystal"]) except RuntimeError as e: # no recovery from LBFGS error, skip event continue metric_P1, metric_C2 = FOI.DRM.get_current_angular_offsets( FOI.x[-4:-1]) print( """LLG Image %06d on %d Bragg spots NLL channels F = %9.1f angular offsets in P1 and C2 (degrees): %8.5f %8.5f""" % (key, len(item), FOI.compute_functional_and_gradients()[0], metric_P1, metric_C2), FOI.x[-4:-1]) # reporting out results to new abc_coverage pickles. Use the old one "item" as a template: # item := [<LS49.work2_for_aca_lsq.abc_background.fit_roi_multichannel>,... one for each spot] # each fit_roi has the following attributes and we modify them as follows: # 'a', the abcG parameters of the original one-spot fit, unused # 'asu_idx_C2_setting', unmodified # 'image_no', same as key, unmodified # 'n', number of parameters for one-spot fit, unused # 'orig_idx_C2_setting', unmodified # 'sb_data', original shoebox data [integers as floats], passed along unmodified # 'simtbx_P1_miller', unmodified # 'simtbx_intensity_7122', unmodified # 'x', the abcG parameters of the original one-spot fit, unused # modify these: # 'bkgrd_a', the spot abc parameters output here, to be passed on to global data fit for ispot in range(len(item)): item[ispot].bkgrd_a = FOI.x[3 * ispot:3 * (ispot + 1)] # 'channels', need to save the new data that was set during update_roi_model_pixels_with_current_rotation() # but only for the Amat, not the derivatives. item[ispot].channels = FOI.new_calc2_dict_last_round[ ispot]["channels"] # 'roi', get new region of interest summation that was set during update_roi_model_pixels_with_current_rotation() item[ispot].roi = FOI.roi_model_pixels[ispot] # put the newly refined background model back into the item per_rank_keys.append(key) per_rank_G.append(FOI.a[-1]) print( "pickling modified abc_coverage file for key %d in rank %d" % (key, logical_rank), ) with open(abc_glob_pixel_ref % (key), "wb") as F: pickle.dump(item, F, pickle.HIGHEST_PROTOCOL) print("rank %d has %d refined images" % (logical_rank, len(per_rank_keys))) N_ranks = self.mpi_helper.comm.reduce(1, self.mpi_helper.MPI.SUM, 0) N_refined_images = self.mpi_helper.comm.reduce(len(per_rank_keys), self.mpi_helper.MPI.SUM, 0) N_input_images = self.mpi_helper.comm.reduce(N_input, self.mpi_helper.MPI.SUM, 0) self.mpi_helper.comm.barrier() if self.mpi_helper.rank == 0: print("final report %d ranks, %d input images, %d refined models" % (N_ranks, N_input_images, N_refined_images)) print("Finished finding initial G and abc factors") self.mpi_helper.comm.barrier()
def run_batch_job(test_without_mpi=False): from LS49.adse13_187.cyto_batch import parse_input as cyto_batch_parse_input params,options = cyto_batch_parse_input() if params.log.by_rank: import io, sys if params.log.rank_profile: import cProfile pr = cProfile.Profile() pr.enable() if test_without_mpi or params.test_without_mpi: from LS49.adse13_196.mock_mpi import mpiEmulator MPI = mpiEmulator() else: from libtbx.mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() import omptbx workaround_nt = int(os.environ.get("OMP_NUM_THREADS",1)) omptbx.omp_set_num_threads(workaround_nt) N_stride = size # total number of worker tasks print("hello from rank %d of %d"%(rank,size),"with omp_threads=",omp_get_num_procs()) import datetime start_comp = time() if params.log.by_rank: expand_dir = os.path.expandvars(params.log.outdir) os.makedirs(expand_dir, exist_ok=True) log_path = os.path.join(expand_dir,"rank_%d.log"%rank) error_path = os.path.join(expand_dir,"rank_%d.err"%rank) #print("Rank %d redirecting stdout/stderr to"%rank, log_path, error_path) sys.stdout = io.TextIOWrapper(open(log_path,'ab', 0), write_through=True) sys.stderr = io.TextIOWrapper(open(error_path,'ab', 0), write_through=True) print(rank, time(), "finished with the rank logger, now delgate parcels") os.environ["CCTBX_RECOMMEND_DEVICE"] = "%d"%(rank % int(os.environ.get("CCTBX_DEVICE_PER_NODE",1))) print("rank", rank, "device", os.environ["CCTBX_RECOMMEND_DEVICE"]) N_start = int(os.environ.get("N_START",0)) comm.barrier() if rank == 0: os.system("nvidia-smi") # client process (requests all the work) import random parcels = list(range(N_start,N_start + params.N_total)) while len(parcels) > 0: idx = parcels[0] # random.choice(parcels) rankreq = comm.recv(source = MPI.ANY_SOURCE) print("Sending parcel",idx,"to rank",rankreq) comm.send(idx,dest = rankreq) parcels.remove(idx) # finally send a stop command to each process for rankreq in range(size-1): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send('endrun',dest=rankreq) else: # server process (does all the work) while True: # inform the client this worker is ready for an event comm.send(rank,dest=0) idx = comm.recv(source=0) if idx == 'endrun': break cache_time = time() print("idx------start-------->",idx,"rank",rank,time()) thin_ds1(idx,frame_params=params) print("idx------finis-------->",idx, "rank",rank,time(),"elapsed %.3fs"%(time()-cache_time)) comm.barrier() print("Overall rank",rank,"at",datetime.datetime.now(), "seconds elapsed after srun startup %.3f"%(time()-start_elapse)) print("Overall rank",rank,"at",datetime.datetime.now(), "seconds elapsed after Python imports %.3f"%(time()-start_comp)) if params.log.rank_profile: pr.disable() pr.dump_stats("cpu_%d.prof"%rank)
def run_detail(show_plot, save_plot): P = Profiler("0. Read data") import sys file_name = sys.argv[1] from xfel.clustering.singleframe import CellOnlyFrame from cctbx import crystal cells = [] for line in open(file_name, "r"): tokens = line.strip().split() unit_cell = tuple(float(x) for x in tokens[0:6]) space_group_symbol = tokens[6] crystal_symmetry = crystal.symmetry( unit_cell=unit_cell, space_group_symbol=space_group_symbol) cells.append(CellOnlyFrame(crystal_symmetry, path=None)) MM = [c.mm for c in cells] # get all metrical matrices MM_double = flex.double() for i in range(len(MM)): Tup = MM[i] for j in range(6): MM_double.append(Tup[j]) print(("There are %d cells" % (len(MM)))) coord_x = flex.double([c.uc[0] for c in cells]) coord_y = flex.double([c.uc[1] for c in cells]) if show_plot or save_plot: import matplotlib if not show_plot: # http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear matplotlib.use('Agg') # use a non-interactive backend from matplotlib import pyplot as plt plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells], "k.", markersize=3.) plt.axes().set_aspect("equal") if save_plot: plt.savefig(plot_name, size_inches=(10, 10), dpi=300, bbox_inches='tight') if show_plot: plt.show() print("Now constructing a Dij matrix.") P = Profiler("1. compute Dij matrix") NN = len(MM) import omptbx omptbx.omp_set_num_threads(64) from cctbx.uctbx.determine_unit_cell import NCDist_matrix, NCDist_flatten #Dij = NCDist_matrix(MM_double) # double loop, less efficient evaluation of upper triangle Dij = NCDist_flatten(MM_double) # loop is flattened #from cctbx.uctbx.determine_unit_cell import NCDist # can this be refactored with MPI? #Dij = flex.double(flex.grid(NN,NN)) #for i in xrange(NN): # for j in xrange(i+1,NN): # Dij[i,j] = NCDist(MM[i], MM[j]) del P d_c = 10 # the distance cutoff, such that average item neighbors 1-2% of all items CM = clustering_manager(Dij=Dij, d_c=d_c) # Summarize the results here n_cluster = 1 + flex.max(CM.cluster_id_final) print(len(cells), "have been analyzed") print(("# ------------ %d CLUSTERS ----------------" % (n_cluster))) for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) print("Cluster %d. Central unit cell: item %d" % (i, item)) cells[item].crystal_symmetry.show_summary() print("Cluster has %d items, or %d after trimming borders" % ((CM.cluster_id_full == i).count(True), (CM.cluster_id_final == i).count(True))) print() appcolors = [ 'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf' ] if show_plot: #Decision graph from matplotlib import pyplot as plt plt.plot(CM.rho, CM.delta, "r.", markersize=3.) for x in range(NN): if CM.cluster_id_maxima[x] >= 0: plt.plot([CM.rho[x]], [CM.delta[x]], "ro") plt.show() #No-halo plot from matplotlib import pyplot as plt colors = [appcolors[i % 10] for i in CM.cluster_id_full] plt.scatter(coord_x, coord_y, marker='o', color=colors, linewidths=0.4, edgecolor='k') for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.') plt.axes().set_aspect("equal") plt.show() #Final plot halo = (CM.cluster_id_final == -1) core = ~halo plt.plot(coord_x.select(halo), coord_y.select(halo), "k.") colors = [appcolors[i % 10] for i in CM.cluster_id_final.select(core)] plt.scatter(coord_x.select(core), coord_y.select(core), marker="o", color=colors, linewidths=0.4, edgecolor='k') for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.') plt.axes().set_aspect("equal") plt.show()