def get_em2d_restraint(assembly, images_selection_file, restraint_params, mode="fast", n_optimized=1): """ Sets a restraint for comparing the model to a set of EM images """ model = assembly.get_model() # Setup the restraint sc = em2d.EM2DScore() r = em2d.Em2DRestraint(model) r.setup(sc, restraint_params) names = em2d.read_selection_file(images_selection_file) names = [base.get_relative_path(images_selection_file, x) for x in names] log.debug("names of the images %s", names) srw = em2d.SpiderImageReaderWriter() imgs = em2d.read_images(names, srw) r.set_images(imgs) ps = atom.get_leaves(assembly) lsc = container.ListSingletonContainer(ps) r.set_particles(lsc) if (mode == "coarse"): r.set_coarse_registration_mode(True) elif (mode == "fast"): r.set_fast_mode(n_optimized) elif (mode == "complete"): pass else: raise ValueError("Em2DRestraint mode not recognized") return r
def get_em2d_restraint( assembly, images_selection_file, restraint_params, mode="fast", n_optimized=1): """ Sets a restraint for comparing the model to a set of EM images """ model = assembly.get_model() # Setup the restraint sc = em2d.EM2DScore() r = em2d.Em2DRestraint() r.setup(sc, restraint_params) names = em2d.read_selection_file(images_selection_file) names = [base.get_relative_path(images_selection_file, x) for x in names] log.debug("names of the images %s", names) srw = em2d.SpiderImageReaderWriter() imgs = em2d.read_images(names, srw) r.set_images(imgs) ps = atom.get_leaves(assembly) lsc = container.ListSingletonContainer(ps) r.set_particles(lsc) if (mode == "coarse"): r.set_coarse_registration_mode(True) elif (mode == "fast"): r.set_fast_mode(n_optimized) elif(mode == "complete"): pass else: raise ValueError("Em2DRestraint mode not recognized") return r
def test_building_an_optimization_problem_with_em2d_restraint(self): """Test that an a optimization with em2d restraint is properly built""" m = IMP.kernel.Model() prot = IMP.atom.read_pdb(self.get_input_file_name("1z5s.pdb"), m, IMP.atom.ATOMPDBSelector()) # get the chains chains = IMP.atom.get_by_type(prot, IMP.atom.CHAIN_TYPE) # set the chains as rigid bodies rigid_bodies = [] native_chain_centers = [] for c in chains: atoms = IMP.core.get_leaves(c) rbd = IMP.core.RigidBody.setup_particle(c, atoms) rbd.set_coordinates_are_optimized(True) rigid_bodies.append(rbd) native_chain_centers.append(rbd.get_coordinates()) self.assertEqual( len(rigid_bodies), 4, "Problem generating rigid bodies") bb = IMP.algebra.BoundingBox3D(IMP.algebra.Vector3D(-25, -40, -60), IMP.algebra.Vector3D(25, 40, 60)) # set distance restraints d01 = IMP.algebra.get_distance(native_chain_centers[0], native_chain_centers[1]) r01 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d01, 1), chains[0], chains[1]) r01.set_name("distance 0-1") d12 = IMP.algebra.get_distance(native_chain_centers[1], native_chain_centers[2]) r12 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d12, 1), chains[1], chains[2]) r12.set_name("distance 1-2") d23 = IMP.algebra.get_distance(native_chain_centers[2], native_chain_centers[3]) r23 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d23, 1), chains[2], chains[3]) r23.set_name("distance 2-3") d30 = IMP.algebra.get_distance(native_chain_centers[3], native_chain_centers[0]) r30 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d30, 1), chains[3], chains[0]) r30.set_name("distance 3-0") # set distance restraints for r in [r01, r12, r23, r30]: m.add_restraint(r) self.assertEqual(m.get_number_of_restraints(), 4, "Incorrect number of distance restraints") # set em2D restraint srw = em2d.SpiderImageReaderWriter() selection_file = self.get_input_file_name("all-1z5s-projections.sel") images_to_read_names = [IMP.base.get_relative_path(selection_file, x) for x in em2d.read_selection_file(selection_file)] em_images = em2d.read_images(images_to_read_names, srw) self.assertEqual(len(em_images), 3, "Incorrect number images read") apix = 1.5 resolution = 1 n_projections = 20 params = em2d.Em2DRestraintParameters(apix, resolution, n_projections) params.save_match_images = False params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING score_function = em2d.EM2DScore() em2d_restraint = em2d.Em2DRestraint(m) em2d_restraint.setup(score_function, params) em2d_restraint.set_images(em_images) em2d_restraint.set_name("em2d restraint") container = IMP.container.ListSingletonContainer( IMP.core.get_leaves(prot)) em2d_restraint.set_particles(container) em2d_restraints_set = IMP.kernel.RestraintSet(m) em2d_restraints_set.add_restraint(em2d_restraint) em2d_restraints_set.set_weight(1000) # weight for the em2D restraint m.add_restraint(em2d_restraints_set) self.assertEqual(m.get_number_of_restraints(), 5, "Incorrect number of restraints") # MONTECARLO OPTIMIZATION s = IMP.core.MonteCarlo(m) # Add movers for the rigid bodies movers = [] for rbd in rigid_bodies: movers.append(IMP.core.RigidBodyMover(rbd, 5, 2)) s.add_movers(movers) self.assertEqual(s.get_number_of_movers(), 4, "Incorrect number of MonteCarlo movers") # Optimizer state to save intermediate configurations o_state = IMP.atom.WritePDBOptimizerState(chains, "intermediate-step-%1%.pdb") o_state.set_period(11) s.add_optimizer_state(o_state) ostate2 = WriteStatisticsOptimizerScore(m) s.add_optimizer_state(ostate2) # Perform optimization temperatures = [200, 100, 60, 40, 20, 5] optimization_steps = 200 # for temp in temperatures: # s.optimize(optimization_steps) # IMP.atom.write_pdb(prot,"solution.pdb") self.assertTrue(True)
import IMP import IMP.em2d as em2d import os """ Conversion of Electron Microscopy Images. """ # Read images fn_selection=em2d.get_example_path("all-1z5s-projections.sel") srw = em2d.SpiderImageReaderWriter() trw = em2d.TIFFImageReaderWriter() fn_images=em2d.read_selection_file(fn_selection) fn_images = [em2d.get_example_path(x) for x in fn_images] images = em2d.read_images(fn_images,srw) # write fn_saved=em2d.create_filenames(3,"1z5s-projection","tif") em2d.save_images(images,fn_saved,trw)
def test_registration(self): """Test the registration of 3 subjects from 1gyt.pdb at 0.5 SNR""" # Get model from PDB file smodel = IMP.Model() ssel = IMP.atom.ATOMPDBSelector() fn_model = self.get_input_file_name("1gyt.pdb") prot = IMP.atom.read_pdb(fn_model, smodel, ssel) particles = IMP.core.get_leaves(prot) # Read subject images srw = em2d.SpiderImageReaderWriter() selection_file = self.get_input_file_name("1gyt-subjects-0.5-SNR.sel") images_to_read_names = em2d.read_selection_file(selection_file) for i in range(0, len(images_to_read_names)): images_to_read_names[i] = self.get_input_file_name( images_to_read_names[i]) subjects = em2d.read_images(images_to_read_names, srw) self.assertEqual(len(subjects), 3, "Problem reading subject images") # Generate 20 evenly distributed projections from the PDB file n_projections = 20 proj_params = em2d.get_evenly_distributed_registration_results( n_projections) rows = 128 cols = 128 pixel_size = 1.5 # for generating projections, use a very high resolution resolution = 8.5 options = em2d.ProjectingOptions(pixel_size, resolution) projections = em2d.get_projections(particles, proj_params, rows, cols, options) self.assertEqual(len(projections), n_projections, "Problem generating projections") # Prepare registration # IMP.set_log_level(IMP.VERBOSE) finder = em2d.ProjectionFinder() score_function = em2d.EM2DScore() params = em2d.Em2DRestraintParameters(pixel_size, resolution, n_projections) params.save_match_images = False params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING params.optimization_steps = 30 params.simplex_initial_length = 0.1 params.simplex_minimum_size = 0.01 finder.setup(score_function, params) finder.set_model_particles(particles) finder.set_subjects(subjects) finder.set_projections(projections) finder.set_fast_mode(2) finder.get_complete_registration() # Recover the registration results: registration_parameters = finder.get_registration_results() fn_registration_results = "my_1gyt_registration.params" em2d.write_registration_results(fn_registration_results, registration_parameters) # Read the correct registration results: correct_parameters = em2d.read_registration_results( self.get_input_file_name("1gyt-subjects-0.5-SNR.params")) print("determined: ") for r in registration_parameters: print(r.get_rotation(), r.get_shift()) print("correct: ") for r in correct_parameters: print(r.get_rotation(), r.get_shift()) for i in range(0, len(registration_parameters)): # Generate the registered projection imgx = em2d.Image() imgx.set_size(rows, cols) em2d.get_projection(imgx, particles, registration_parameters[i], options) ccc = em2d.get_cross_correlation_coefficient( subjects[i].get_data(), imgx.get_data()) print(i, "ccc", ccc) snr = 0.5 theoretical_ccc = (snr / (1. + snr))**.5 self.assertAlmostEqual( ccc, theoretical_ccc, delta=0.02, msg="Error in registration of subject %d: ccc %8.3f " "theoretical_ccc %8.3f " % (i, ccc, theoretical_ccc)) os.remove(fn_registration_results)
def test_registration(self): """Test the registration of 3 subjects from 1gyt.pdb at 0.5 SNR""" # Get model from PDB file smodel = IMP.Model() ssel = IMP.atom.ATOMPDBSelector() fn_model = self.get_input_file_name("1gyt.pdb") prot = IMP.atom.read_pdb(fn_model, smodel, ssel) particles = IMP.core.get_leaves(prot) # Read subject images srw = em2d.SpiderImageReaderWriter() selection_file = self.get_input_file_name("1gyt-subjects-0.5-SNR.sel") images_to_read_names = em2d.read_selection_file(selection_file) for i in range(0, len(images_to_read_names)): images_to_read_names[i] = self.get_input_file_name( images_to_read_names[i]) subjects = em2d.read_images(images_to_read_names, srw) self.assertEqual(len(subjects), 3, "Problem reading subject images") # Generate 20 evenly distributed projections from the PDB file n_projections = 20 proj_params = em2d.get_evenly_distributed_registration_results( n_projections) rows = 128 cols = 128 pixel_size = 1.5 # for generating projections, use a very high resolution resolution = 1 options = em2d.ProjectingOptions(pixel_size, resolution) projections = em2d.get_projections(particles, proj_params, rows, cols, options) self.assertEqual(len(projections), n_projections, "Problem generating projections") # Prepare registration # IMP.set_log_level(IMP.VERBOSE) finder = em2d.ProjectionFinder() score_function = em2d.EM2DScore() params = em2d.Em2DRestraintParameters(pixel_size, resolution, n_projections) params.save_match_images = False params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING params.optimization_steps = 30 params.simplex_initial_length = 0.1 params.simplex_minimum_size = 0.01 finder.setup(score_function, params) finder.set_model_particles(particles) finder.set_subjects(subjects) finder.set_projections(projections) finder.set_fast_mode(2) finder.get_complete_registration() # Recover the registration results: registration_parameters = finder.get_registration_results() fn_registration_results = "my_1gyt_registration.params" em2d.write_registration_results(fn_registration_results, registration_parameters) # Read the correct registration results: correct_parameters = em2d.read_registration_results( self.get_input_file_name("1gyt-subjects-0.5-SNR.params")) print("determined: ") for r in registration_parameters: print(r.get_rotation(), r.get_shift()) print("correct: ") for r in correct_parameters: print(r.get_rotation(), r.get_shift()) for i in range(0, len(registration_parameters)): # Generate the registered projection imgx = em2d.Image() imgx.set_size(rows, cols) em2d.get_projection(imgx, particles, registration_parameters[i], options) ccc = em2d.get_cross_correlation_coefficient( subjects[i].get_data(), imgx.get_data()) print(i, "ccc", ccc) snr = 0.5 theoretical_ccc = (snr / (1. + snr)) ** .5 self.assertAlmostEqual(ccc, theoretical_ccc, delta=0.02, msg="Error in registration of subject %d: ccc %8.3f " "theoretical_ccc %8.3f " % (i, ccc, theoretical_ccc)) os.remove(fn_registration_results)
#!/usr/bin/python import IMP import IMP.em2d as em2d import os from math import * import re import sys if(len(sys.argv) <2): print "Parameters: <1>" print "<1> Selection file with the names of the the EM images" sys.exit() fn_images_names = sys.argv[1] images_names = em2d.read_selection_file(fn_images_names) srw = em2d.SpiderImageReaderWriter() images=em2d.read_images(images_names,srw) # Replace the extension .spi for .jpg jpg_names=[] for s in images_names: jpg_names.append(re.sub(".spi",".jpg",s)) jrw = em2d.JPGImageReaderWriter() em2d.save_images(images,jpg_names,jrw)
r30 = core.DistanceRestraint(core.Harmonic(d30,1),chains[3],chains[0]) r30.set_name("distance 3-0") print "Distances in the solution: d01 =", \ d01,"d12 =",d12,"d23 =",d23,"d30 =",d30 # set distance restraints print "adding distance restraints " for r in [r01,r12,r23,r30]: m.add_restraint(r) print "model has ",m.get_number_of_restraints(),"restraints" # set em2D restraint srw = em2d.SpiderImageReaderWriter() selection_file=em2d.get_example_path("all-1z5s-projections.sel") images_to_read_names=[IMP.base.get_relative_path(selection_file, x) for x in em2d.read_selection_file(selection_file)] em_images =em2d.read_images(images_to_read_names,srw) print len(em_images),"images read" em2d_restraint = em2d.Em2DRestraint() apix=1.5 # sampling rate of the available EM images # resolution at which you want to generate the projections of the model # In principle you want "perfect" projections, so use the highest resolution resolution=1 # Number of projections to use for the initial registration # (coarse registration) to estimate the registration parameters n_projections=20 params = em2d.Em2DRestraintParameters(apix,resolution,n_projections) # This method (recommended) uses preprocessing of the images and projections # to speed-up the registration
def test_building_an_optimization_problem_with_em2d_restraint(self): """Test that an a optimization with em2d restraint is properly built""" m = IMP.Model() prot = IMP.atom.read_pdb(self.get_input_file_name("1z5s.pdb"), m,IMP.atom.ATOMPDBSelector()) # get the chains chains = IMP.atom.get_by_type(prot,IMP.atom.CHAIN_TYPE) # set the chains as rigid bodies rigid_bodies= [] native_chain_centers=[] for c in chains: atoms=IMP.core.get_leaves(c) rbd=IMP.core.RigidBody.setup_particle(c,atoms) rigid_bodies.append(rbd) native_chain_centers.append(rbd.get_coordinates()) self.assertEqual(len(rigid_bodies),4,"Problem generating rigid bodies") bb=IMP.algebra.BoundingBox3D(IMP.algebra.Vector3D(-25, -40,-60), IMP.algebra.Vector3D( 25, 40, 60)) # set distance restraints d01 = IMP.algebra.get_distance(native_chain_centers[0], native_chain_centers[1]) r01 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d01,1), chains[0], chains[1]) r01.set_name("distance 0-1") d12 = IMP.algebra.get_distance(native_chain_centers[1], native_chain_centers[2]) r12 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d12,1), chains[1], chains[2]) r12.set_name("distance 1-2") d23 = IMP.algebra.get_distance(native_chain_centers[2], native_chain_centers[3]) r23 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d23,1), chains[2], chains[3]) r23.set_name("distance 2-3") d30 = IMP.algebra.get_distance(native_chain_centers[3], native_chain_centers[0]) r30 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d30,1), chains[3], chains[0]) r30.set_name("distance 3-0") # set distance restraints for r in [r01,r12,r23,r30]: m.add_restraint(r) self.assertEqual(m.get_number_of_restraints(),4, "Incorrect number of distance restraints") # set em2D restraint srw = em2d.SpiderImageReaderWriter() selection_file=self.get_input_file_name("all-1z5s-projections.sel") images_to_read_names=[IMP.base.get_relative_path(selection_file, x) \ for x in em2d.read_selection_file(selection_file)] em_images =em2d.read_images(images_to_read_names,srw) self.assertEqual(len(em_images),3,"Incorrect number images read") apix=1.5 resolution=1 n_projections=20 params = em2d.Em2DRestraintParameters(apix,resolution,n_projections) params.save_match_images = False params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING score_function=em2d.EM2DScore() em2d_restraint = em2d.Em2DRestraint() em2d_restraint.setup(score_function, params) em2d_restraint.set_images(em_images) em2d_restraint.set_name("em2d restraint") container = IMP.container.ListSingletonContainer( IMP.core.get_leaves(prot)) em2d_restraint.set_particles(container) em2d_restraints_set=IMP.RestraintSet() em2d_restraints_set.add_restraint(em2d_restraint) em2d_restraints_set.set_weight(1000) # weight for the em2D restraint m.add_restraint(em2d_restraints_set) self.assertEqual(m.get_number_of_restraints(),5, "Incorrect number of restraints") # MONTECARLO OPTIMIZATION s=IMP.core.MonteCarlo(m) # Add movers for the rigid bodies movers=[] for rbd in rigid_bodies: movers.append(IMP.core.RigidBodyMover(rbd,5,2)) s.add_movers(movers) self.assertEqual(s.get_number_of_movers(),4, "Incorrect number of MonteCarlo movers") # Optimizer state to save intermediate configurations o_state=IMP.atom.WritePDBOptimizerState(chains, "intermediate-step-%1%.pdb") o_state.set_skip_steps(10) s.add_optimizer_state(o_state) ostate2 = WriteStatisticsOptimizerScore() s.add_optimizer_state(ostate2) # Perform optimization temperatures=[200,100,60,40,20,5] optimization_steps = 200 # for temp in temperatures: # s.optimize(optimization_steps) # IMP.atom.write_pdb(prot,"solution.pdb") self.assertTrue(True)
min_value = sequence[i] min_index =i # print "argmin - selecting ",min_value,min_index return min_value,min_index #*************************** fn_selection = em2d.get_example_path("all-models-1z5s.sel") fn_em2d_scores = em2d.get_example_path("em2d_scores_for_clustering.data") # Load models print "Reading models ..." model = IMP.Model() ssel = atom.ATOMPDBSelector() coords =[] fn_models = em2d.read_selection_file(fn_selection) n_models = len(fn_models) hierarchies=[] for fn in fn_models: fn_model=em2d.get_example_path(fn) h=atom.read_pdb(fn_model,model,ssel,True) hierarchies.append(h) xyz=core.XYZs(atom.get_leaves(h)) coords.append( [x.get_coordinates() for x in xyz]) print "Computing matrix of RMSD ..." rmsds=[[0.0 for i in range(0,n_models)] for n in range(0,n_models)] transformations=[[[] for i in range(0,n_models)] for j in range(0,n_models)] # fill rmsd and transformations for i in xrange(0,n_models): for j in xrange(i+1,n_models):
def score_model(complete_fn_model, images_sel_file, pixel_size, n_projections=20, resolution=1, images_per_batch=250): """ Score a model Scores a model against the images in the selection file. Reads the images in batchs to avoid memory problems resolution and pixel_size are used for generating projections of the model The finder is an em2d.ProjectionFinder used for projection matching and optimizations """ print "SCORING MODEL:",complete_fn_model cwd = os.getcwd() images_dir, nil = os.path.split(images_sel_file) images_names = em2d.read_selection_file(images_sel_file) n_images = len(images_names) if(n_images == 0): raise ValueError(" Scoring with a empty set of images") # TYPICAL OPTIMIZER PARAMETERS params = em2d.Em2DRestraintParameters(pixel_size, resolution) params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING params.optimization_steps = 4 params.simplex_initial_length = 0.1 params.simplex_minimum_size=0.02 params.save_match_images = True score_function = em2d.EM2DScore() finder = em2d.ProjectionFinder() finder.setup(score_function, params) # Get the number of rows and cols from the 1st image srw = em2d.SpiderImageReaderWriter() test_imgs = em2d.read_images([os.path.join(images_dir, images_names[0])], srw) rows = test_imgs[0].get_header().get_number_of_columns() cols = test_imgs[0].get_header().get_number_of_rows() model = IMP.Model() ssel = atom.ATOMPDBSelector() prot = atom.read_pdb(complete_fn_model, model, ssel) particles = IMP.core.get_leaves(prot) # generate projections proj_params = em2d.get_evenly_distributed_registration_results(n_projections) opts = em2d.ProjectingOptions(pixel_size, resolution) projections = em2d.get_projections(particles, proj_params, rows, cols, opts) finder.set_model_particles(particles) finder.set_projections(projections) optimized_solutions= 2 finder.set_fast_mode(optimized_solutions) # read the images in blocks to avoid memory problems all_registration_results = [] init_set = 0 init_time = time.time() while(init_set < n_images): end_set = min( init_set + images_per_batch, n_images ) if(images_dir != ""): os.chdir(images_dir) subjects = em2d.read_images(images_names[init_set:end_set], srw) # register finder.set_subjects(subjects) os.chdir(cwd) finder.get_complete_registration() # Recover the registration results: registration_results = finder.get_registration_results() for reg in registration_results: all_registration_results.append(reg) init_set += images_per_batch os.chdir(cwd) em2d.write_registration_results("registration.params", all_registration_results) print "score_model: time complete registration",time.time()-init_time print "coarse registration time",finder.get_coarse_registration_time() print "fine registration time",finder.get_fine_registration_time() return all_registration_results
## \example em2d/em_images_conversion.py # Conversion of Electron Microscopy Images. # import IMP import IMP.em2d as em2d import os """ Conversion of Electron Microscopy Images. """ # Read images fn_selection = em2d.get_example_path("all-1z5s-projections.sel") srw = em2d.SpiderImageReaderWriter() trw = em2d.TIFFImageReaderWriter() fn_images = em2d.read_selection_file(fn_selection) fn_images = [em2d.get_example_path(x) for x in fn_images] images = em2d.read_images(fn_images, srw) # write fn_saved = em2d.create_filenames(3, "1z5s-projection", "tif") em2d.save_images(images, fn_saved, trw)