def get_em2d_restraint(assembly,
images_selection_file,
restraint_params,
mode="fast",
n_optimized=1):
""" Sets a restraint for comparing the model to a set of EM images
"""
model = assembly.get_model()
# Setup the restraint
sc = em2d.EM2DScore()
r = em2d.Em2DRestraint(model)
r.setup(sc, restraint_params)
names = em2d.read_selection_file(images_selection_file)
names = [base.get_relative_path(images_selection_file, x) for x in names]
log.debug("names of the images %s", names)
srw = em2d.SpiderImageReaderWriter()
imgs = em2d.read_images(names, srw)
r.set_images(imgs)
ps = atom.get_leaves(assembly)
lsc = container.ListSingletonContainer(ps)
r.set_particles(lsc)
if (mode == "coarse"):
r.set_coarse_registration_mode(True)
elif (mode == "fast"):
r.set_fast_mode(n_optimized)
elif (mode == "complete"):
pass
else:
raise ValueError("Em2DRestraint mode not recognized")
return r
def get_em2d_restraint( assembly,
images_selection_file,
restraint_params,
mode="fast",
n_optimized=1):
""" Sets a restraint for comparing the model to a set of EM images
"""
model = assembly.get_model()
# Setup the restraint
sc = em2d.EM2DScore()
r = em2d.Em2DRestraint()
r.setup(sc, restraint_params)
names = em2d.read_selection_file(images_selection_file)
names = [base.get_relative_path(images_selection_file, x) for x in names]
log.debug("names of the images %s", names)
srw = em2d.SpiderImageReaderWriter()
imgs = em2d.read_images(names, srw)
r.set_images(imgs)
ps = atom.get_leaves(assembly)
lsc = container.ListSingletonContainer(ps)
r.set_particles(lsc)
if (mode == "coarse"):
r.set_coarse_registration_mode(True)
elif (mode == "fast"):
r.set_fast_mode(n_optimized)
elif(mode == "complete"):
pass
else:
raise ValueError("Em2DRestraint mode not recognized")
return r
def test_building_an_optimization_problem_with_em2d_restraint(self):
"""Test that an a optimization with em2d restraint is properly built"""
m = IMP.kernel.Model()
prot = IMP.atom.read_pdb(self.get_input_file_name("1z5s.pdb"),
m, IMP.atom.ATOMPDBSelector())
# get the chains
chains = IMP.atom.get_by_type(prot, IMP.atom.CHAIN_TYPE)
# set the chains as rigid bodies
rigid_bodies = []
native_chain_centers = []
for c in chains:
atoms = IMP.core.get_leaves(c)
rbd = IMP.core.RigidBody.setup_particle(c, atoms)
rbd.set_coordinates_are_optimized(True)
rigid_bodies.append(rbd)
native_chain_centers.append(rbd.get_coordinates())
self.assertEqual(
len(rigid_bodies),
4,
"Problem generating rigid bodies")
bb = IMP.algebra.BoundingBox3D(IMP.algebra.Vector3D(-25, -40, -60),
IMP.algebra.Vector3D(25, 40, 60))
# set distance restraints
d01 = IMP.algebra.get_distance(native_chain_centers[0],
native_chain_centers[1])
r01 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d01, 1),
chains[0],
chains[1])
r01.set_name("distance 0-1")
d12 = IMP.algebra.get_distance(native_chain_centers[1],
native_chain_centers[2])
r12 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d12, 1),
chains[1],
chains[2])
r12.set_name("distance 1-2")
d23 = IMP.algebra.get_distance(native_chain_centers[2],
native_chain_centers[3])
r23 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d23, 1),
chains[2],
chains[3])
r23.set_name("distance 2-3")
d30 = IMP.algebra.get_distance(native_chain_centers[3],
native_chain_centers[0])
r30 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d30, 1),
chains[3],
chains[0])
r30.set_name("distance 3-0")
# set distance restraints
for r in [r01, r12, r23, r30]:
m.add_restraint(r)
self.assertEqual(m.get_number_of_restraints(), 4,
"Incorrect number of distance restraints")
# set em2D restraint
srw = em2d.SpiderImageReaderWriter()
selection_file = self.get_input_file_name("all-1z5s-projections.sel")
images_to_read_names = [IMP.base.get_relative_path(selection_file, x)
for x in em2d.read_selection_file(selection_file)]
em_images = em2d.read_images(images_to_read_names, srw)
self.assertEqual(len(em_images), 3, "Incorrect number images read")
apix = 1.5
resolution = 1
n_projections = 20
params = em2d.Em2DRestraintParameters(apix, resolution, n_projections)
params.save_match_images = False
params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING
score_function = em2d.EM2DScore()
em2d_restraint = em2d.Em2DRestraint(m)
em2d_restraint.setup(score_function, params)
em2d_restraint.set_images(em_images)
em2d_restraint.set_name("em2d restraint")
container = IMP.container.ListSingletonContainer(
IMP.core.get_leaves(prot))
em2d_restraint.set_particles(container)
em2d_restraints_set = IMP.kernel.RestraintSet(m)
em2d_restraints_set.add_restraint(em2d_restraint)
em2d_restraints_set.set_weight(1000) # weight for the em2D restraint
m.add_restraint(em2d_restraints_set)
self.assertEqual(m.get_number_of_restraints(), 5,
"Incorrect number of restraints")
# MONTECARLO OPTIMIZATION
s = IMP.core.MonteCarlo(m)
# Add movers for the rigid bodies
movers = []
for rbd in rigid_bodies:
movers.append(IMP.core.RigidBodyMover(rbd, 5, 2))
s.add_movers(movers)
self.assertEqual(s.get_number_of_movers(), 4,
"Incorrect number of MonteCarlo movers")
# Optimizer state to save intermediate configurations
o_state = IMP.atom.WritePDBOptimizerState(chains,
"intermediate-step-%1%.pdb")
o_state.set_period(11)
s.add_optimizer_state(o_state)
ostate2 = WriteStatisticsOptimizerScore(m)
s.add_optimizer_state(ostate2)
# Perform optimization
temperatures = [200, 100, 60, 40, 20, 5]
optimization_steps = 200
# for temp in temperatures:
# s.optimize(optimization_steps)
# IMP.atom.write_pdb(prot,"solution.pdb")
self.assertTrue(True)
import IMP
import IMP.em2d as em2d
import os
"""
Conversion of Electron Microscopy Images.
"""
# Read images
fn_selection=em2d.get_example_path("all-1z5s-projections.sel")
srw = em2d.SpiderImageReaderWriter()
trw = em2d.TIFFImageReaderWriter()
fn_images=em2d.read_selection_file(fn_selection)
fn_images = [em2d.get_example_path(x) for x in fn_images]
images = em2d.read_images(fn_images,srw)
# write
fn_saved=em2d.create_filenames(3,"1z5s-projection","tif")
em2d.save_images(images,fn_saved,trw)
def test_registration(self):
"""Test the registration of 3 subjects from 1gyt.pdb at 0.5 SNR"""
# Get model from PDB file
smodel = IMP.Model()
ssel = IMP.atom.ATOMPDBSelector()
fn_model = self.get_input_file_name("1gyt.pdb")
prot = IMP.atom.read_pdb(fn_model, smodel, ssel)
particles = IMP.core.get_leaves(prot)
# Read subject images
srw = em2d.SpiderImageReaderWriter()
selection_file = self.get_input_file_name("1gyt-subjects-0.5-SNR.sel")
images_to_read_names = em2d.read_selection_file(selection_file)
for i in range(0, len(images_to_read_names)):
images_to_read_names[i] = self.get_input_file_name(
images_to_read_names[i])
subjects = em2d.read_images(images_to_read_names, srw)
self.assertEqual(len(subjects), 3, "Problem reading subject images")
# Generate 20 evenly distributed projections from the PDB file
n_projections = 20
proj_params = em2d.get_evenly_distributed_registration_results(
n_projections)
rows = 128
cols = 128
pixel_size = 1.5
# for generating projections, use a very high resolution
resolution = 8.5
options = em2d.ProjectingOptions(pixel_size, resolution)
projections = em2d.get_projections(particles, proj_params, rows, cols,
options)
self.assertEqual(len(projections), n_projections,
"Problem generating projections")
# Prepare registration
# IMP.set_log_level(IMP.VERBOSE)
finder = em2d.ProjectionFinder()
score_function = em2d.EM2DScore()
params = em2d.Em2DRestraintParameters(pixel_size, resolution,
n_projections)
params.save_match_images = False
params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING
params.optimization_steps = 30
params.simplex_initial_length = 0.1
params.simplex_minimum_size = 0.01
finder.setup(score_function, params)
finder.set_model_particles(particles)
finder.set_subjects(subjects)
finder.set_projections(projections)
finder.set_fast_mode(2)
finder.get_complete_registration()
# Recover the registration results:
registration_parameters = finder.get_registration_results()
fn_registration_results = "my_1gyt_registration.params"
em2d.write_registration_results(fn_registration_results,
registration_parameters)
# Read the correct registration results:
correct_parameters = em2d.read_registration_results(
self.get_input_file_name("1gyt-subjects-0.5-SNR.params"))
print("determined: ")
for r in registration_parameters:
print(r.get_rotation(), r.get_shift())
print("correct: ")
for r in correct_parameters:
print(r.get_rotation(), r.get_shift())
for i in range(0, len(registration_parameters)):
# Generate the registered projection
imgx = em2d.Image()
imgx.set_size(rows, cols)
em2d.get_projection(imgx, particles, registration_parameters[i],
options)
ccc = em2d.get_cross_correlation_coefficient(
subjects[i].get_data(), imgx.get_data())
print(i, "ccc", ccc)
snr = 0.5
theoretical_ccc = (snr / (1. + snr))**.5
self.assertAlmostEqual(
ccc,
theoretical_ccc,
delta=0.02,
msg="Error in registration of subject %d: ccc %8.3f "
"theoretical_ccc %8.3f " % (i, ccc, theoretical_ccc))
os.remove(fn_registration_results)
def test_registration(self):
"""Test the registration of 3 subjects from 1gyt.pdb at 0.5 SNR"""
# Get model from PDB file
smodel = IMP.Model()
ssel = IMP.atom.ATOMPDBSelector()
fn_model = self.get_input_file_name("1gyt.pdb")
prot = IMP.atom.read_pdb(fn_model, smodel, ssel)
particles = IMP.core.get_leaves(prot)
# Read subject images
srw = em2d.SpiderImageReaderWriter()
selection_file = self.get_input_file_name("1gyt-subjects-0.5-SNR.sel")
images_to_read_names = em2d.read_selection_file(selection_file)
for i in range(0, len(images_to_read_names)):
images_to_read_names[i] = self.get_input_file_name(
images_to_read_names[i])
subjects = em2d.read_images(images_to_read_names, srw)
self.assertEqual(len(subjects), 3, "Problem reading subject images")
# Generate 20 evenly distributed projections from the PDB file
n_projections = 20
proj_params = em2d.get_evenly_distributed_registration_results(
n_projections)
rows = 128
cols = 128
pixel_size = 1.5
# for generating projections, use a very high resolution
resolution = 1
options = em2d.ProjectingOptions(pixel_size, resolution)
projections = em2d.get_projections(particles, proj_params,
rows, cols, options)
self.assertEqual(len(projections), n_projections,
"Problem generating projections")
# Prepare registration
# IMP.set_log_level(IMP.VERBOSE)
finder = em2d.ProjectionFinder()
score_function = em2d.EM2DScore()
params = em2d.Em2DRestraintParameters(pixel_size,
resolution, n_projections)
params.save_match_images = False
params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING
params.optimization_steps = 30
params.simplex_initial_length = 0.1
params.simplex_minimum_size = 0.01
finder.setup(score_function, params)
finder.set_model_particles(particles)
finder.set_subjects(subjects)
finder.set_projections(projections)
finder.set_fast_mode(2)
finder.get_complete_registration()
# Recover the registration results:
registration_parameters = finder.get_registration_results()
fn_registration_results = "my_1gyt_registration.params"
em2d.write_registration_results(fn_registration_results,
registration_parameters)
# Read the correct registration results:
correct_parameters = em2d.read_registration_results(
self.get_input_file_name("1gyt-subjects-0.5-SNR.params"))
print("determined: ")
for r in registration_parameters:
print(r.get_rotation(), r.get_shift())
print("correct: ")
for r in correct_parameters:
print(r.get_rotation(), r.get_shift())
for i in range(0, len(registration_parameters)):
# Generate the registered projection
imgx = em2d.Image()
imgx.set_size(rows, cols)
em2d.get_projection(imgx, particles,
registration_parameters[i], options)
ccc = em2d.get_cross_correlation_coefficient(
subjects[i].get_data(),
imgx.get_data())
print(i, "ccc", ccc)
snr = 0.5
theoretical_ccc = (snr / (1. + snr)) ** .5
self.assertAlmostEqual(ccc, theoretical_ccc, delta=0.02,
msg="Error in registration of subject %d: ccc %8.3f "
"theoretical_ccc %8.3f " % (i, ccc, theoretical_ccc))
os.remove(fn_registration_results)
#!/usr/bin/python
import IMP
import IMP.em2d as em2d
import os
from math import *
import re
import sys
if(len(sys.argv) <2):
print "Parameters: <1>"
print "<1> Selection file with the names of the the EM images"
sys.exit()
fn_images_names = sys.argv[1]
images_names = em2d.read_selection_file(fn_images_names)
srw = em2d.SpiderImageReaderWriter()
images=em2d.read_images(images_names,srw)
# Replace the extension .spi for .jpg
jpg_names=[]
for s in images_names:
jpg_names.append(re.sub(".spi",".jpg",s))
jrw = em2d.JPGImageReaderWriter()
em2d.save_images(images,jpg_names,jrw)
r30 = core.DistanceRestraint(core.Harmonic(d30,1),chains[3],chains[0])
r30.set_name("distance 3-0")
print "Distances in the solution: d01 =", \
d01,"d12 =",d12,"d23 =",d23,"d30 =",d30
# set distance restraints
print "adding distance restraints "
for r in [r01,r12,r23,r30]:
m.add_restraint(r)
print "model has ",m.get_number_of_restraints(),"restraints"
# set em2D restraint
srw = em2d.SpiderImageReaderWriter()
selection_file=em2d.get_example_path("all-1z5s-projections.sel")
images_to_read_names=[IMP.base.get_relative_path(selection_file, x) for x in
em2d.read_selection_file(selection_file)]
em_images =em2d.read_images(images_to_read_names,srw)
print len(em_images),"images read"
em2d_restraint = em2d.Em2DRestraint()
apix=1.5 # sampling rate of the available EM images
# resolution at which you want to generate the projections of the model
# In principle you want "perfect" projections, so use the highest resolution
resolution=1
# Number of projections to use for the initial registration
# (coarse registration) to estimate the registration parameters
n_projections=20
params = em2d.Em2DRestraintParameters(apix,resolution,n_projections)
# This method (recommended) uses preprocessing of the images and projections
# to speed-up the registration
def test_building_an_optimization_problem_with_em2d_restraint(self):
"""Test that an a optimization with em2d restraint is properly built"""
m = IMP.Model()
prot = IMP.atom.read_pdb(self.get_input_file_name("1z5s.pdb"),
m,IMP.atom.ATOMPDBSelector())
# get the chains
chains = IMP.atom.get_by_type(prot,IMP.atom.CHAIN_TYPE)
# set the chains as rigid bodies
rigid_bodies= []
native_chain_centers=[]
for c in chains:
atoms=IMP.core.get_leaves(c)
rbd=IMP.core.RigidBody.setup_particle(c,atoms)
rigid_bodies.append(rbd)
native_chain_centers.append(rbd.get_coordinates())
self.assertEqual(len(rigid_bodies),4,"Problem generating rigid bodies")
bb=IMP.algebra.BoundingBox3D(IMP.algebra.Vector3D(-25, -40,-60),
IMP.algebra.Vector3D( 25, 40, 60))
# set distance restraints
d01 = IMP.algebra.get_distance(native_chain_centers[0],
native_chain_centers[1])
r01 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d01,1),
chains[0],
chains[1])
r01.set_name("distance 0-1")
d12 = IMP.algebra.get_distance(native_chain_centers[1],
native_chain_centers[2])
r12 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d12,1),
chains[1],
chains[2])
r12.set_name("distance 1-2")
d23 = IMP.algebra.get_distance(native_chain_centers[2],
native_chain_centers[3])
r23 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d23,1),
chains[2],
chains[3])
r23.set_name("distance 2-3")
d30 = IMP.algebra.get_distance(native_chain_centers[3],
native_chain_centers[0])
r30 = IMP.core.DistanceRestraint(IMP.core.Harmonic(d30,1),
chains[3],
chains[0])
r30.set_name("distance 3-0")
# set distance restraints
for r in [r01,r12,r23,r30]:
m.add_restraint(r)
self.assertEqual(m.get_number_of_restraints(),4,
"Incorrect number of distance restraints")
# set em2D restraint
srw = em2d.SpiderImageReaderWriter()
selection_file=self.get_input_file_name("all-1z5s-projections.sel")
images_to_read_names=[IMP.base.get_relative_path(selection_file, x) \
for x in em2d.read_selection_file(selection_file)]
em_images =em2d.read_images(images_to_read_names,srw)
self.assertEqual(len(em_images),3,"Incorrect number images read")
apix=1.5
resolution=1
n_projections=20
params = em2d.Em2DRestraintParameters(apix,resolution,n_projections)
params.save_match_images = False
params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING
score_function=em2d.EM2DScore()
em2d_restraint = em2d.Em2DRestraint()
em2d_restraint.setup(score_function, params)
em2d_restraint.set_images(em_images)
em2d_restraint.set_name("em2d restraint")
container = IMP.container.ListSingletonContainer(
IMP.core.get_leaves(prot))
em2d_restraint.set_particles(container)
em2d_restraints_set=IMP.RestraintSet()
em2d_restraints_set.add_restraint(em2d_restraint)
em2d_restraints_set.set_weight(1000) # weight for the em2D restraint
m.add_restraint(em2d_restraints_set)
self.assertEqual(m.get_number_of_restraints(),5,
"Incorrect number of restraints")
# MONTECARLO OPTIMIZATION
s=IMP.core.MonteCarlo(m)
# Add movers for the rigid bodies
movers=[]
for rbd in rigid_bodies:
movers.append(IMP.core.RigidBodyMover(rbd,5,2))
s.add_movers(movers)
self.assertEqual(s.get_number_of_movers(),4,
"Incorrect number of MonteCarlo movers")
# Optimizer state to save intermediate configurations
o_state=IMP.atom.WritePDBOptimizerState(chains,
"intermediate-step-%1%.pdb")
o_state.set_skip_steps(10)
s.add_optimizer_state(o_state)
ostate2 = WriteStatisticsOptimizerScore()
s.add_optimizer_state(ostate2)
# Perform optimization
temperatures=[200,100,60,40,20,5]
optimization_steps = 200
# for temp in temperatures:
# s.optimize(optimization_steps)
# IMP.atom.write_pdb(prot,"solution.pdb")
self.assertTrue(True)
min_value = sequence[i]
min_index =i
# print "argmin - selecting ",min_value,min_index
return min_value,min_index
#***************************
fn_selection = em2d.get_example_path("all-models-1z5s.sel")
fn_em2d_scores = em2d.get_example_path("em2d_scores_for_clustering.data")
# Load models
print "Reading models ..."
model = IMP.Model()
ssel = atom.ATOMPDBSelector()
coords =[]
fn_models = em2d.read_selection_file(fn_selection)
n_models = len(fn_models)
hierarchies=[]
for fn in fn_models:
fn_model=em2d.get_example_path(fn)
h=atom.read_pdb(fn_model,model,ssel,True)
hierarchies.append(h)
xyz=core.XYZs(atom.get_leaves(h))
coords.append( [x.get_coordinates() for x in xyz])
print "Computing matrix of RMSD ..."
rmsds=[[0.0 for i in range(0,n_models)] for n in range(0,n_models)]
transformations=[[[] for i in range(0,n_models)] for j in range(0,n_models)]
# fill rmsd and transformations
for i in xrange(0,n_models):
for j in xrange(i+1,n_models):
def score_model(complete_fn_model,
images_sel_file,
pixel_size,
n_projections=20,
resolution=1,
images_per_batch=250):
""" Score a model
Scores a model against the images in the selection file.
Reads the images in batchs to avoid memory problems
resolution and pixel_size are used for generating projections of the model
The finder is an em2d.ProjectionFinder used for projection matching and optimizations
"""
print "SCORING MODEL:",complete_fn_model
cwd = os.getcwd()
images_dir, nil = os.path.split(images_sel_file)
images_names = em2d.read_selection_file(images_sel_file)
n_images = len(images_names)
if(n_images == 0):
raise ValueError(" Scoring with a empty set of images")
# TYPICAL OPTIMIZER PARAMETERS
params = em2d.Em2DRestraintParameters(pixel_size, resolution)
params.coarse_registration_method = em2d.ALIGN2D_PREPROCESSING
params.optimization_steps = 4
params.simplex_initial_length = 0.1
params.simplex_minimum_size=0.02
params.save_match_images = True
score_function = em2d.EM2DScore()
finder = em2d.ProjectionFinder()
finder.setup(score_function, params)
# Get the number of rows and cols from the 1st image
srw = em2d.SpiderImageReaderWriter()
test_imgs = em2d.read_images([os.path.join(images_dir, images_names[0])], srw)
rows = test_imgs[0].get_header().get_number_of_columns()
cols = test_imgs[0].get_header().get_number_of_rows()
model = IMP.Model()
ssel = atom.ATOMPDBSelector()
prot = atom.read_pdb(complete_fn_model, model, ssel)
particles = IMP.core.get_leaves(prot)
# generate projections
proj_params = em2d.get_evenly_distributed_registration_results(n_projections)
opts = em2d.ProjectingOptions(pixel_size, resolution)
projections = em2d.get_projections(particles, proj_params, rows, cols, opts)
finder.set_model_particles(particles)
finder.set_projections(projections)
optimized_solutions= 2
finder.set_fast_mode(optimized_solutions)
# read the images in blocks to avoid memory problems
all_registration_results = []
init_set = 0
init_time = time.time()
while(init_set < n_images):
end_set = min( init_set + images_per_batch, n_images )
if(images_dir != ""):
os.chdir(images_dir)
subjects = em2d.read_images(images_names[init_set:end_set], srw)
# register
finder.set_subjects(subjects)
os.chdir(cwd)
finder.get_complete_registration()
# Recover the registration results:
registration_results = finder.get_registration_results()
for reg in registration_results:
all_registration_results.append(reg)
init_set += images_per_batch
os.chdir(cwd)
em2d.write_registration_results("registration.params", all_registration_results)
print "score_model: time complete registration",time.time()-init_time
print "coarse registration time",finder.get_coarse_registration_time()
print "fine registration time",finder.get_fine_registration_time()
return all_registration_results
## \example em2d/em_images_conversion.py
# Conversion of Electron Microscopy Images.
#
import IMP
import IMP.em2d as em2d
import os
"""
Conversion of Electron Microscopy Images.
"""
# Read images
fn_selection = em2d.get_example_path("all-1z5s-projections.sel")
srw = em2d.SpiderImageReaderWriter()
trw = em2d.TIFFImageReaderWriter()
fn_images = em2d.read_selection_file(fn_selection)
fn_images = [em2d.get_example_path(x) for x in fn_images]
images = em2d.read_images(fn_images, srw)
# write
fn_saved = em2d.create_filenames(3, "1z5s-projection", "tif")
em2d.save_images(images, fn_saved, trw)
