def import_column_data(self, column_label, get_resolution=True):
if self.mtz_in_path is not None:
# Convert to ascii from if unicode
if isinstance(self.mtz_in_path, unicode):
self.mtz_in_path = self.mtz_in_path.encode('utf8')
self.mtz.open_read(self.mtz_in_path)
self.mtz.import_hkl_info(self.hkl_info)
self.spacegroup = self.hkl_info.spacegroup()
self.cell = self.hkl_info.cell()
f_phi = clipper.HKL_data_F_phi_float(self.hkl_info)
self.mtz.import_hkl_data(f_phi, '/*/*/' + column_label)
self.mtz.close_read()
# Convert to numpy
f_phi_np = numpy.zeros((f_phi.data_size() * len(f_phi)),
numpy.float)
f_phi.getDataNumpy(f_phi_np)
# Reshape and transpose
f_phi_np = numpy.reshape(f_phi_np, (-1, 2))
f_phi_np = numpy.transpose(f_phi_np)
# Convert to rec array to store col names
names = [n for n in f_phi.data_names().split()]
f_phi_np = np.core.records.fromarrays(f_phi_np,
names=names,
formats='float64, float64')
# Append to dictionary
self.column_data[column_label] = f_phi_np
# Get resolution column
if get_resolution:
res_np = numpy.zeros(f_phi_np.shape[0])
for n in xrange(f_phi_np.shape[0]):
r = self.hkl_info.invresolsq(n)
res_np[n] = r
self.column_data['resolution_1/Ang^2'] = res_np
def calculate_structure_factors(fsigf=None,
hklinfo=None,
mmol=None,
bulk_solvent=True):
log_string = "\n >> clipper_tools: structure_factors"
log_string += "\n bulk_solvent: %s" % bulk_solvent
xml_root = etree.Element('structure_factors')
xml_root.attrib['bulk_solvent'] = str(bulk_solvent)
crystal = clipper.MTZcrystal()
atoms = mmol.atom_list()
fc = clipper.HKL_data_F_phi_float(hklinfo, crystal)
if bulk_solvent:
sfcb = clipper.SFcalc_obs_bulk_float()
sfcb(fc, fsigf, atoms)
bulkfrc = sfcb.bulk_frac()
bulkscl = sfcb.bulk_scale()
etree.SubElement(xml_root, 'bulk_fraction').text = str(bulkfrc)
etree.SubElement(xml_root, 'bulk_scale').text = str(bulkscl)
log_string += "\n bulk_fraction: %f " % bulkfrc
log_string += "\n bulk_scale: %f " % bulkscl
else:
sfc = clipper.SFcalc_obs_base_float()
sfc(fc, fsigf, atoms)
log_string += "\n << structure_factors has finished\n"
xml_root.attrib['ok'] = 'yes'
return log_string, xml_root, fc
def prepare_map(mapin='', resol=8.0, callback=_callbacks.interactive_flush):
"""Reads EM map, sets origin to 0, pads cell and computes finely-sampled structure factors
Parameters:
mapin -- a string path to a map that will be read into a clipper.NXmap_float object
resol -- estimated resolution (float)
callback -- a function that takes care of log string and xml flushing
Returns:
a plain text log string, an XML etree and a clipper.HKL_data_F_phi_float object"""
def determine_extent(numpy_in, tolerance):
"""Reads numpy array, determines the extent of the electron density
Parameters:
numpy_in -- a numpy array containing grid points
tolerance -- number of points in a plane with value greater than 1 sigma
Returns:
a vector of grid indices: (min_u, max_u, min_v, max_v, min_w, max_w)"""
log_string = ''
min = clipper.Coord_orth()
max = clipper.Coord_orth()
map_mean = numpy.mean(map_numpy)
map_std = numpy.std(map_numpy)
mask = map_numpy > map_mean + map_std
sum_u = sum(sum(mask))
sum_w = sum(sum(numpy.transpose(mask)))
sum_v = sum(numpy.transpose(sum(mask)))
log_string += '\n >> dumping 1D summaries of the map\'s content:\n\n >> U:\n %s\n' % sum_u
log_string += '\n >> V:\n %s\n' % sum_v
log_string += '\n >> W:\n %s\n' % sum_w
point_list = []
for idx_u, val_u in enumerate(sum_u):
if val_u > tolerance:
point_list.append(idx_u)
min_u = point_list[0]
max_u = point_list[-1]
log_string += '\n >> First meaningful U: %i ; Last meaningful U: %i' % (
min_u, max_u)
point_list = []
for idx_v, val_v in enumerate(sum_v):
if val_v > tolerance:
point_list.append(idx_v)
min_v = point_list[0]
max_v = point_list[-1]
log_string += '\n >> First meaningful V: %i ; Last meaningful V: %i' % (
min_v, max_v)
point_list = []
for idx_w, val_w in enumerate(sum_w):
if val_w > tolerance:
point_list.append(idx_w)
min_w = point_list[0]
max_w = point_list[-1]
log_string += '\n >> First meaningful W: %i ; Last meaningful W: %i\n' % (
min_w, max_w)
extent = [min_u, max_u, min_v, max_v, min_w, max_w]
return extent, log_string
################# end determine_extent ################
############### main function ################
# create log string so console-based apps get some feedback
log_string = '\n >> clipper_tools: mr_from_em.prepare_map'
log_string += '\n mapin: %s' % mapin
log_string += '\n resol: %s' % resol
# create XML tree, to be merged in a global structured results file
xml_root = etree.Element('structure_factors')
xml_root.attrib['mapin'] = mapin
xml_root.attrib['resol'] = str(resol)
callback(log_string, xml_root)
phaser_params = {}
nxmap = clipper.NXmap_double()
xmap = clipper.Xmap_double()
map_file = clipper.CCP4MAPfile()
sg = clipper.Spacegroup.p1()
resol *= 0.9
resolution = clipper.Resolution(resol)
# nothing in, nothing out
if mapin == '':
return log_string, xml_root, None
# read the cryoEM map into nxmap, get map data irrespective of origin
map_file.open_read(mapin)
map_file.import_nxmap_double(nxmap)
map_file.close_read()
log_string += '\n >> file %s has been read as nxmap' % mapin
callback(log_string, xml_root)
# read the cryoEM map into xmap to get cell dimensions, etc.
map_file.open_read(mapin)
map_file.import_xmap_double(xmap)
map_file.close_read()
log_string += '\n >> file %s has been read as xmap' % mapin
callback(log_string, xml_root)
log_string += '\n >> cell parameters: %s' % xmap.cell().format()
log_string += '\n original translation: %s' % nxmap.operator_orth_grid(
).trn()
# put map content in a numpy data structure
map_numpy = numpy.zeros(
(nxmap.grid().nu(), nxmap.grid().nv(), nxmap.grid().nw()),
dtype='double')
log_string += '\n >> exporting a numpy array of %i x %i x %i grid points' \
% (nxmap.grid().nu(), nxmap.grid().nv(), nxmap.grid().nw())
callback(log_string, xml_root)
data_points = nxmap.export_numpy(map_numpy)
log_string += '\n >> %i data points have been exported' % data_points
callback(log_string, xml_root)
map_mean = numpy.mean(map_numpy)
map_stdv = numpy.std(map_numpy)
log_string += '\n >> map mean (stdev): %.4f (%.4f)' % (map_mean, map_stdv)
# compute the extent
extent, temp_log = determine_extent(map_numpy, 30)
log_string += temp_log
extent_list = [
extent[1] - extent[0], extent[3] - extent[2], extent[5] - extent[4]
]
max_extent = max(extent_list)
# create padded xmap and import numpy array
origin_trans = clipper.vec3_double(
extent[0] + ((extent[1] - extent[0]) / 2),
extent[2] + ((extent[3] - extent[2]) / 2),
extent[4] + ((extent[5] - extent[4]) / 2))
large_a = (xmap.cell().a() *
(max_extent + xmap.grid_asu().nu())) / xmap.grid_asu().nu()
large_b = (xmap.cell().b() *
(max_extent + xmap.grid_asu().nv())) / xmap.grid_asu().nv()
large_c = (xmap.cell().c() *
(max_extent + xmap.grid_asu().nw())) / xmap.grid_asu().nw()
cell_desc = clipper.Cell_descr(large_a, large_b, large_c, \
xmap.cell().alpha(), xmap.cell().beta(), xmap.cell().gamma())
large_p1_cell = clipper.Cell(cell_desc)
large_grid_sampling = clipper.Grid_sampling(
max_extent + xmap.grid_asu().nu(), max_extent + xmap.grid_asu().nv(),
max_extent + xmap.grid_asu().nw())
large_xmap = clipper.Xmap_double(sg, large_p1_cell, large_grid_sampling)
log_string += '\n >> new grid: nu=%i nv=%i nw=%i' % (large_xmap.grid_asu(
).nu(), large_xmap.grid_asu().nv(), large_xmap.grid_asu().nw())
log_string += '\n >> putting map into a large p1 cell...'
log_string += '\n >> new cell parameters: %s' % large_p1_cell.format()
callback(log_string, xml_root)
large_xmap.import_numpy(map_numpy)
# dump map to disk
map_file = clipper.CCP4MAPfile()
map_file.open_write('mapout_padded.mrc')
map_file.export_xmap_double(large_xmap)
map_file.close_write()
log_string += '\n >> map file mapout_padded.mrc written to disk'
callback(log_string, xml_root)
# import it back to nxmap so we can trivially shift the origin
map_file.open_read('mapout_padded.mrc')
map_file.import_nxmap_double(nxmap)
map_file.close_read()
log_string += '\n >> file mapout_padded.mrc has been read back as nxmap'
callback(log_string, xml_root)
# now shift the origin
rtop_zero = clipper.RTop_double(nxmap.operator_orth_grid().rot(),
origin_trans)
log_string += '\n >> moving origin...'
log_string += '\n original translation: %s new origin: %s' % (
nxmap.operator_orth_grid().trn(), rtop_zero.trn())
callback(log_string, xml_root)
nxmap_zero = clipper.NXmap_double(nxmap.grid(), rtop_zero)
nxmap_zero.import_numpy(map_numpy)
# dump map to disk
map_file.open_write('mapout_padded_zero.mrc')
map_file.export_nxmap_double(nxmap_zero)
map_file.close_write()
log_string += '\n >> map file mapout_padded_zero.mrc written to disk'
callback(log_string, xml_root)
# read it back to an xmap so we can fft-it
new_xmap = clipper.Xmap_double()
map_file.open_read('mapout_padded_zero.mrc')
map_file.import_xmap_double(new_xmap)
map_file.close_read()
log_string += '\n >> map file mapout_padded_zero.mrc read back as xmap'
callback(log_string, xml_root)
# create HKL_info using user-supplied resolution parameter
hkl_info = clipper.HKL_info(sg, large_p1_cell, resolution, True)
# fft the map
f_phi = clipper.HKL_data_F_phi_float(hkl_info, large_p1_cell)
log_string += '\n >> now computing map coefficients to %0.1f A resolution...' % resol
callback(log_string, xml_root)
new_xmap.fft_to(f_phi)
log_string += '\n >> writing map coefficients to MTZ file mapout_padded_zero.mtz'
callback(log_string, xml_root)
# setup an MTZ file so we can export our map coefficients
mtzout = clipper.CCP4MTZfile()
mtzout.open_write('mapout_padded_zero.mtz')
mtzout.export_hkl_info(f_phi.hkl_info())
mtzout.export_hkl_data(f_phi, '*/*/[F, PHI]')
mtzout.close_write()
log_string += '\n >> all done'
callback(log_string, xml_root)
return log_string, xml_root, f_phi, phaser_params
def calculate_map ( mtzin = "",
colin_fo = "F,SIGF",
colin_dano = "",
resol = None,
callback = callbacks.interactive_flush ) :
"""Reads EM map, sets origin to 0, pads cell and computes finely-sampled structure factors
Parameters:
mtzin -- a string path to an MTZ file containing measured amplitudes
colin-fo -- configurable column name, defaults to F,SIGF
colin-dano -- empty by default; if specified, patterson will be calculated on anomalous differences
resol -- resolution cutoff (float)
callback -- a function that takes care of log string and xml flushing
Returns:
a plain text log string, an XML etree and a clipper.HKL_data_F_phi_float object"""
# create log string so console-based apps get some feedback
log_string = "\n >> clipper_tools: xray.patterson.calculate_map"
log_string += "\n mtzin: %s" % mtzin
log_string += "\n colin_fo: %s" % colin_fo
log_string += "\n colin_dano: %s" % colin_dano
# create XML tree, to be merged in a global structured results file
xml_root = etree.Element('structure_factors')
xml_root.attrib['mtzin'] = mtzin
xml_root.attrib['colin_fo'] = colin_fo
xml_root.attrib['colin_dano'] = colin_dano
from clipper_tools.io.structure_factors import read_from_mtz
log_sub, xml_sub, hkl_info, hkl_data = read_from_mtz ( mtzin, colin_fo )
if resol is not None :
resolution_cutoff = clipper.Resolution(resol)
else :
resolution_cutoff = hkl_info.resolution()
log_string += "\n resol: %.2f" % ( resolution_cutoff.limit() )
xml_root.attrib['resol'] = "%.2f" % ( resolution_cutoff.limit() )
callback( log_string, xml_root )
p_spgr = clipper.Spacegroup ( clipper.Spgr_descr (hkl_info.spacegroup().generator_ops().patterson_ops()) )
p_hklinfo = clipper.HKL_info ( p_spgr, hkl_info.cell(), resolution_cutoff, True )
f_phi = clipper.HKL_data_F_phi_float (p_hklinfo) # map coefficients to be calculated later
numpy_data = numpy.zeros ((hkl_data.data_size() * len ( hkl_data )), numpy.float)
numpy_coef = numpy.zeros ((hkl_data.data_size() * len ( hkl_data )), numpy.float)
print "data array size is %i" % (hkl_data.data_size() * len ( hkl_data ))
n_data = hkl_data.export_numpy ( numpy_data )
numpy_data = numpy_data.reshape ( ( -1, 2) ) # so we have column 0 = F, column 1 = SIGF
print "len(f_phi)=%i len(original_data)= %i len(numpy_f_phi)=%i and n_data is %i" % (len(f_phi), len(hkl_data), len(numpy_data), n_data)
print numpy_data
numpy_data[:,1] = 0 # we are going to re-use the F,SIGF numpy array as F,PHI
# so we need to set PHI = 0 for the Patterson map
numpy_data[:,0] = numpy.power ( numpy_data[:,0], 2 ) # square the amplitudes
numpy_data = numpy_data.reshape ( -1 ) # need to put it back in a 1-D array for export
f_phi.import_numpy ( numpy_data )
from clipper_tools.io.map_coefficients import write_to_mtz
log_sub, xml_sub = write_to_mtz ( f_phi, "patterson.mtz" )
callback (log_string, xml_root )
return
fsig[ih] = clipper::data32::F_sigF();
// calculate E-F combination
for ( HRI ih = fsig.first(); !ih.last(); ih.next() )
if ( !fsig[ih].missing() )
fsig[ih].scale( pow( escale.f(ih), 0.5*weight_e ) );
"""
# get Patterson spacegroup
print spgr
print spgr.generator_ops()
pspgr = clipper.Spacegroup( clipper.Spgr_descr( spgr.generator_ops().patterson_ops() ) );
hklp.init( pspgr, cell, reso, True );
# make patterson coeffs
fphi = clipper.HKL_data_F_phi_float( hklp );
"""
for ( HRI ih = fphi.first(); !ih.last(); ih.next() ) {
clipper::data32::F_sigF f = fsig[ih.hkl()];
if ( !f.missing() ) {
fphi[ih].f() = f.f()*f.f();
fphi[ih].phi() = 0.0 ;
}
}
"""
# origin removal
if oremv:
basis_fp = clipper.BasisFn_spline( fphi, nprm, 2.0 );
print basis_fp
target_fp = clipper.TargetFn_meanFnth_F_phi(fphi, 1.0)
print cell print cell.a(), cell.b(), cell.c(), cell.alpha(), cell.beta(), cell.gamma() myfsigf = clipper.HKL_data_F_sigF_float(mydata) status = clipper.HKL_data_Flag(mydata) cif.import_hkl_data(myfsigf) cif.import_hkl_data(status) cif.close_read() print mydata.num_reflections() cxtl = clipper.MTZcrystal() fm = clipper.MMDBfile() fm.read_file(sys.argv[3]) mmol = clipper.MiniMol() fm.import_minimol(mmol) atoms = mmol.atom_list() print len(atoms) fc = clipper.HKL_data_F_phi_float(mydata, cxtl) sfcb = clipper.SFcalc_obs_bulk_float() print(fc, myfsigf, atoms) sfcb(fc, myfsigf, atoms) bulkfrc = sfcb.bulk_frac() bulkscl = sfcb.bulk_scale() print "Calculated structure factors ", bulkfrc, bulkscl fb = clipper.HKL_data_F_phi_float(mydata, cxtl) fd = clipper.HKL_data_F_phi_float(mydata, cxtl) phiw = clipper.HKL_data_Phi_fom_float(mydata, cxtl) flag = clipper.HKL_data_Flag(mydata, cxtl) freeflag = 1 print flag, dir(flag), flag.num_obs(), flag.data_size() # Unfortunately, this is horribly slow. A much better way is required. """ for ih in range(len(flag)):
def cut_by_model(mapin="",
pdbin="",
ipradius=1.5,
ipresol=8.0,
ipbfact=0.0,
callback=callbacks.interactive_flush):
xmap = clipper.Xmap_double()
sg = clipper.Spacegroup.p1()
resolution = clipper.Resolution(ipresol)
# create log string so console-based apps get some feedback
log_string = "\n >> clipper_tools: em.cut_density.cut_by_model"
log_string += "\n mapin: %s" % mapin
log_string += "\n pdbin: %s" % pdbin
log_string += "\n bfact: %s" % ipbfact
log_string += "\n resol: %s" % ipresol
log_string += "\n radius: %s" % ipradius
# create XML tree, to be merged in a global structured results file
xml_root = etree.Element('program')
xml_root.attrib['name'] = 'cut_by_model'
xml_root.attrib['user'] = getpass.getuser()
xml_root.attrib['date'] = time.strftime("%c")
params = etree.SubElement(xml_root, 'parameters')
params.attrib['mapin'] = mapin
params.attrib['pdbin'] = pdbin
params.attrib['b_factor'] = str(ipbfact)
params.attrib['resolution'] = str(ipresol)
params.attrib['mask_radius'] = str(ipradius)
callback(log_string, xml_root)
# nothing in, nothing out
if mapin == "" or pdbin == "":
return log_string, xml_root, None
# read the input atomic model
from clipper_tools.io.molecules import read_pdb
log_string_sub, xml_sub, mmol = read_pdb(pdbin)
log_string += log_string_sub
xml_root.append(xml_sub)
callback(log_string, xml_root)
# read the cryoEM map into xmap to get cell dimensions, etc.
from clipper_tools.io.maps import read_xmap
log_sub, xml_sub, xmap = read_xmap(mapin)
log_string += log_sub
xml_root.append(xml_sub)
callback(log_string, xml_root)
grid_sampling = clipper.Grid_sampling(xmap.grid_asu().nu(),
xmap.grid_asu().nv(),
xmap.grid_asu().nw())
log_string += "\n >> cell parameters: %s" % xmap.cell().format()
callback(log_string, xml_root)
# put map content in a numpy data structure
import numpy
map_numpy = numpy.zeros(
(xmap.grid_asu().nu(), xmap.grid_asu().nv(), xmap.grid_asu().nw()),
dtype='double')
log_string += "\n >> exporting a numpy array of %i x %i x %i grid points" \
% (xmap.grid_asu().nu(), xmap.grid_asu().nv(), xmap.grid_asu().nw())
data_points = xmap.export_numpy(map_numpy)
callback(log_string, xml_root)
atom_list = mmol.model().atom_list()
mask = clipper.Xmap_float(xmap.spacegroup(), xmap.cell(), grid_sampling)
masker = clipper.EDcalc_mask_float(ipradius)
masker.compute(mask, atom_list)
mask_matrix = numpy.zeros(
(xmap.grid_asu().nu(), xmap.grid_asu().nv(), xmap.grid_asu().nw()),
dtype='double')
mask_points = mask.export_numpy(mask_matrix)
log_string += "\n >> the original map has %i points and the computed mask has %i points" % (
data_points, mask_points)
callback(log_string, xml_root)
masked_array = map_numpy * mask_matrix
log_string += "\n >> non-zero values: original= %i ; mask=%i ; product=%i" % (
numpy.count_nonzero(map_numpy), numpy.count_nonzero(mask_matrix),
numpy.count_nonzero(masked_array))
xmap.import_numpy(masked_array)
# create HKL_info using user-supplied resolution parameter
hkl_info = clipper.HKL_info(xmap.spacegroup(), xmap.cell(), resolution,
True)
# fft the map
f_phi = clipper.HKL_data_F_phi_float(hkl_info, xmap.cell())
log_string += "\n >> now computing map coefficients to %0.1f A resolution..." % ipresol
callback(log_string, xml_root)
xmap.fft_to(f_phi)
log_string += "\n >> writing map coefficients to MTZ file mapout_cut_density.mtz"
callback(log_string, xml_root)
if ipbfact != 0.0:
f_phi.compute_scale_u_iso_fphi(1.0, clipper.Util.b2u(-ipbfact), f_phi)
log_string += "\n >> and applying B factor correction - using %3.2f\n" % ipbfact
# setup an MTZ file so we can export our map coefficients
from clipper_tools.io.map_coefficients import write_to_mtz
log_sub, xml_sub = write_to_mtz(f_phi, "mapout_cut_density.mtz")
log_string += log_sub
xml_root.append(xml_sub)
log_string += "\n >> all done"
xml_root.attrib['ok'] = 'yes'
callback(log_string, xml_root)
from clipper_tools.callbacks import offline_flush
offline_flush(log_string, xml_root)