def no_images(xds_inp):
firstlast, phi, records = rj_parse_idxref_xds_inp(
open(xds_inp, 'r').readlines())
images = [j for j in range(firstlast[0], firstlast[1] + 1)]
# first run with all images, then define the 'correct' lattice
# as the one which results from this autoindex step.
xds_inp = open('XDS.INP', 'w')
for record in records:
xds_inp.write('%s\n' % record)
xds_inp.write('SPOT_RANGE= %d %d\n' % firstlast)
xds_inp.close()
output = rj_run_job('xds', [], [])
cell = rj_parse_idxref_lp(open('IDXREF.LP', 'r').readlines())
result = lattice_symmetry(cell)
lattice = sort_lattices(result.keys())[-1]
score = result[lattice]['penalty']
metrics = []
for count in range(10):
result = calculate_images(images, phi, count + 1)
xds_inp = open('XDS.INP', 'w')
for record in records:
xds_inp.write('%s\n' % record)
for pair in result:
xds_inp.write('SPOT_RANGE= %d %d\n' % pair)
xds_inp.close()
output = rj_run_job('xds', [], [])
cell = rj_parse_idxref_lp(open('IDXREF.LP', 'r').readlines())
result = lattice_symmetry(cell)
l = sort_lattices(result.keys())[-1]
if l != lattice:
raise RuntimeError, 'cell refinement gave wrong lattice'
metrics.append(result[l]['penalty'])
return metrics, score
def no_images(labelit_log):
# 3 images per wedge, maximum of 30 => 1 to 10 wedges.
beam, lattice, metric, cell, image = rj_parse_labelit_log_file(labelit_log)
template, directory = rj_get_template_directory(image)
images = rj_find_matching_images(image)
phi = rj_get_phi(image)
if lattice == 'aP':
raise RuntimeError, 'triclinic lattices useless'
# right, what I want to do is autoindex with images at 0, 45, 90 or
# thereabouts (in P1), then do the cell refinement, then score the
# resulting cell constants
ai_images = calculate_images_ai(images, phi, 3)
metrics = []
for count in range(1, 10):
result = calculate_images(images, phi, count + 1)
# first autoindex commands
commands = [
'template %s' % template,
'directory %s' % directory,
'beam %f %f' % beam]
commands.append('symm P1')
for image in ai_images:
commands.append('autoindex dps refine image %d' % image)
commands.append('mosaic estimate')
commands.append('go')
# the cell refinement commands
commands.append('postref multi segments 3')
for pair in result:
commands.append('process %d %d' % pair)
commands.append('go')
output = rj_run_job('ipmosflm-7.0.3', [], commands)
cell, mosaic = rj_parse_mosflm_cr_log(output)
result = lattice_symmetry(cell)
l = sort_lattices(result.keys())[-1]
if l != lattice:
raise RuntimeError, 'cell refinement gave wrong lattice'
metrics.append(result[l]['penalty'])
return metrics
def phi_spacing(labelit_log):
beam, lattice, metric, cell, image = rj_parse_labelit_log_file(labelit_log)
template, directory = rj_get_template_directory(image)
images = rj_find_matching_images(image)
phi = rj_get_phi(image)
if lattice == 'aP':
raise RuntimeError, 'triclinic lattices useless'
# then copy the dataset_preferences.py somewhere safe
if os.path.exists('dataset_preferences.py'):
shutil.copyfile('dataset_preferences.py',
'dataset_preferences.bak')
# write out a dataset preferences file
fout = open('dataset_preferences.py', 'w')
fout.write('beam = (%f, %f)\n' % beam)
fout.write('wedgelimit = 3\n')
fout.write('beam_search_scope = 1.0\n')
fout.close()
# generate the list of phi values...
phis = [float(j + 1) for j in range(5, 45)]
image_numbers = []
for p in phis:
result = calculate_images(images, phi, p)
if not result in image_numbers:
image_numbers.append(result)
# now run labelit with phi spacing 6-45
metrics = []
spacings = []
for i_n in image_numbers:
spacing = phi * (i_n[2] - i_n[1])
image_names = [rj_image_name(template, directory, i) for i in i_n]
output = rj_run_job('labelit.screen --index_only',
image_names, [])
b, l, m, c, i = rj_parse_labelit_log(output)
if l != lattice:
raise RuntimeError, 'incorrect result with %d images' % (count + 1)
metrics.append(m)
spacings.append(spacing)
if os.path.exists('dataset_preferences.bak'):
shutil.copyfile('dataset_preferences.bak', 'dataset_preferences.py')
return metrics, spacings
def no_images(labelit_log):
# first parse this
beam, lattice, metric, cell, image = rj_parse_labelit_log_file(labelit_log)
template, directory = rj_get_template_directory(image)
images = rj_find_matching_images(image)
phi = rj_get_phi(image)
if lattice == 'aP':
raise RuntimeError, 'triclinic lattices useless'
# then copy the dataset_preferences.py somewhere safe
if os.path.exists('dataset_preferences.py'):
shutil.copyfile('dataset_preferences.py',
'dataset_preferences.bak')
# write out a dataset preferences file
fout = open('dataset_preferences.py', 'w')
fout.write('beam = (%f, %f)\n' % beam)
fout.write('wedgelimit = 15\n')
fout.write('beam_search_scope = 1.0\n')
fout.close()
# now run labelit with 1 - 15 images
metrics = []
times = []
for count in range(15):
result = calculate_images(images, phi, count + 1)
image_names = [rj_image_name(template, directory, i) for i in result]
t0 = time.time()
output = rj_run_job('labelit.screen --index_only',
image_names, [])
t1 = time.time()
times.append((t1 - t0))
b, l, m, c, i = rj_parse_labelit_log(output)
if l != lattice:
raise RuntimeError, 'incorrect result with %d images' % (count + 1)
metrics.append(m)
if os.path.exists('dataset_preferences.bak'):
shutil.copyfile('dataset_preferences.bak', 'dataset_preferences.py')
return metrics, times
def gather(pmin, pmax, files):
data = { }
for j in range(1, 10):
data[j + 1] = []
here = os.getcwd()
for f in files:
# hack to get the image name, so that I can then get (and test) the
# phi range...
directory = os.path.split(f)[0]
os.chdir(directory)
output = rj_run_job('labelit.stats_index', [], [])
os.chdir(here)
b, l, m, c, i = rj_parse_labelit_log(output)
phi = rj_get_phi(i)
if phi < pmin or phi > pmax:
continue
records = open(f, 'r').readlines()
if not len(records) == 9:
continue
for r in records:
s = r.split()
n = int(s[0])
m = float(s[1])
data[n].append(m)
for j in range(1, 10):
positive_data = []
for d in data[j + 1]:
if d > 0:
positive_data.append(d)
m, s = meansd(positive_data)
print '%d %.3f %.3f' % (j + 1, m, s)
print '%d points' % len(data[2])
def phi_spacing(xds_inp):
firstlast, phi, records = rj_parse_idxref_xds_inp(
open(xds_inp, 'r').readlines())
images = [j for j in range(firstlast[0], firstlast[1] + 1)]
# first run with all images, then define the 'correct' lattice
# as the one which results from this autoindex step.
xds_inp = open('XDS.INP', 'w')
for record in records:
xds_inp.write('%s\n' % record)
xds_inp.write('SPOT_RANGE= %d %d\n' % firstlast)
xds_inp.close()
output = rj_run_job('xds', [], [])
cell = rj_parse_idxref_lp(open('IDXREF.LP', 'r').readlines())
result = lattice_symmetry(cell)
lattice = sort_lattices(result.keys())[-1]
score = result[lattice]['penalty']
metrics = []
spacings = []
phis = [float(j + 1) for j in range(10, 45)]
image_numbers = []
for p in phis:
result = calculate_images(images, phi, p)
if phi * (result[-1][-1] - result[0][0] + 1) > 90.0:
continue
if not result in image_numbers:
image_numbers.append(result)
for result in image_numbers:
spacing = nint(phi * (result[1][0] - result[0][0]))
spacings.append(spacing)
xds_inp = open('XDS.INP', 'w')
for record in records:
xds_inp.write('%s\n' % record)
for pair in result:
xds_inp.write('SPOT_RANGE= %d %d\n' % pair)
xds_inp.close()
output = rj_run_job('xds', [], [])
cell = rj_parse_idxref_lp(open('IDXREF.LP', 'r').readlines())
result = lattice_symmetry(cell)
l = sort_lattices(result.keys())[-1]
if l != lattice:
raise RuntimeError, 'cell refinement gave wrong lattice'
metrics.append(result[l]['penalty'])
return metrics, spacings, score
def cr_test(labelit_log):
beam, lattice, metric, cell, image = rj_parse_labelit_log_file(labelit_log)
lattices, cells = rj_parse_labelit_log_lattices(
open(labelit_log).readlines())
template, directory = rj_get_template_directory(image)
images = rj_find_matching_images(image)
phi = rj_get_phi(image)
if lattice == 'aP':
raise RuntimeError, 'triclinic lattices useless'
wedges = calculate_images(images, phi)
ai_images = calculate_images_ai(images, phi, 3)
# run a quick autoindex (or re-read the labelit log file above) to
# generate the list of possible unit cell etc.
rmsds_all = { }
# then loop over these
for lattice in lattices:
commands = [
'template %s' % template,
'directory %s' % directory,
'beam %f %f' % beam]
commands.append('symm %d' % lattice_spacegroup(lattice))
commands.append('cell %f %f %f %f %f %f' % tuple(cells[lattice]))
for image in ai_images:
commands.append('autoindex dps refine image %d' % image)
commands.append('mosaic estimate')
commands.append('go')
# the cell refinement commands
commands.append('postref multi segments 3')
for pair in wedges:
commands.append('process %d %d' % pair)
commands.append('go')
for c in commands:
# print c
pass
output = rj_run_job('ipmosflm-7.0.3', [], commands)
images, rmsds = rj_parse_mosflm_cr_log_rmsd(output)
rmsds_all[lattice] = rmsds
# and finally calculate the RMSD ratios.
# break up by lattice, image and cycle
for lattice in lattices[:-1]:
print lattice
values = []
for cycle in rmsds_all[lattice]:
if not cycle in rmsds_all['aP']:
continue
record = '%3d' % cycle
for j in range(len(images)):
record += ' %.3f' % (rmsds_all[lattice][cycle][j] /
rmsds_all['aP'][cycle][j])
values.append((rmsds_all[lattice][cycle][j] /
rmsds_all['aP'][cycle][j]))
print record
m, s = meansd(values)
print ':: %s %.3f %.3f' % (lattice, m, s)
def lattice_test(integrate_lp, xds_inp_file):
images, phi, cell, records = rj_parse_integrate_lp(
open(integrate_lp).readlines())
# next work through the XDS.INP file to get the proper name template
# out...
nt = None
distance = None
for record in open(xds_inp_file, 'r').readlines():
if 'NAME_TEMPLATE_OF_DATA_FRAMES' in record:
nt = record.strip()
if 'DETECTOR_DISTANCE' in record:
distance = record.strip()
if not nt:
raise RuntimeError, 'filename template not found in %s' % xds_inp_file
if not distance:
raise RuntimeError, 'distance not found in %s' % xds_inp_file
r_new = [distance]
for r in records:
if not 'NAME_TEMPLATE_OF_DATA_FRAMES' in r:
r_new.append(r)
else:
r_new.append(nt)
records = r_new
# ok, in here need to rerun XDS with all of the data from all of
# the images and the triclinic target cell, then parse out the
# solutions from the CORRECT.LP file (applying the cell constants -
# done in the parser) and then use *these* as the target, as the
# lattice symmetry code (interestingly) does not always give the
# right answer...
standard = [
'JOB=CORRECT',
'MAXIMUM_NUMBER_OF_PROCESSORS=4',
'CORRECTIONS=!',
'REFINE(CORRECT)=CELL',
'OVERLOAD=65000',
'DIRECTION_OF_DETECTOR_X-AXIS=1.0 0.0 0.0',
'DIRECTION_OF_DETECTOR_Y-AXIS=0.0 1.0 0.0',
'TRUSTED_REGION=0.0 1.41'
]
# first get the list of possible lattices - do this by running CORRECT
# with all of the images, then looking at the favourite settings for the
# P1 result (or something) - meh.
fout = open('XDS.INP', 'w')
for record in standard:
fout.write('%s\n' % record)
for record in records:
fout.write('%s\n' % record)
fout.write('DATA_RANGE= %d %d\n' % images)
fout.write('OSCILLATION_RANGE= %.2f\n' % phi)
fout.write(
'UNIT_CELL_CONSTANTS= %.2f %.2f %.2f %.2f %.2f %.2f\n' % tuple(cell))
fout.write('SPACE_GROUP_NUMBER=%d\n' % 1)
fout.close()
output = rj_run_job('xds_par', [], [])
# read CORRECT.LP to get the right solutions...
result = rj_parse_xds_correct_lp(open('CORRECT.LP', 'r').readlines())
for lattice in result:
cp = '%.2f %.2f %.2f %.2f %.2f %.2f' % result[lattice]['cell']
# print '%s %s' % (lattice, cp)
# result = lattice_symmetry(cell)
lattices = sort_lattices(result)
# then iterate through them...
data = { }
for l in lattices:
data[l] = { }
c = result[l]['cell']
fout = open('XDS.INP', 'w')
for record in standard:
fout.write('%s\n' % record)
for record in records:
fout.write('%s\n' % record)
fout.write('DATA_RANGE= %d %d\n' % (images))
fout.write('OSCILLATION_RANGE= %.2f\n' % phi)
fout.write(
'UNIT_CELL_CONSTANTS= %.2f %.2f %.2f %.2f %.2f %.2f\n' % tuple(c))
fout.write('SPACE_GROUP_NUMBER=%d\n' % lattice_spacegroup(l))
fout.close()
output = rj_run_job('xds_par', [], [])
# now read out the records I want from CORRECT.LP...
rmsd = None
rmsp = None
for record in open('CORRECT.LP').readlines():
if 'STANDARD DEVIATION OF SPOT POSITION' in record:
rmsd = float(record.split()[-1])
if 'STANDARD DEVIATION OF SPINDLE POSITION' in record:
rmsp = float(record.split()[-1])
if not rmsp or not rmsd:
raise RuntimeError, 'refinement failed'
print '%s rmsd %f rmsp %f' % (l, rmsd, rmsp)
def phi_spacing(labelit_log):
# 3 images per wedge, maximum of 30 => 1 to 10 wedges.
beam, lattice, metric, cell, image = rj_parse_labelit_log_file(labelit_log)
template, directory = rj_get_template_directory(image)
images = rj_find_matching_images(image)
phi = rj_get_phi(image)
if lattice == 'aP':
raise RuntimeError, 'triclinic lattices useless'
# right, what I want to do is autoindex with images at 0, 45, 90 or
# thereabouts (in P1), then do the cell refinement, then score the
# resulting cell constants
ai_images = calculate_images_ai(images, phi, 3)
metrics = []
spacings = []
phis = [float(j + 1) for j in range(10, 45)]
image_numbers = []
for p in phis:
result = calculate_images(images, phi, p)
if phi * (result[-1][-1] - result[0][0] + 1) > 90.0:
continue
if not result in image_numbers:
image_numbers.append(result)
for result in image_numbers:
# first autoindex commands
spacing = nint(phi * (result[1][0] - result[0][0]))
spacings.append(spacing)
commands = [
'template %s' % template,
'directory %s' % directory,
'beam %f %f' % beam]
commands.append('symm P1')
for image in ai_images:
commands.append('autoindex dps refine image %d' % image)
commands.append('mosaic estimate')
commands.append('go')
# the cell refinement commands
commands.append('postref multi segments 3')
for pair in result:
commands.append('process %d %d' % pair)
commands.append('go')
output = rj_run_job('ipmosflm-7.0.3', [], commands)
try:
cell, mosaic = rj_parse_mosflm_cr_log(output)
except RuntimeError, e:
for record in output:
print record[:-1]
raise e
result = lattice_symmetry(cell)
l = sort_lattices(result.keys())[-1]
if l != lattice:
raise RuntimeError, 'cell refinement gave wrong lattice'
metrics.append(result[l]['penalty'])
def lattice_test(integrate_lp, xds_inp_file):
images, phi, cell, records = rj_parse_integrate_lp(
open(integrate_lp).readlines())
# next work through the XDS.INP file to get the proper name template
# out...
nt = None
distance = None
for record in open(xds_inp_file, 'r').readlines():
if 'NAME_TEMPLATE_OF_DATA_FRAMES' in record:
nt = record.strip()
if 'DETECTOR_DISTANCE' in record:
distance = record.strip()
if not nt:
raise RuntimeError, 'filename template not found in %s' % xds_inp_file
if not distance:
raise RuntimeError, 'distance not found in %s' % xds_inp_file
r_new = [distance]
for r in records:
if not 'NAME_TEMPLATE_OF_DATA_FRAMES' in r:
r_new.append(r)
else:
r_new.append(nt)
records = r_new
# ok, in here need to rerun XDS with all of the data from all of
# the images and the triclinic target cell, then parse out the
# solutions from the CORRECT.LP file (applying the cell constants -
# done in the parser) and then use *these* as the target, as the
# lattice symmetry code (interestingly) does not always give the
# right answer...
standard = [
'JOB=CORRECT',
'MAXIMUM_NUMBER_OF_PROCESSORS=4',
'CORRECTIONS=!',
'REFINE(CORRECT)=CELL',
'OVERLOAD=65000',
'DIRECTION_OF_DETECTOR_X-AXIS=1.0 0.0 0.0',
'DIRECTION_OF_DETECTOR_Y-AXIS=0.0 1.0 0.0',
'TRUSTED_REGION=0.0 1.41'
]
# first get the list of possible lattices - do this by running CORRECT
# with all of the images, then looking at the favourite settings for the
# P1 result (or something) - meh.
fout = open('XDS.INP', 'w')
for record in standard:
fout.write('%s\n' % record)
for record in records:
fout.write('%s\n' % record)
fout.write('DATA_RANGE= %d %d\n' % images)
fout.write('OSCILLATION_RANGE= %.2f\n' % phi)
fout.write(
'UNIT_CELL_CONSTANTS= %.2f %.2f %.2f %.2f %.2f %.2f\n' % tuple(cell))
fout.write('SPACE_GROUP_NUMBER=%d\n' % 1)
fout.close()
output = rj_run_job('xds_par', [], [])
# read CORRECT.LP to get the right solutions...
result = rj_parse_xds_correct_lp(open('CORRECT.LP', 'r').readlines())
for lattice in result:
cp = '%.2f %.2f %.2f %.2f %.2f %.2f' % result[lattice]['cell']
# print '%s %s' % (lattice, cp)
# result = lattice_symmetry(cell)
lattices = sort_lattices(result)
# then iterate through them...
data = { }
for l in lattices:
data[l] = { }
c = result[l]['cell']
# print 'Lattice: %s' % l
# print 'Cell: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(c)
# then iterate through the image ranges
w = nint(10.0/phi)
m = nint((images[1] - images[0] + 1) / w)
for j in range(m):
start = j * w + 1
end = j * w + w
data[l][j] = { }
fout = open('XDS.INP', 'w')
for record in standard:
fout.write('%s\n' % record)
for record in records:
fout.write('%s\n' % record)
fout.write('DATA_RANGE= %d %d\n' % (start, end))
fout.write('OSCILLATION_RANGE= %.2f\n' % phi)
fout.write(
'UNIT_CELL_CONSTANTS= %.2f %.2f %.2f %.2f %.2f %.2f\n' % tuple(c))
fout.write('SPACE_GROUP_NUMBER=%d\n' % lattice_spacegroup(l))
fout.close()
output = rj_run_job('xds_par', [], [])
# now read out the records I want from CORRECT.LP...
rmsd = None
rmsp = None
for record in open('CORRECT.LP').readlines():
if 'STANDARD DEVIATION OF SPOT POSITION' in record:
rmsd = float(record.split()[-1])
if 'STANDARD DEVIATION OF SPINDLE POSITION' in record:
rmsp = float(record.split()[-1])
if not rmsp or not rmsd:
raise RuntimeError, 'refinement failed'
data[l][j] = {'d':rmsd,
'p':rmsp}
# now tabulate the results
for j in range(m):
record = '%d' % j
for l in lattices[1:]:
record += ' %.3f %.3f' % (data[l][j]['d'] / data['aP'][j]['d'],
data[l][j]['p'] / data['aP'][j]['p'])
print record
# now print out the averages, sd's
recordm = 'M'
records = 'S'
sigma = { }
for l in lattices[1:]:
values = [(data[l][j]['d'] / data['aP'][j]['d']) for j in range(m)]
md, sd = meansd(values)
values = [(data[l][j]['p'] / data['aP'][j]['p']) for j in range(m)]
mp, sp = meansd(values)
recordm += ' %.3f %.3f' % (md, mp)
records += ' %.3f %.3f' % (sd, sp)
sigma[l] = { }
if sd > 0:
sigma[l]['d'] = ((md - 1) / sd)
else:
sigma[l]['d'] = 0.0
if sp > 0:
sigma[l]['p'] = ((mp - 1) / sp)
else:
sigma[l]['p'] = 0.0
print recordm
print records
for l in lattices[1:]:
d = sigma[l]['d']
p = sigma[l]['p']
print '= %s %.3f %.3f' % (l, d, p)
