def main():
'''Main routine for fast_dp.'''
os.environ['FAST_DP_FORKINTEGRATE'] = '1'
commandline = ' '.join(sys.argv)
parser = OptionParser(usage="fast_dp [options] imagefile")
parser.add_option("-?", action="help", help=SUPPRESS_HELP)
parser.add_option('-b',
'--beam',
dest='beam',
help='Beam centre: x, y (mm)')
parser.add_option('-d',
'--distance',
dest='distance',
help='Detector distance: d (mm)')
parser.add_option('-w',
'--wavelength',
dest='lambd',
help='Wavelength: lambd (Angstroms)')
parser.add_option('-a', '--atom', dest='atom', help='Atom type (e.g. Se)')
parser.add_option('-j',
'--number-of-jobs',
dest='number_of_jobs',
help='Number of jobs for integration')
parser.add_option('-k',
'--number-of-cores',
dest='number_of_cores',
help='Number of cores for integration')
parser.add_option('-J',
'--maximum-number-of-jobs',
dest='maximum_number_of_jobs',
help='Maximum number of jobs for integration')
parser.add_option(
'-l',
'--lib',
metavar="PLUGIN_LIBRARY",
dest='plugin_library',
help='image reader plugin path, ending with .so or "None"')
parser.add_option('-5',
'--h5toxds',
metavar="H5TOXDS",
dest='h5toxds',
help='program name or path for H5ToXds binary')
parser.add_option(
'-e',
'--execution-hosts',
'-n',
'--cluster-nodes',
metavar="CLUSTER_NODES",
dest='execution_hosts',
help='names or ip addresses for execution hosts for forkxds')
parser.add_option(
'-p',
'--pointless-aimless-host',
metavar="POINTLESS_AIMLESS_HOST",
dest='pa_host',
help='name or ip address for execution host for pointless and aimless')
parser.add_option('-c',
'--cell',
dest='cell',
help='Cell constants for processing, needs spacegroup')
parser.add_option('-s',
'--spacegroup',
metavar="SPACEGROUP",
dest='spacegroup',
help='Spacegroup for scaling and merging')
parser.add_option('-1',
'--first-image',
dest='first_image',
help='First image for processing')
parser.add_option('-N',
'--last-image',
dest='last_image',
help='First image for processing')
parser.add_option('-r',
'--resolution-high',
dest='resolution_high',
help='High resolution limit')
parser.add_option('-R',
'--resolution-low',
dest='resolution_low',
help='Low resolution limit')
parser.add_option(
'-o',
'--component-offsets',
metavar="COMPONENT",
dest='component_offsets',
help=
'Component offsets into working directory path: log_offset, prefix_start, prefix_end'
)
(options, args) = parser.parse_args()
if len(args) != 1:
sys.exit("You must point to one image of the dataset to process")
image = args[0]
xia2_format = re.match(r"^(.*):(\d+):(\d+)$", image)
if xia2_format:
# Image can be given in xia2-style format, ie.
# set_of_images_00001.cbf:1:5000
# to select images 1 to 5000. Resolve any conflicts
# with -1/-N in favour of the explicit arguments.
image = xia2_format.group(1)
if not options.first_image:
options.first_image = xia2_format.group(2)
if not options.last_image:
options.last_image = xia2_format.group(3)
#set up logging, at designated component if found, or in CWD otherwise
# The default is to just write fast_dp.log in CWD
# Component -1 is CWD itself. The next level up is -2, etc.
# if name_start is -1, the log name is fast_dp.log. If log_offset is
# also -1 or is 0, no additional log is written, otherwise the '_' separated
# concatenation of the components prefixed to '_fast_dp.log' is the
# name of the log file
log_archive_path = os.getcwd()
log_archive_prefix = ''
if not options.component_offsets:
options.component_offsets = os.getenv('FAST_DP_LOG_COMPONENT_OFFSETS',
'0,0,0')
log_offset, prefix_start, prefix_end = options.component_offsets.split(',')
log_offset = int(log_offset)
prefix_start = int(prefix_start)
prefix_end = int(prefix_end)
cur_offset = -1
head = log_archive_path
components = {}
paths = {}
while head:
paths[cur_offset] = head
(head, tail) = os.path.split(head)
components[cur_offset] = tail
cur_offset = cur_offset - 1
if head == '/':
break
if log_offset <= -1 and log_offset > cur_offset:
try:
log_archive_path = paths[log_offset]
except:
log_archive_path = os.getcwd()
if prefix_start <= prefix_end and prefix_end <= 0 and prefix_start > cur_offset:
cur_offset = prefix_start
while cur_offset <= prefix_end:
log_archive_prefix = log_archive_prefix + components[
cur_offset] + '_'
cur_offset = cur_offset + 1
if version == 2:
try:
write('log_archive_path: %s' % log_archive_path)
write('log_archive_prefix: %s' % log_archive_prefix)
except:
pass
else:
write('log_archive_path: {}'.format(log_archive_path))
write('log_archive_prefix: {}'.format(log_archive_prefix))
if (log_offset < -1):
set_afilepath(log_archive_path)
set_afileprefix(log_archive_prefix)
set_afilename(
os.path.join(log_archive_path, log_archive_prefix + "fast_dp.log"))
try:
fast_dp = FastDP()
fast_dp._commandline = commandline
if version == 2:
try:
write('Fast_DP installed in: %s' % os.environ['FAST_DP_ROOT'])
write('Starting image: %s' % image)
except:
pass
else:
write('Fast_DP installed in: {}'.format(
os.environ['FAST_DP_ROOT']))
write('Starting image: {}'.format(image))
missing = fast_dp.set_start_image(image)
if options.beam:
x, y = tuple(map(float, options.beam.split(',')))
fast_dp.set_beam((x, y))
if options.distance:
fast_dp.set_distance(float(options.distance))
if options.lambd:
fast_dp.set_wavelength(float(options.lambd))
if options.atom:
fast_dp.set_atom(options.atom)
if options.maximum_number_of_jobs:
fast_dp.set_max_n_jobs(int(options.maximum_number_of_jobs))
if options.execution_hosts:
fast_dp.set_execution_hosts(options.execution_hosts.split(','))
if version == 2:
try:
write('Execution hosts: %s' %
' '.join(fast_dp.get_execution_hosts()))
except:
pass
else:
write('Execution hosts: {}'.format(' '.join(
fast_dp.get_execution_hosts())))
if options.pa_host:
fast_dp.set_pa_host(options.pa_host)
if version == 2:
try:
write('pointless/aimless host: %s' % fast_dp.get_pa_host())
except:
pass
else:
write('pointless/aimless host: {}'.format(
fast_dp.get_pa_host()))
if options.number_of_jobs:
if options.maximum_number_of_jobs:
n_jobs = min(int(options.number_of_jobs),
int(options.maximum_number_of_jobs))
else:
n_jobs = int(options.number_of_jobs)
fast_dp.set_n_jobs(n_jobs)
if options.number_of_cores:
n_cores = int(options.number_of_cores)
fast_dp.set_n_cores(n_cores)
if options.plugin_library:
fast_dp.set_plugin_library(options.plugin_library)
else:
fast_dp.set_plugin_library(" ")
if options.h5toxds:
fast_dp.set_h5toxds(options.h5toxds)
fast_dp.set_plugin_library("None")
else:
fast_dp.set_h5toxds(" ")
if options.first_image:
first_image = int(options.first_image)
missing = [m for m in missing if m >= first_image]
fast_dp.set_first_image(first_image)
if options.last_image:
last_image = int(options.last_image)
missing = [m for m in missing if m <= last_image]
fast_dp.set_last_image(last_image)
if missing:
if version == 2:
try:
raise RuntimeError, 'images missing: %s' % \
' '.join(map(str, missing))
except:
pass
else:
raise RuntimeError('images missing: {}'.format(' '.join(
map(str, missing))))
if options.resolution_low:
fast_dp.set_resolution_low(float(options.resolution_low))
if options.resolution_high:
fast_dp.set_resolution_high(float(options.resolution_high))
# must input spacegroup first as unpacking of the unit cell
# will depend on the centering operation...
if options.spacegroup:
if version == 2:
try:
try:
spacegroup = check_spacegroup_name(options.spacegroup)
fast_dp.set_input_spacegroup(spacegroup)
write('Set spacegroup: %s' % spacegroup)
except RuntimeError:
write('Spacegroup %s not recognised: ignoring' % \
options.spacegroup)
except:
pass
else:
try:
spacegroup = check_spacegroup_name(options.spacegroup)
fast_dp.set_input_spacegroup(spacegroup)
write('Set spacegroup: {}'.format(spacegroup))
except RuntimeError:
write('Spacegroup {} not recognised: ignoring'.format(
options.spacegroup))
if options.cell:
assert (options.spacegroup)
cell = check_split_cell(options.cell)
if version == 2:
try:
write('Set cell: %.2f %.2f %.2f %.2f %.2f %.2f' % cell)
except:
pass
else:
write('Set cell: {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}'.
format(cell))
fast_dp.set_input_cell(cell)
fast_dp.process()
except Exception as e:
traceback.print_exc(file=open('fast_dp.error', 'w'))
if version == 2:
try:
write('Fast DP error: %s' % str(e))
except:
pass
else:
write('Fast DP error: {}'.format(str(e)))
fdpelogpath = get_afilepath()
fdpelogprefix = get_afileprefix()
if fdpelogpath:
if version == 2:
try:
try:
shutil.copyfile(
'fast_dp.error',
os.path.join(fdpelogpath,
fdpelogprefix + 'fast_dp.error'))
write('Archived fast_dp.error to %s' % os.path.join(
fdpelogpath, fdpelogprefix + 'fast_dp.error'))
except:
write(
'fast_dp.error not archived to %s' % os.path.join(
fdpelogpath, fdpelogprefix + 'fast_dp.error'))
except:
pass
else:
try:
shutil.copyfile(
'fast_dp.error',
os.path.join(fdpelogpath,
fdpelogprefix + 'fast_dp.error'))
write('Archived fast_dp.error to {}'.format(
os.path.join(fdpelogpath,
fdpelogprefix + 'fast_dp.error')))
except:
write('fast_dp.error not archived to {}'.format(
os.path.join(fdpelogpath,
fdpelogprefix + 'fast_dp.error')))
json_stuff = {}
for prop in dir(fast_dp):
ignore = ['_read_image_metadata']
if not prop.startswith('_') or prop.startswith('__'):
continue
if prop in ignore:
continue
json_stuff[prop] = getattr(fast_dp, prop)
with open('fast_dp.state', 'wb') as fh:
json.dump(json_stuff, fh)
def decide_pointgroup(p1_unit_cell, metadata, input_spacegroup=None):
'''
Run POINTLESS to get the list of allowed pointgroups (N.B. will
insist on triclinic symmetry for this scaling step) then run
pointless on the resulting reflection file to get the idea of the
best pointgroup to use. Then return the correct pointgroup and
cell.
Parameters
----------
p1_unit_cell : tuple
metadata : dict
input_spacegroup : tuple, optional
Returns
-------
returns several values of different types
unit_cell <tuple>, space_group_number <int>
resolution_high <float>
'''
assert (p1_unit_cell)
# write correct input file for the triclinic solution, in the
# working directory
xds_inp = 'P1.INP'
write_xds_inp_correct(metadata, p1_unit_cell, 1, xds_inp, turn_subset=True)
shutil.copyfile(xds_inp, 'XDS.INP')
run_job('xds_par')
shutil.copyfile('CORRECT.LP', 'P1.LP')
# get the list of allowed lattices
results = read_xds_idxref_lp('CORRECT.LP')
# also read out the resolution limit
resolution_high = read_correct_lp_get_resolution('CORRECT.LP')
# run pointless, get the list of suggested lattices and pointgroups
# FIXME should use the program manager for this... yes, this will
# check that the executable is available too!
xdsin = 'XDS_ASCII.HKL'
xmlout = 'pointless.xml'
pointless_log = run_job('pointless_wrapper',
arguments=['xdsin', xdsin, 'xmlout', xmlout],
stdin=['systematicabsences off'])
fout = open('pointless.log', 'w')
for record in pointless_log:
fout.write(record)
fout.close()
# now read the XML file
pointless_results = read_pointless_xml(xmlout)
# select the top solution which is allowed, return this
if input_spacegroup:
sg_accepted = False
input_spacegroup = "".join(input_spacegroup.split())
pointgroup = ersatz_pointgroup(input_spacegroup)
if pointgroup.startswith('H'):
pointgroup = pointgroup.replace('H', 'R')
lattice = spacegroup_to_lattice(input_spacegroup)
for r in pointless_results:
result_sg = "".join(check_spacegroup_name(r[1]).split(' '))
if lattice_to_spacegroup(lattice) in results and \
ersatz_pointgroup(result_sg) == pointgroup:
space_group_number = r[1]
unit_cell = results[lattice_to_spacegroup(r[0])][1]
write('Happy with sg# {}'.format(space_group_number))
write(
'{0[0]:6.2f} {0[1]:6.2f} {0[2]:6.2f} {0[3]:6.2f} {0[4]:6.2f} {0[5]:6.2f}'
.format(unit_cell))
sg_accepted = True
break
if not sg_accepted:
write('No indexing solution for spacegroup {} so ignoring'.format(
input_spacegroup))
input_spacegroup = None
# if input space group obviously nonsense, allow to ignore just warn
if not input_spacegroup:
for r in pointless_results:
if lattice_to_spacegroup(r[0]) in results:
space_group_number = r[1]
unit_cell = results[lattice_to_spacegroup(r[0])][1]
write('Happy with sg# {}'.format(space_group_number))
write(
'{0[0]:6.2f} {0[1]:6.2f} {0[2]:6.2f} {0[3]:6.2f} {0[4]:6.2f} {0[5]:6.2f}'
.format(unit_cell))
break
else:
write('Rejected solution {0[0]} {0[1]:3d}'.format(r))
# this should probably be a proper check...
assert (space_group_number)
# also save the P1 XDS_ASCII.HKL file see
# http://trac.diamond.ac.uk/scientific_software/ticket/1106
shutil.copyfile('XDS_ASCII.HKL', 'XDS_P1.HKL')
return unit_cell, space_group_number, resolution_high
def main():
'''Main routine for fast_rdp.'''
os.environ['FAST_DP_FORKINTEGRATE'] = '1'
from optparse import OptionParser
commandline = ' '.join(sys.argv)
parser = OptionParser()
parser.add_option('-a', '--atom', dest = 'atom',
help = 'Atom type (e.g. Se)')
parser.add_option('-c', '--cell', dest = 'cell',
help = 'Cell constants for processing, needs spacegroup')
parser.add_option('-s', '--spacegroup', dest = 'spacegroup',
help = 'Spacegroup for scaling and merging')
parser.add_option('-1', '--first-image', dest = 'first_image',
help = 'First image for processing')
parser.add_option('-N', '--last-image', dest = 'last_image',
help = 'First image for processing')
parser.add_option('-r', '--resolution-high', dest = 'resolution_high',
help = 'High resolution limit')
parser.add_option('-R', '--resolution-low', dest = 'resolution_low',
help = 'Low resolution limit')
(options, args) = parser.parse_args()
assert(len(args) < 2)
# if arg given then assume that this is a directory with a fast_dp
# job it in, but where $user does not have access to write - so first
# copy the files needed across
if len(args) == 1:
if not os.path.isdir(args[0]):
if version == 2:
try:
raise RuntimeError, 'in this mode, provide /path/to/fast_dp/dir'
except:
pass
else:
raise RuntimeError('in this mode, provide /path/to/fast_dp/dir')
from_dir = args[0]
for filename in os.listdir(from_dir):
if os.path.isdir(os.path.join(from_dir, filename)):
continue
import shutil
shutil.copyfile(os.path.join(from_dir, filename),
os.path.join(os.getcwd(), filename))
else:
from_dir = None
try:
fast_rdp = FastRDP()
fast_rdp._commandline = commandline
if version == 2:
try:
write('Fast_RDP installed in: %s' % os.environ['FAST_DP_ROOT'])
write('Working in: %s' % os.getcwd())
except:
pass
else:
write('Fast_RDP installed in: {}'.format(os.environ['FAST_DP_ROOT']))
write('Working in: {}'.format(os.getcwd()))
if from_dir:
if version == 2:
try:
write('Working from: %s' % from_dir)
except:
pass
else:
write('Working from: {}'.format(from_dir))
if options.atom:
fast_rdp.set_atom(options.atom)
if options.first_image:
first_image = int(options.first_image)
fast_rdp.set_first_image(first_image)
if options.last_image:
last_image = int(options.last_image)
fast_rdp.set_last_image(last_image)
if options.resolution_low:
fast_rdp.set_resolution_low(float(options.resolution_low))
if options.resolution_high:
fast_rdp.set_resolution_high(float(options.resolution_high))
# must input spacegroup first as unpacking of the unit cell
# will depend on the centering operation...
if options.spacegroup:
if version == 2:
try:
spacegroup = check_spacegroup_name(options.spacegroup)
fast_rdp.set_input_spacegroup(spacegroup)
write('Set spacegroup: %s' % spacegroup)
except RuntimeError as e:
write('Spacegroup %s not recognised: ignoring' % \
options.spacegroup)
else:
try:
spacegroup = check_spacegroup_name(options.spacegroup)
fast_rdp.set_input_spacegroup(spacegroup)
write('Set spacegroup: {}'.format(spacegroup))
except RuntimeError as e:
write('Spacegroup {} not recognised: ignoring'.format(
options.spacegroup))
if options.cell:
assert(options.spacegroup)
cell = check_split_cell(options.cell)
if version == 2:
try:
write('Set cell: %.2f %.2f %.2f %.2f %.2f %.2f' % cell)
except:
pass
else:
write('Set cell: {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}'.format(cell))
fast_rdp.set_input_cell(cell)
fast_rdp.reprocess()
except Exception as e:
traceback.print_exc(file = open('fast_rdp.error', 'w'))
if version == 2:
try:
write('Fast RDP error: %s' % str(e))
except:
pass
else:
write('Fast RDP error: {}'.format(str(e)))