def search_spots(img_template, n_images, config_mgr):
filenames = dataset.template_to_filenames(img_template, 1, n_images)
img_files, idxes = [], []
for i, imgf in enumerate(filenames):
if os.path.isfile(imgf):
img_files.append(imgf)
idxes.append(i+1)
# Assuming the images are all on the same configuration.
detkey = config_mgr.get_key_by_img(img_files[0])
params = config_mgr.get_params_by_key(detkey)
file_stats = spot_finder_generic.run(img_files, params)
return [(idx, f, stat) for idx, (f, stat) in zip(idxes, file_stats)]
def search_spots(img_template, n_images, config_mgr):
filenames = dataset.template_to_filenames(img_template, 1, n_images)
img_files, idxes = [], []
for i, imgf in enumerate(filenames):
if os.path.isfile(imgf):
img_files.append(imgf)
idxes.append(i + 1)
# Assuming the images are all on the same configuration.
detkey = config_mgr.get_key_by_img(img_files[0])
params = config_mgr.get_params_by_key(detkey)
file_stats = spot_finder_generic.run(img_files, params)
return [(idx, f, stat) for idx, (f, stat) in zip(idxes, file_stats)]
def all_image_files_exist(self, nr=None, debug=True):
from yamtbx.dataproc import eiger
import h5py
master_h5 = self.get_master_h5_if_exists()
if master_h5: # XXX check only for nr?
if nr:
frames_requested = set(range(nr[0], nr[1] + 1))
frames_available = set(
eiger.get_available_frame_numbers(master_h5))
return frames_requested.issubset(frames_available)
else:
if debug:
print "Checking if related files exist for %s" % master_h5
try:
files = eiger.get_masterh5_related_filenames(master_h5)
except:
if debug:
print "Error when reading master h5 (%s)" % master_h5
print traceback.format_exc()
return False
flag_ng = False
for f in files:
if os.path.isfile(f):
try:
h5py.File(f, "r")
except:
if debug:
print " file incomplete or broken: %s" % f
flag_ng = True
else:
if debug: print " not exists: %s" % f
flag_ng = True
return not flag_ng
else:
if nr:
filenames = template_to_filenames(self.filename, nr[0], nr[1])
return all(map(lambda x: os.path.exists(x), filenames))
else:
return True # XXX
def run(params):
xds_inp = os.path.join(params.xds_dir, "XDS.INP")
spot_xds = os.path.join(params.xds_dir, "SPOT.XDS")
spots = idxreflp.SpotXds(
spot_xds).indexed_and_unindexed_by_frame_on_detector()
inputs = get_xdsinp_keyword(xds_inp)
filename_template = dict(inputs).get("NAME_TEMPLATE_OF_DATA_FRAMES", "")
if filename_template == "":
print "Error! Can't find filename from XDS.INP"
return
# Start adxv
adxv = Adxv(params.adxv_bin)
adxv.start(params.xds_dir)
type_indexed = adxv.define_spot("blue")
type_unindexed = adxv.define_spot("red")
num = params.image
while True:
print "Showing image %d" % num
img_file = dataset.template_to_filenames(filename_template, num,
num)[0]
adxv.open_image(img_file)
uninds = map(lambda x: [x[0], x[1], type_unindexed],
spots["unindexed"].get(num, []))
inds = map(lambda x: [x[0], x[1], type_indexed],
spots["indexed"].get(num, []))
print "Showing %d Indexed spots (blue)" % len(inds)
print "Showing %d Unindexed spots (red)" % len(uninds)
adxv.load_spots(inds + uninds)
time.sleep(1) # wait until adxv finishes process..
num = int(raw_input("Next image number?: "))
def run(params):
xds_inp = os.path.join(params.xds_dir, "XDS.INP")
spot_xds = os.path.join(params.xds_dir, "SPOT.XDS")
spots = idxreflp.SpotXds(spot_xds).indexed_and_unindexed_by_frame_on_detector()
inputs = get_xdsinp_keyword(xds_inp)
filename_template = dict(inputs).get("NAME_TEMPLATE_OF_DATA_FRAMES", "")
if filename_template == "":
print "Error! Can't find filename from XDS.INP"
return
# Start adxv
adxv = Adxv(params.adxv_bin)
adxv.start(params.xds_dir)
type_indexed = adxv.define_spot("blue")
type_unindexed = adxv.define_spot("red")
num = params.image
while True:
print "Showing image %d" % num
img_file = dataset.template_to_filenames(filename_template, num, num)[0]
adxv.open_image(img_file)
uninds = map(lambda x: [x[0], x[1], type_unindexed], spots["unindexed"].get(num, []))
inds = map(lambda x: [x[0], x[1], type_indexed], spots["indexed"].get(num, []))
print "Showing %d Indexed spots (blue)" % len(inds)
print "Showing %d Unindexed spots (red)" % len(uninds)
adxv.load_spots(inds+uninds)
time.sleep(1) # wait until adxv finishes process..
num = int(raw_input("Next image number?: "))
def parse(self):
re_scanstart = re.compile("Diffraction scan \(([0-9A-Za-z/:\[\] ]+)\)")
re_nums = re.compile("([0-9]+) +([\-0-9\.]+|dummy) +([\-0-9\.]+|dummy) +([\-0-9\.]+|dummy)")
re_point_step = re.compile("point: *([0-9]+) *step: *([0-9\.]+)")
re_osc = re.compile("Oscillation start: ([-0-9\.]+) \[deg\], step: ([-0-9\.]+) \[deg\]")
re_att = re.compile("Attenuator: +([^ ]+) +([0-9]+)um")
re_exp = re.compile("Exp\. time: ([\.0-9]+) ")
re_beam = re.compile("hor\. beam size: +([\.0-9]+)\[um\], ver\. beamsize: +([\.0-9]+)\[um\]") # old bss, wrong (need to swap h/v)
re_beam2 = re.compile("horizontal size: +([\.0-9]+)\[um\], vertical size: +([\.0-9]+)\[um\]") # new bss (2015-Apr), correct
re_fixed_spindle = re.compile("Fixed spindle angle: ([-0-9\.]+) \[deg\]")
re_frame_rate = re.compile("Frame rate: ([-0-9\.]+) \[frame/s\]")
self.scans = []
self.filename_gonio_gc = OrderedDict()
for l in open(self.scanlog):
r_scanstart = re_scanstart.search(l)
if r_scanstart:
datestr = re.sub("\[[A-Za-z]+\] ", "", r_scanstart.group(1))
if datestr.count(":") == 3: datestr = datestr[:datestr.rindex(":")] # from bss jan29-2015, millisec time is recorded; but we discard it here.
date = datetime.datetime.strptime(datestr, "%Y/%m/%d %H:%M:%S")
self.scans.append(ScanInfo())
self.scans[-1].date = date
continue
r_osc = re_osc.search(l)
if r_osc:
self.scans[-1].osc_start = float(r_osc.group(1))
self.scans[-1].osc_step = float(r_osc.group(2))
continue
r_exp = re_exp.search(l)
if r_exp:
self.scans[-1].exp_time = float(r_exp.group(1))
continue
r_beam = re_beam.search(l)
if r_beam:
self.scans[-1].beam_hsize = float(r_beam.group(2))
self.scans[-1].beam_vsize = float(r_beam.group(1))
continue
r_beam2 = re_beam2.search(l)
if r_beam2:
self.scans[-1].beam_hsize = float(r_beam2.group(1))
self.scans[-1].beam_vsize = float(r_beam2.group(2))
continue
r_fspindle = re_fixed_spindle.search(l)
if r_fspindle:
self.scans[-1].fixed_spindle = float(r_fspindle.group(1))
continue
r_frate = re_frame_rate.search(l)
if r_frate:
self.scans[-1].frame_rate = float(r_frate.group(1))
continue
if "No dummy image generated in each scan" in l:
# this scan does not have dummy images.
self.scans[-1].has_extra_images = False
self.scans[-1].need_treatment_on_image_numbers = False
continue
if "Dummy images generated in each scan!" in l:
# this scan actually has dummy images but diffscan.log has "dummy" lines
self.scans[-1].has_extra_images = True
self.scans[-1].need_treatment_on_image_numbers = False
continue
if "Scan direction:" in l:
self.scans[-1].scan_direction = l[l.index(":")+1:].strip()
continue
if "Scan path:" in l:
self.scans[-1].scan_path = l[l.index(":")+1:].strip()
continue
if "Wavelength: " in l:
self.scans[-1].wavelength = float(l.strip().split()[1])
continue
if "Attenuator:" in l:
r_att = re_att.search(l)
if r_att:
self.scans[-1].attenuator = (r_att.group(1), int(r_att.group(2)))
continue
if "Cameralength: " in l:
self.scans[-1].distance = float(l.strip().split()[1])
continue
if "FILE_NAME = " in l:
self.scans[-1].filename_template = os.path.basename(l[l.index("FILE_NAME = ")+12:].strip())
continue
if "Vertical scan" in l:
vpoint, vstep = re_point_step.search(l.strip()).groups()
self.scans[-1].vpoints, self.scans[-1].vstep = int(vpoint), float(vstep)
continue
if "Horizontal scan" in l:
hpoint, hstep = re_point_step.search(l.strip()).groups()
self.scans[-1].hpoints, self.scans[-1].hstep = int(hpoint), float(hstep)
continue
r = re_nums.search(l)
if r:
if "dummy" in r.groups():
assert not self.scans[-1].need_treatment_on_image_numbers
continue
else:
gonio = tuple(map(lambda x:float(x), r.groups()[1:]))
num = int(r.group(1))
if self.scans[-1].need_treatment_on_image_numbers and self.scans[-1].is_shutterless() and self.scans[-1].hpoints > 1:
# This if section is not needed *if* BSS no longer creates such non-sense files.
# this should be an option.
num += int(math.ceil(float(num)/self.scans[-1].hpoints)) - 1
filename = template_to_filenames(self.scans[-1].filename_template, num, num)[0]
grid_coord = self.get_grid_coord_internal(self.scans[-1].vpoints, self.scans[-1].vstep,
self.scans[-1].hpoints, self.scans[-1].hstep,
num, self.scans[-1].is_shutterless() and self.scans[-1].has_extra_images,
self.scans[-1].scan_direction, self.scans[-1].scan_path)
# XXX Need to check if the filename is already registered (file is overwritten!!)
self.scans[-1].filename_coords.append((filename, (gonio, grid_coord)))
self.filename_gonio_gc[os.path.basename(filename)] = (gonio, grid_coord)
"""
from yamtbx.dataproc import XIO
from yamtbx.dataproc.dataset import template_to_filenames, find_data_sets
def timestamps(img_files):
img_date = []
for f in img_files:
im = XIO.Image(f)
img_date.append((f, im.header["DateSeconds"]))
for i in xrange(len(img_files)):
print img_files[i],
if i == 0:
print 0
else:
print img_date[i][1] - img_date[i-1][1]
if __name__ == "__main__":
import sys
if len(sys.argv) > 1:
wdir = sys.argv[1]
else:
wdir = os.getcwd()
for img_template, min_frame, max_frame in find_data_sets(wdir):
img_files = template_to_filenames(img_template, min_frame, max_frame)
timestamps(img_files)
def parse(self):
re_scanstart = re.compile("Diffraction scan \(([0-9A-Za-z/:\[\] ]+)\)")
re_nums = re.compile("([0-9]+) +([\-0-9\.]+|dummy) +([\-0-9\.]+|dummy) +([\-0-9\.]+|dummy)")
re_point_step = re.compile("point: *([0-9]+) *step: *([0-9\.]+)")
re_osc = re.compile("Oscillation start: ([-0-9\.]+) \[deg\], step: ([-0-9\.]+) \[deg\]")
re_att = re.compile("Attenuator: +([^ ]+) +([0-9]+)um")
re_att2 = re.compile("Attenuator transmission: +([^ ]+) +\(([^ ]+) attenuator: ([0-9]+)\[um\]\)")
re_exp = re.compile("Exp\. time: ([\.0-9]+) ")
re_beam = re.compile("hor\. beam size: +([\.0-9]+)\[um\], ver\. beamsize: +([\.0-9]+)\[um\]") # old bss, wrong (need to swap h/v)
re_beam2 = re.compile("horizontal size: +([\.0-9]+)\[um\], vertical size: +([\.0-9]+)\[um\]") # new bss (2015-Apr), correct
re_fixed_spindle = re.compile("Fixed spindle angle: ([-0-9\.]+) \[deg\]")
re_frame_rate = re.compile("Frame rate: ([-0-9\.]+) \[frame/s\]")
self.scans = []
self.filename_gonio_gc = OrderedDict()
for l in open(self.scanlog):
r_scanstart = re_scanstart.search(l)
if r_scanstart:
datestr = re.sub("\[[A-Za-z]+\] ", "", r_scanstart.group(1))
if datestr.count(":") == 3: datestr = datestr[:datestr.rindex(":")] # from bss jan29-2015, millisec time is recorded; but we discard it here.
date = datetime.datetime.strptime(datestr, "%Y/%m/%d %H:%M:%S")
self.scans.append(ScanInfo())
self.scans[-1].date = date
continue
r_osc = re_osc.search(l)
if r_osc:
self.scans[-1].osc_start = float(r_osc.group(1))
self.scans[-1].osc_step = float(r_osc.group(2))
continue
r_exp = re_exp.search(l)
if r_exp:
self.scans[-1].exp_time = float(r_exp.group(1))
continue
r_beam = re_beam.search(l)
if r_beam:
self.scans[-1].beam_hsize = float(r_beam.group(2))
self.scans[-1].beam_vsize = float(r_beam.group(1))
continue
r_beam2 = re_beam2.search(l)
if r_beam2:
self.scans[-1].beam_hsize = float(r_beam2.group(1))
self.scans[-1].beam_vsize = float(r_beam2.group(2))
continue
r_fspindle = re_fixed_spindle.search(l)
if r_fspindle:
self.scans[-1].fixed_spindle = float(r_fspindle.group(1))
continue
r_frate = re_frame_rate.search(l)
if r_frate:
self.scans[-1].frame_rate = float(r_frate.group(1))
continue
if "No dummy image generated in each scan" in l:
# this scan does not have dummy images.
self.scans[-1].has_extra_images = False
self.scans[-1].need_treatment_on_image_numbers = False
continue
if "Dummy images generated in each scan!" in l:
# this scan actually has dummy images but diffscan.log has "dummy" lines
self.scans[-1].has_extra_images = True
self.scans[-1].need_treatment_on_image_numbers = False
continue
if "Scan direction:" in l:
self.scans[-1].scan_direction = l[l.index(":")+1:].strip()
continue
if "Scan path:" in l:
self.scans[-1].scan_path = l[l.index(":")+1:].strip()
continue
if "Wavelength: " in l:
self.scans[-1].wavelength = float(l.strip().split()[1])
continue
if "Attenuator:" in l:
r_att = re_att.search(l)
r_att2 = re_att2.search(l)
if r_att:
self.scans[-1].attenuator = (r_att.group(1), int(r_att.group(2)))
elif r_att2:
self.scans[-1].attenuator = (r_att2.group(2), int(r_att2.group(3))) # (1) is transmisttance
continue
if "Cameralength: " in l:
self.scans[-1].distance = float(l.strip().split()[1])
continue
if "FILE_NAME = " in l:
self.scans[-1].filename_template = os.path.basename(l[l.index("FILE_NAME = ")+12:].strip())
# VERY DIRTY FIX!
# Explanation: if diffscan.log contains ".h5", we assume it's Eiger hdf5 and hit-finding is done by streaming mode.
if self.scans[-1].filename_template.endswith(".h5"):
self.scans[-1].filename_template = self.scans[-1].filename_template[:-3] + ".img"
continue
if "Vertical scan" in l:
vpoint, vstep = re_point_step.search(l.strip()).groups()
self.scans[-1].vpoints, self.scans[-1].vstep = int(vpoint), float(vstep)
continue
if "Horizontal scan" in l:
hpoint, hstep = re_point_step.search(l.strip()).groups()
self.scans[-1].hpoints, self.scans[-1].hstep = int(hpoint), float(hstep)
continue
r = re_nums.search(l)
if r:
if "dummy" in r.groups():
assert not self.scans[-1].need_treatment_on_image_numbers
continue
else:
gonio = tuple(map(lambda x:float(x), r.groups()[1:]))
num = int(r.group(1))
if self.scans[-1].need_treatment_on_image_numbers and self.scans[-1].is_shutterless() and self.scans[-1].hpoints > 1:
# This if section is not needed *if* BSS no longer creates such non-sense files.
# this should be an option.
num += int(math.ceil(float(num)/self.scans[-1].hpoints)) - 1
filename = template_to_filenames(self.scans[-1].filename_template, num, num)[0]
grid_coord = self.get_grid_coord_internal(self.scans[-1].vpoints, self.scans[-1].vstep,
self.scans[-1].hpoints, self.scans[-1].hstep,
num, self.scans[-1].is_shutterless() and self.scans[-1].has_extra_images,
self.scans[-1].scan_direction, self.scans[-1].scan_path)
# XXX Need to check if the filename is already registered (file is overwritten!!)
self.scans[-1].filename_coords.append((filename, (gonio, grid_coord)))
self.scans[-1].filename_idxes.append((filename, num))
if len(l.split())==5: # for SACLA
self.scans[-1].filename_tags.append((filename, int(l.split()[-1])))
self.filename_gonio_gc[os.path.basename(filename)] = (gonio, grid_coord)
def import_xds_as_still(xdsinp, xparm_in):
#from dxtbx.serialize import xds
from dxtbx.datablock import DataBlockFactory
# Get the sweep from the XDS files
#sweep = xds.to_imageset(xds_inp, xds_other)
from iotbx.xds import xds_inp
from dxtbx.imageset import ImageSetFactory
import dxtbx
# Read the input filename
handle = xds_inp.reader()
handle.read_file(xdsinp)
# Get the template
template = handle.name_template_of_data_frames[0]
image_range = handle.data_range
detector_name = handle.detector
#assert image_range[0] == image_range[1]
im_nr = int((image_range[1] - image_range[0] + 1) / 2)
from yamtbx.dataproc.dataset import template_to_filenames
# Create the imageset
#imageset = ImageSetFactory.from_template(template, image_range=image_range, check_format=False)[0]
imageset = ImageSetFactory.make_imageset(
[os.path.realpath(template_to_filenames(template, im_nr, im_nr)[0])])
models = dxtbx.load(xparm_in)
detector = models.get_detector()
if detector_name.strip() in ('PILATUS',
'EIGER') or handle.silicon is not None:
from dxtbx.model import ParallaxCorrectedPxMmStrategy
from cctbx.eltbx import attenuation_coefficient
if handle.silicon is None:
table = attenuation_coefficient.get_table("Si")
wavelength = models.get_beam().get_wavelength()
mu = table.mu_at_angstrom(wavelength) / 10.0
else:
mu = handle.silicon
t0 = handle.sensor_thickness
for panel in detector:
panel.set_px_mm_strategy(ParallaxCorrectedPxMmStrategy(mu, t0))
panel.set_trusted_range(
(handle.minimum_valid_pixel_value, handle.overload))
imageset.set_beam(models.get_beam())
imageset.set_detector(detector)
imageset.set_goniometer(None)
imageset.set_scan(None)
#imageset.set_goniometer(models.get_goniometer())
# take the image range from XDS.INP
#scan = models.get_scan()
#scan.set_image_range(image_range)
#imageset.set_scan(scan)
from dxtbx.serialize import xds
# Get the crystal from the XDS files
crystal = xds.to_crystal(xparm_in)
# Create the experiment list
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
# Set the crystal in the experiment list
assert (len(experiments) == 1)
# Return the experiment list
return experiments
from yamtbx.dataproc import XIO
from yamtbx.dataproc.dataset import template_to_filenames, find_data_sets
def timestamps(img_files):
img_date = []
for f in img_files:
im = XIO.Image(f)
img_date.append((f, im.header["DateSeconds"]))
for i in xrange(len(img_files)):
print img_files[i],
if i == 0:
print 0
else:
print img_date[i][1] - img_date[i - 1][1]
if __name__ == "__main__":
import sys
if len(sys.argv) > 1:
wdir = sys.argv[1]
else:
wdir = os.getcwd()
for img_template, min_frame, max_frame in find_data_sets(wdir):
img_files = template_to_filenames(img_template, min_frame, max_frame)
timestamps(img_files)