def _find_spots(self, args=None): # type: (Optional[List[str]]) -> None
"""
Call `dials.find_spots` on the imported experiment list.
Args:
args (optional): List of any additional PHIL parameters to be used by
dials.import.
"""
info("\nFinding spots...")
dials_start = timeit.default_timer()
# Use some choice fillets from dials.find_spots
# Ignore `args`, use `self.params`
# Loop through all the imagesets and find the strong spots
self.refls = flex.reflection_table.from_observations(
self.expts, self.params.dials_find_spots)
# Add n_signal column - before deleting shoeboxes
good = MaskCode.Foreground | MaskCode.Valid
self.refls["n_signal"] = self.refls["shoebox"].count_mask_values(good)
# Delete the shoeboxes
if not self.params.dials_find_spots.output.shoeboxes:
del self.refls["shoebox"]
info(
60 * "-" + "\n%s\n" + 60 * "-" +
"\nSuccessfully completed (%.1f sec)",
spot_counts_per_image_plot(self.refls),
timeit.default_timer() - dials_start,
)
def _index_prepare(self):
logger.notice(banner("Spotfinding %s" % self.get_indexer_sweep_name()))
super(XDSIndexerII, self)._index_prepare()
reflections_file = spot_xds_to_reflection_file(
self._indxr_payload["SPOT.XDS"],
working_directory=self.get_working_directory(),
)
refl = flex.reflection_table.from_file(reflections_file)
logger.info(spot_counts_per_image_plot(refl))
def _index_prepare(self):
Chatter.banner('Spotfinding %s' % self.get_indexer_sweep_name())
super(XDSIndexerII, self)._index_prepare()
from dials.array_family import flex
from dials.util.ascii_art import spot_counts_per_image_plot
reflection_pickle = spot_xds_to_reflection_pickle(
self._indxr_payload['SPOT.XDS'],
working_directory=self.get_working_directory())
refl = flex.reflection_table.from_pickle(reflection_pickle)
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
def _index_prepare(self):
Chatter.banner('Spotfinding %s' %self.get_indexer_sweep_name())
super(XDSIndexerII, self)._index_prepare()
from dials.array_family import flex
from dials.util.ascii_art import spot_counts_per_image_plot
reflection_pickle = spot_xds_to_reflection_pickle(
self._indxr_payload['SPOT.XDS'],
working_directory=self.get_working_directory())
refl = flex.reflection_table.from_pickle(reflection_pickle)
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
def _find_spots(self, additional_parameters=None):
if additional_parameters is None:
additional_parameters = []
info("\nSpot finding...")
command = [ "dials.find_spots", self.json_file, "nproc=%s" % self.nproc ] + additional_parameters
result = run_process(command, print_stdout=False, debug=procrunner_debug)
debug("result = %s" % self._prettyprint_dictionary(result))
if result['exitcode'] != 0:
warn("Failed with exit code %d" % result['exitcode'])
sys.exit(1)
info(60 * '-')
from libtbx import easy_pickle
from dials.util.ascii_art import spot_counts_per_image_plot
refl = easy_pickle.load('strong.pickle')
info(spot_counts_per_image_plot(refl))
info(60 * '-')
info("Successfully completed (%.1f sec)" % result['runtime'])
def _index_prepare(self):
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
#all_images = self.get_matching_images()
#first = min(all_images)
#last = max(all_images)
spot_lists = []
datablocks = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
Chatter.banner('Spotfinding %s' % xsweep.get_name())
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
sweep_filename = os.path.join(
self.get_working_directory(),
'%s_datablock.json' % xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
genmask = self.GenerateMask()
genmask.set_input_datablock(sweep_filename)
genmask.set_output_datablock(
os.path.join(
self.get_working_directory(), '%s_%s_datablock.json' %
(genmask.get_xpid(), xsweep.get_name())))
genmask.set_params(PhilIndex.params.dials.masking)
sweep_filename, mask_pickle = genmask.run()
Debug.write('Generated mask for %s: %s' %
(xsweep.get_name(), mask_pickle))
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
Chatter.write('Estimated gain: %.2f' % gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
#offset = self.get_frame_offset()
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
'%s_%s_datablock.json' %
(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
'%s_%s_strong.pickle' %
(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if PhilIndex.params.dials.find_spots.phil_file is not None:
spotfinder.set_phil_file(
PhilIndex.params.dials.find_spots.phil_file)
min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size
if min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == 'SENSOR_PAD':
min_spot_size = 3
else:
min_spot_size = None
if min_spot_size is not None:
spotfinder.set_min_spot_size(min_spot_size)
min_local = PhilIndex.params.dials.find_spots.min_local
if min_local is not None:
spotfinder.set_min_local(min_local)
sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong
if sigma_strong:
spotfinder.set_sigma_strong(sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings
if filter_ice_rings:
spotfinder.set_filter_ice_rings(filter_ice_rings)
kernel_size = PhilIndex.params.dials.find_spots.kernel_size
if kernel_size:
spotfinder.set_kernel_size(kernel_size)
global_threshold = PhilIndex.params.dials.find_spots.global_threshold
if global_threshold is not None:
spotfinder.set_global_threshold(global_threshold)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist." %
os.path.basename(spot_filename))
spot_lists.append(spot_filename)
datablocks.append(spotfinder.get_output_sweep_filename())
from libtbx import easy_pickle
from dials.util.ascii_art import spot_counts_per_image_plot
refl = easy_pickle.load(spot_filename)
if not len(refl):
raise RuntimeError('No spots found in sweep %s' %
xsweep.get_name())
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(datablocks[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
offset = imageset.get_scan().get_image_range()[0]
blank_regions = json['strong']['blank_regions']
if len(blank_regions):
blank_regions = [(int(s), int(e))
for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
Chatter.write(
'WARNING: Potential blank images: %i -> %i' %
(blank_start + 1, blank_end))
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append(
(last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end + 1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.get_xsample()
sweep_name = xsweep.get_name()
import string
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start -
start:nb_end - start]
xsweep._frames_to_process = (nb_start + 1,
nb_end + 1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[
self._indxr_imagesets.index(
imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(
sub_imageset)
else:
new_name = '_'.join(
(sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(
new_name,
xsample,
directory=os.path.join(
os.path.basename(
xsweep.get_directory()), new_name),
image=imageset.get_path(nb_start - start),
frames_to_process=(nb_start + 1, nb_end),
)
Chatter.write(
"Generating new sweep: %s (%s:%i:%i)" %
(new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1]))
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.DiscoverBetterExperimentalModel()
discovery.set_sweep_filename(datablocks[-1])
discovery.set_spot_filename(spot_filename)
#wedges = self._index_select_images_i(imageset)
#discovery.set_scan_ranges(wedges)
#discovery.set_scan_ranges([(first + offset, last + offset)])
try:
discovery.run()
except Exception as e:
Debug.write('DIALS beam centre search failed: %s' % str(e))
else:
# overwrite datablock.json in datablocks list
datablocks[
-1] = discovery.get_optimized_datablock_filename()
self.set_indexer_payload("spot_lists", spot_lists)
self.set_indexer_payload("datablocks", datablocks)
return
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_datablocks
from time import time
from dials.util import log
from libtbx.utils import Sorry
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Ensure we have a data block
datablocks = flatten_datablocks(params.input.datablock)
if len(datablocks) == 0:
self.parser.print_help()
return
elif len(datablocks) != 1:
raise Sorry('only 1 datablock can be processed at a time')
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table.from_observations(
datablocks[0], params)
# Delete the shoeboxes
if not params.output.shoeboxes:
del reflections['shoebox']
# ascii spot count per image plot
from dials.util.ascii_art import spot_counts_per_image_plot
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
ascii_plot = spot_counts_per_image_plot(
reflections.select(reflections['id'] == i))
if len(ascii_plot):
logger.info(
'\nHistogram of per-image spot count for imageset %i:' % i)
logger.info(ascii_plot)
# Save the reflections to file
logger.info('\n' + '-' * 80)
reflections.as_pickle(params.output.reflections)
logger.info('Saved {0} reflections to {1}'.format(
len(reflections), params.output.reflections))
# Save the datablock
if params.output.datablock:
from dxtbx.datablock import DataBlockDumper
logger.info('Saving datablocks to {0}'.format(
params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
# Print some per image statistics
if params.per_image_statistics:
from dials.algorithms.spot_finding import per_image_analysis
from cStringIO import StringIO
s = StringIO()
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
print >> s, "Number of centroids per image for imageset %i:" % i
stats = per_image_analysis.stats_imageset(
imageset,
reflections.select(reflections['id'] == i),
resolution_analysis=False)
per_image_analysis.print_table(stats, out=s)
logger.info(s.getvalue())
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
def run(self, args=None):
"""Execute the script."""
from dials.array_family import flex
from dials.util.options import flatten_experiments
from time import time
from dials.util import log
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(args=args,
show_diff_phil=False)
if __name__ == "__main__":
# Configure the logging
log.config(verbosity=options.verbose, logfile=params.output.log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Ensure we have a data block
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table.from_observations(
experiments, params)
# Add n_signal column - before deleting shoeboxes
from dials.algorithms.shoebox import MaskCode
good = MaskCode.Foreground | MaskCode.Valid
reflections["n_signal"] = reflections["shoebox"].count_mask_values(
good)
# Delete the shoeboxes
if not params.output.shoeboxes:
del reflections["shoebox"]
# ascii spot count per image plot
from dials.util.ascii_art import spot_counts_per_image_plot
for i, experiment in enumerate(experiments):
ascii_plot = spot_counts_per_image_plot(
reflections.select(reflections["id"] == i))
if len(ascii_plot):
logger.info(
"\nHistogram of per-image spot count for imageset %i:" % i)
logger.info(ascii_plot)
# Save the reflections to file
logger.info("\n" + "-" * 80)
reflections.as_file(params.output.reflections)
logger.info("Saved {} reflections to {}".format(
len(reflections), params.output.reflections))
# Save the experiments
if params.output.experiments:
logger.info("Saving experiments to {}".format(
params.output.experiments))
experiments.as_file(params.output.experiments)
# Print some per image statistics
if params.per_image_statistics:
from dials.algorithms.spot_finding import per_image_analysis
from six.moves import cStringIO as StringIO
s = StringIO()
for i, experiment in enumerate(experiments):
print("Number of centroids per image for imageset %i:" % i,
file=s)
imageset = experiment.imageset
stats = per_image_analysis.stats_imageset(
imageset,
reflections.select(reflections["id"] == i),
resolution_analysis=False,
)
per_image_analysis.print_table(stats, out=s)
logger.info(s.getvalue())
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
if params.output.experiments:
return experiments, reflections
else:
return reflections
def _index_prepare(self):
Citations.cite("dials")
# all_images = self.get_matching_images()
# first = min(all_images)
# last = max(all_images)
spot_lists = []
experiments_filenames = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
logger.notice(banner("Spotfinding %s" % xsweep.get_name()))
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.model.experiment_list import ExperimentListFactory
sweep_filename = os.path.join(self.get_working_directory(),
"%s_import.expt" % xsweep.get_name())
ExperimentListFactory.from_imageset_and_crystal(
imageset, None).as_file(sweep_filename)
genmask = self.GenerateMask()
genmask.set_input_experiments(sweep_filename)
genmask.set_output_experiments(
os.path.join(
self.get_working_directory(),
"%s_%s_masked.expt" %
(genmask.get_xpid(), xsweep.get_name()),
))
genmask.set_params(PhilIndex.params.dials.masking)
sweep_filename, mask_pickle = genmask.run()
logger.debug("Generated mask for %s: %s", xsweep.get_name(),
mask_pickle)
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
logger.info("Estimated gain: %.2f", gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
# offset = self.get_frame_offset()
dfs_params = PhilIndex.params.dials.find_spots
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
"%s_%s_strong.expt" %
(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
"%s_%s_strong.refl" %
(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if dfs_params.phil_file is not None:
spotfinder.set_phil_file(dfs_params.phil_file)
if dfs_params.min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == "SENSOR_PAD":
dfs_params.min_spot_size = 3
else:
dfs_params.min_spot_size = None
if dfs_params.min_spot_size is not None:
spotfinder.set_min_spot_size(dfs_params.min_spot_size)
if dfs_params.min_local is not None:
spotfinder.set_min_local(dfs_params.min_local)
if dfs_params.sigma_strong:
spotfinder.set_sigma_strong(dfs_params.sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
if dfs_params.filter_ice_rings:
spotfinder.set_filter_ice_rings(dfs_params.filter_ice_rings)
if dfs_params.kernel_size:
spotfinder.set_kernel_size(dfs_params.kernel_size)
if dfs_params.global_threshold is not None:
spotfinder.set_global_threshold(dfs_params.global_threshold)
if dfs_params.threshold.algorithm is not None:
spotfinder.set_threshold_algorithm(
dfs_params.threshold.algorithm)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist." %
os.path.basename(spot_filename))
spot_lists.append(spot_filename)
experiments_filenames.append(
spotfinder.get_output_sweep_filename())
refl = flex.reflection_table.from_file(spot_filename)
if not len(refl):
raise RuntimeError("No spots found in sweep %s" %
xsweep.get_name())
logger.info(spot_counts_per_image_plot(refl))
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(experiments_filenames[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
blank_regions = json["strong"]["blank_regions"]
if len(blank_regions):
blank_regions = [(int(s), int(e))
for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
logger.info(
"WARNING: Potential blank images: %i -> %i",
blank_start + 1,
blank_end,
)
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append(
(last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end + 1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.sample
sweep_name = xsweep.get_name()
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start -
start:nb_end - start]
xsweep._frames_to_process = (nb_start + 1,
nb_end + 1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[
self._indxr_imagesets.index(
imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(
sub_imageset)
else:
min_images = (PhilIndex.params.xia2.settings.
input.min_images)
if (nb_end - nb_start) < min_images:
continue
new_name = "_".join(
(sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(
new_name,
xsample,
directory=os.path.join(
os.path.basename(
xsweep.get_directory()),
new_name,
),
image=imageset.get_path(nb_start - start),
frames_to_process=(nb_start + 1, nb_end),
beam=xsweep.get_beam_centre(),
reversephi=xsweep.get_reversephi(),
distance=xsweep.get_distance(),
gain=xsweep.get_gain(),
dmin=xsweep.get_resolution_high(),
dmax=xsweep.get_resolution_low(),
polarization=xsweep.get_polarization(),
user_lattice=xsweep.get_user_lattice(),
user_cell=xsweep.get_user_cell(),
epoch=xsweep._epoch,
ice=xsweep._ice,
excluded_regions=xsweep._excluded_regions,
)
logger.info(
"Generating new sweep: %s (%s:%i:%i)",
new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1],
)
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.SearchBeamPosition()
discovery.set_sweep_filename(experiments_filenames[-1])
discovery.set_spot_filename(spot_filename)
# set scan_range to correspond to not more than 180 degrees
# if we have > 20000 reflections
width = imageset.get_scan().get_oscillation()[1]
if (last - first) * width > 180.0 and len(refl) > 20000:
end = first + int(round(180.0 / width)) - 1
logger.debug("Using %d to %d for beam search", first, end)
discovery.set_image_range((first, end))
try:
discovery.run()
result = discovery.get_optimized_experiments_filename()
# overwrite indexed.expt in experiments list
experiments_filenames[-1] = result
except Exception as e:
logger.debug("DIALS beam centre search failed: %s",
str(e),
exc_info=True)
self.set_indexer_payload("spot_lists", spot_lists)
self.set_indexer_payload("experiments", experiments_filenames)
def run(self, args=None):
"""Execute the script."""
# Parse the command line
params, options = self.parser.parse_args(args=args,
show_diff_phil=False)
if __name__ == "__main__":
# Configure the logging
log.config(verbosity=options.verbose, logfile=params.output.log)
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Ensure we have a data block
experiments = flatten_experiments(params.input.experiments)
# did input have identifier?
had_identifiers = False
if all(i != "" for i in experiments.identifiers()):
had_identifiers = True
else:
generate_experiment_identifiers(
experiments
) # add identifier e.g. if coming straight from images
if len(experiments) == 0:
self.parser.print_help()
return
# If maximum_trusted_value assigned, use this temporarily for the
# spot finding
if params.maximum_trusted_value is not None:
logger.info("Overriding maximum trusted value to %.1f",
params.maximum_trusted_value)
input_trusted_ranges = {}
for _d, detector in enumerate(experiments.detectors()):
for _p, panel in enumerate(detector):
trusted = panel.get_trusted_range()
input_trusted_ranges[(_d, _p)] = trusted
panel.set_trusted_range(
(trusted[0], params.maximum_trusted_value))
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table.from_observations(
experiments, params)
# Add n_signal column - before deleting shoeboxes
good = MaskCode.Foreground | MaskCode.Valid
reflections["n_signal"] = reflections["shoebox"].count_mask_values(
good)
# Delete the shoeboxes
if not params.output.shoeboxes:
del reflections["shoebox"]
# ascii spot count per image plot - per imageset
imagesets = []
for i, experiment in enumerate(experiments):
if experiment.imageset not in imagesets:
imagesets.append(experiment.imageset)
for imageset in imagesets:
selected = flex.bool(reflections.nrows(), False)
for i, experiment in enumerate(experiments):
if experiment.imageset is not imageset:
continue
selected.set_selected(reflections["id"] == i, True)
ascii_plot = spot_counts_per_image_plot(
reflections.select(selected))
if len(ascii_plot):
logger.info(
"\nHistogram of per-image spot count for imageset %i:" % i)
logger.info(ascii_plot)
# Save the reflections to file
logger.info("\n" + "-" * 80)
# If started with images and not saving experiments, then remove id mapping
# as the experiment linked to will no longer exists after exit.
if not had_identifiers:
if not params.output.experiments:
for k in reflections.experiment_identifiers().keys():
del reflections.experiment_identifiers()[k]
reflections.as_file(params.output.reflections)
logger.info("Saved {} reflections to {}".format(
len(reflections), params.output.reflections))
# Reset the trusted ranges
if params.maximum_trusted_value is not None:
for _d, detector in enumerate(experiments.detectors()):
for _p, panel in enumerate(detector):
trusted = input_trusted_ranges[(_d, _p)]
panel.set_trusted_range(trusted)
# Save the experiments
if params.output.experiments:
logger.info("Saving experiments to {}".format(
params.output.experiments))
experiments.as_file(params.output.experiments)
# Print some per image statistics
if params.per_image_statistics:
for i, experiment in enumerate(experiments):
logger.info("Number of centroids per image for imageset %i:",
i)
refl = reflections.select(reflections["id"] == i)
refl.centroid_px_to_mm([experiment])
refl.map_centroids_to_reciprocal_space([experiment])
stats = per_image_analysis.stats_per_image(
experiment, refl, resolution_analysis=False)
logger.info(str(stats))
if params.output.experiments:
return experiments, reflections
else:
return reflections
def test_spot_counts_per_image_plot(dials_regression):
from libtbx import easy_pickle
from dials.util import ascii_art
data_dir = os.path.join(dials_regression, "indexing_test_data",
"i04_weak_data")
pickle_path = os.path.join(data_dir, "full.pickle")
refl = easy_pickle.load(pickle_path)
output = ascii_art.spot_counts_per_image_plot(refl)
expected_output = """\
116082 spots found on 540 images (max 2075 / bin)
* * * * * * * * ** *
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
1 image 540"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
output = ascii_art.spot_counts_per_image_plot(refl,
char="o",
width=80,
height=15)
expected_output = """\
116082 spots found on 540 images (max 1627 / bin)
o
o oo o o o o o o o o o o o o o o o oo oo o o o o o o oo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
1 image 540"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
output = ascii_art.spot_counts_per_image_plot(refl,
char="#",
width=7,
height=10)
expected_output = """\
116082 spots found on 540 images (max 16752 / bin)
#######
#######
#######
#######
#######
#######
#######
#######
#######
#######
1 540"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
output = ascii_art.spot_counts_per_image_plot(
refl.select(refl["xyzobs.px.value"].parts()[2] <= 9.5),
char="#",
width=10,
height=15,
)
expected_output = """\
2065 spots found on 10 images (max 361 / bin)
#
#
#
#
#
# #
# # #
## ## ##
##########
##########
##########
##########
##########
##########
##########
1 image 10"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_datablocks
from time import time
from dials.util import log
from libtbx.utils import Sorry
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Ensure we have a data block
datablocks = flatten_datablocks(params.input.datablock)
if len(datablocks) == 0:
self.parser.print_help()
return
elif len(datablocks) != 1:
raise Sorry('only 1 datablock can be processed at a time')
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table.from_observations(
datablocks[0], params)
# Delete the shoeboxes
if not params.output.shoeboxes:
del reflections['shoebox']
# ascii spot count per image plot
from dials.util.ascii_art import spot_counts_per_image_plot
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
ascii_plot = spot_counts_per_image_plot(
reflections.select(reflections['id'] == i))
if len(ascii_plot):
logger.info('\nHistogram of per-image spot count for imageset %i:' %i)
logger.info(ascii_plot)
# Save the reflections to file
logger.info('\n' + '-' * 80)
reflections.as_pickle(params.output.reflections)
logger.info('Saved {0} reflections to {1}'.format(
len(reflections), params.output.reflections))
# Save the datablock
if params.output.datablock:
from dxtbx.datablock import DataBlockDumper
logger.info('Saving datablocks to {0}'.format(
params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
# Print some per image statistics
if params.per_image_statistics:
from dials.algorithms.spot_finding import per_image_analysis
from cStringIO import StringIO
s = StringIO()
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
print >> s, "Number of centroids per image for imageset %i:" %i
stats = per_image_analysis.stats_imageset(
imageset, reflections.select(reflections['id'] == i),
resolution_analysis=False)
per_image_analysis.print_table(stats, out=s)
logger.info(s.getvalue())
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
def test_spot_counts_per_image_plot(dials_regression):
from dials.util import ascii_art
data_dir = os.path.join(dials_regression, "indexing_test_data",
"i04_weak_data")
pickle_path = os.path.join(data_dir, "full.pickle")
with open(pickle_path, "rb") as fh:
refl = pickle.load(fh, encoding="bytes")
output = ascii_art.spot_counts_per_image_plot(refl)
expected_output = """\
116082 spots found on 540 images (max 2075 / bin)
* * * * * * * * ** *
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
1 image 540"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
output = ascii_art.spot_counts_per_image_plot(refl,
char="o",
width=80,
height=15)
expected_output = """\
116082 spots found on 540 images (max 1627 / bin)
o
o oo o o o o o o o o o o o o o o o oo oo o o o o o o oo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
1 image 540"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
output = ascii_art.spot_counts_per_image_plot(refl,
char="#",
width=7,
height=10)
expected_output = """\
116082 spots found on 540 images (max 16752 / bin)
#######
#######
#######
#######
#######
#######
#######
#######
#######
#######
1 540"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
output = ascii_art.spot_counts_per_image_plot(
refl.select(refl["xyzobs.px.value"].parts()[2] <= 9.5),
char="#",
width=10,
height=15,
)
expected_output = """\
2065 spots found on 10 images (max 361 / bin)
#
#
#
#
#
# #
# # #
## ## ##
##########
##########
##########
##########
##########
##########
##########
1 image 10"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
# If we do spotfinding on the first 3 images, then the spot centroids will be in the
# range z=0.5 to z=2.5
output = ascii_art.spot_counts_per_image_plot(
refl.select(refl["xyzobs.px.value"].parts()[2] <= 2.5),
char="#",
width=10,
height=15,
)
expected_output = """\
624 spots found on 3 images (max 294 / bin)
#
#
#
#
#
#
##
###
###
###
###
###
###
###
###
1 3"""
output = "\n".join(line.rstrip() for line in output.splitlines())
assert output == expected_output
def exercise_spot_counts_per_image_plot():
from libtbx import easy_pickle
from dials.util import ascii_art
from libtbx.test_utils import show_diff
if not have_dials_regression:
print 'Skipping exercise_spot_counts_per_image_plot(): dials_regression not available'
return
data_dir = os.path.join(dials_regression, "indexing_test_data",
"i04_weak_data")
pickle_path = os.path.join(data_dir, "full.pickle")
refl = easy_pickle.load(pickle_path)
output = ascii_art.spot_counts_per_image_plot(refl)
expected_output = '''\
116082 spots found on 540 images (max 2075 / bin)
* * * * * * * * ** *
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
1 image 540'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
output = ascii_art.spot_counts_per_image_plot(refl,
char='o',
width=80,
height=15)
expected_output = '''\
116082 spots found on 540 images (max 1627 / bin)
o
o oo o o o o o o o o o o o o o o o oo oo o o o o o o oo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
1 image 540'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
output = ascii_art.spot_counts_per_image_plot(refl,
char='#',
width=7,
height=10)
expected_output = '''\
116082 spots found on 540 images (max 16752 / bin)
#######
#######
#######
#######
#######
#######
#######
#######
#######
#######
1 540'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
output = ascii_art.spot_counts_per_image_plot(
refl.select(refl['xyzobs.px.value'].parts()[2] <= 9.5),
char='#',
width=10,
height=15)
expected_output = '''\
2065 spots found on 10 images (max 361 / bin)
#
#
#
#
#
# #
# # #
## ## ##
##########
##########
##########
##########
##########
##########
##########
1 image 10'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
def exercise_spot_counts_per_image_plot():
from libtbx import easy_pickle
from dials.util import ascii_art
from libtbx.test_utils import show_diff
if not have_dials_regression:
print 'Skipping exercise_spot_counts_per_image_plot(): dials_regression not available'
return
data_dir = os.path.join(dials_regression, "indexing_test_data", "i04_weak_data")
pickle_path = os.path.join(data_dir, "full.pickle")
refl = easy_pickle.load(pickle_path)
output = ascii_art.spot_counts_per_image_plot(refl)
expected_output = '''\
116082 spots found on 540 images (max 2082 / bin)
* * * * * * * * ** *
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
************************************************************
1 image 540'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
output = ascii_art.spot_counts_per_image_plot(
refl, char='o', width=80, height=15)
expected_output = '''\
116082 spots found on 540 images (max 1568 / bin)
o o o o o o o o o o o
o ooo ooo o o ooo ooo ooo ooo oooo oo ooo ooo ooo ooo ooooooo ooo ooo ooo ooo oo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo
1 image 540'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
output = ascii_art.spot_counts_per_image_plot(
refl, char='#', width=7, height=10)
expected_output = '''\
116082 spots found on 540 images (max 16765 / bin)
#######
#######
#######
#######
#######
#######
#######
#######
#######
#######
1 540'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
output = ascii_art.spot_counts_per_image_plot(
refl.select(refl['xyzobs.px.value'].parts()[2] <= 9.5), char='#', width=10, height=15)
expected_output = '''\
2065 spots found on 10 images (max 259 / bin)
#
#
### #
##### ##
#########
#########
##########
##########
##########
##########
##########
##########
##########
##########
##########
1 image 10'''
output = '\n'.join(line.rstrip() for line in output.splitlines())
assert not show_diff(output, expected_output)
def _index_prepare(self):
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
#all_images = self.get_matching_images()
#first = min(all_images)
#last = max(all_images)
spot_lists = []
datablocks = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
Chatter.banner('Spotfinding %s' %xsweep.get_name())
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
sweep_filename = os.path.join(
self.get_working_directory(), '%s_datablock.json' %xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
Chatter.write('Estimated gain: %.2f' %gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
#offset = self.get_frame_offset()
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
'%s_%s_datablock.json' %(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
'%s_%s_strong.pickle' %(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if PhilIndex.params.dials.find_spots.phil_file is not None:
spotfinder.set_phil_file(PhilIndex.params.dials.find_spots.phil_file)
min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size
if min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == 'SENSOR_PAD':
min_spot_size = 3
else:
min_spot_size = None
if min_spot_size is not None:
spotfinder.set_min_spot_size(min_spot_size)
min_local = PhilIndex.params.dials.find_spots.min_local
if min_local is not None:
spotfinder.set_min_local(min_local)
sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong
if sigma_strong:
spotfinder.set_sigma_strong(sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings
if filter_ice_rings:
spotfinder.set_filter_ice_rings(filter_ice_rings)
kernel_size = PhilIndex.params.dials.find_spots.kernel_size
if kernel_size:
spotfinder.set_kernel_size(kernel_size)
global_threshold = PhilIndex.params.dials.find_spots.global_threshold
if global_threshold is not None:
spotfinder.set_global_threshold(global_threshold)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist."
%os.path.basename(spot_filename))
spot_lists.append(spot_filename)
datablocks.append(spotfinder.get_output_sweep_filename())
from libtbx import easy_pickle
from dials.util.ascii_art import spot_counts_per_image_plot
refl = easy_pickle.load(spot_filename)
if not len(refl):
raise RuntimeError('No spots found in sweep %s' %xsweep.get_name())
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(datablocks[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
offset = imageset.get_scan().get_image_range()[0]
blank_regions = json['strong']['blank_regions']
if len(blank_regions):
blank_regions = [(int(s), int(e)) for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
Chatter.write('WARNING: Potential blank images: %i -> %i' %(
blank_start+1, blank_end))
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append((last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end+1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.get_xsample()
sweep_name = xsweep.get_name()
import string
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start-start:nb_end-start]
xsweep._frames_to_process = (nb_start+1, nb_end+1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[self._indxr_imagesets.index(imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(sub_imageset)
else:
new_name = '_'.join((sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(new_name,
xsample,
directory=os.path.join(
os.path.basename(xsweep.get_directory()), new_name),
image=imageset.get_path(nb_start-start),
frames_to_process=(nb_start+1, nb_end),
)
Chatter.write("Generating new sweep: %s (%s:%i:%i)" %(
new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1]))
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.DiscoverBetterExperimentalModel()
discovery.set_sweep_filename(datablocks[-1])
discovery.set_spot_filename(spot_filename)
#wedges = self._index_select_images_i(imageset)
#discovery.set_scan_ranges(wedges)
#discovery.set_scan_ranges([(first + offset, last + offset)])
try:
discovery.run()
except Exception, e:
Debug.write('DIALS beam centre search failed: %s' %str(e))
else:
# overwrite datablock.json in datablocks list
datablocks[-1] = discovery.get_optimized_datablock_filename()
