def run_benchmarks(files, dark_sample, num_runs):
"""Run benchmarks using the files given in the arguments
Example: "python benchmark_stem_image.py -d darksample.dat /path/to/data/data*.dat"
"""
if len(files) == 0:
sys.exit('No files found')
times = []
for i in range(num_runs):
# You can use a command like this to clear the caches, where
# `clearcaches` should be a bash script.
# subprocess.Popen('clearcaches').wait()
start = time.time()
reader = io.reader(dark_sample, version=io.FileVersion.VERSION2)
dark = image.calculate_average(reader)
reader = io.reader(files, version=io.FileVersion.VERSION2)
frame_events = image.electron_count(reader, dark, scan_width=40,
scan_height=40)
end = time.time()
times.append(end - start)
print('Run ' + str(len(times)) + ': {:0.2f} seconds'.format(times[-1]))
print('Number of runs was:', num_runs)
print('Average time: {:0.2f} seconds'.format(sum(times) / num_runs))
def main(input_path, output_path, scan_num, threshold):
if input_path is None:
input_path = Path(f'/mnt/nvmedata1/temp/data_scan{scan_num}.h5')
if output_path is None:
output_path = Path(f'/mnt/hdd1/data_scan{scan_num}_th{threshold}_electrons.h5')
print(output_path)
dark0 = np.zeros((576,576))
print('Opening: {}'.format(input_path))
with h5py.File(input_path, 'r') as f0:
sReader = stio.reader(f0)
print('start counting')
t0 = time.time()
ee = stim.electron_count(sReader, dark0, number_of_samples=1200,
verbose=True,
xray_threshold_n_sigma=175,
background_threshold_n_sigma=threshold)
t1 = time.time()
print('total time = {}'.format(t1-t0))
ii = 0
while output_path.exists():
ii += 1
output_path = Path(output_path.stem + '_{:03d}'.format(ii))
print('Saving to {}'.format(output_path))
stio.save_electron_counts(str(output_path), ee)
def main(output_path, scan_num, threshold):
if output_path is None:
output_path = Path(f'/mnt/hdd1/data_scan{scan_num}_th{threshold}_electrons.h5')
scanName = f'data_scan{scan_num}_'
# Setup the data drives
drives = []
for ii in range(1,5):
drives.append( (Path('/mnt/nvmedata{}/'.format(ii))))
print('Looking for files in:')
for d in drives:
print(d)
dark0 = np.zeros((576,576))
iFiles = []
for drive in drives:
files = drive.glob(scanName + '*.data')
for f in files:
iFiles.append(str(f))
iFiles = sorted(iFiles)
# Electron count the data
sReader = stio.reader(iFiles,stio.FileVersion.VERSION5, backend='thread')
print('start counting')
t0 = time.time()
ee = stim.electron_count(sReader,dark0,number_of_samples=1200,
verbose=False,threshold_num_blocks=20,
xray_threshold_n_sigma=175,
background_threshold_n_sigma=threshold)
t1 = time.time()
print('total time = {}'.format(t1-t0))
ii = 0
while output_path.exists():
ii += 1
output_path = Path(output_path.stem + '_{:03d}'.format(ii))
print('Saving to {}'.format(output_path))
stio.save_electron_counts(str(output_path), ee)
if args.verbose:
print('Number of files = {}'.format(len(iFiles)))
# Electron count the data
if args.verbose:
#print('NOTE: Using file version 4')
print(f'backend = {backend}')
sReader = stio.reader(iFiles, stio.FileVersion.VERSION5, backend=backend)
if args.verbose:
print('start counting')
t0 = time.time()
ee = stim.electron_count(sReader,
dark0,
gain=gain0,
number_of_samples=1200,
verbose=args.verbose,
threshold_num_blocks=20,
xray_threshold_n_sigma=175,
background_threshold_n_sigma=threshold)
t1 = time.time()
full_time = t1 - t0
if args.verbose:
print('total time = {}'.format(full_time))
outPath = out_dir / Path('data_scan{}_id0000_electrons.h5'.format(scanNum))
ii = 0
# Test for existence of the file and change name instead of overwriting
if outPath.exists():
outPath2 = outPath
def make_stem_hdf5(files, dark_sample, width, height, inner_radius,
outer_radius, reader_version, dark_reader_version, save_raw,
zip_raw, output):
"""Make an HDF5 file containing a STEM image
Example: "python create_hdf5.py -d darksample.dat /path/to/data/data*.dat"
"""
if len(files) == 0:
sys.exit('No files found')
if reader_version == 1:
reader_version = io.FileVersion.VERSION1
elif reader_version == 2:
reader_version = io.FileVersion.VERSION2
else:
sys.exit('Unknown reader version:', reader_version)
if dark_reader_version == 1:
dark_reader_version = io.FileVersion.VERSION1
elif dark_reader_version == 2:
dark_reader_version = io.FileVersion.VERSION2
else:
sys.exit('Unknown dark reader version:', dark_reader_version)
scan_dimensions = (width, height)
if dark_sample:
reader = io.reader(dark_sample, version=dark_reader_version)
dark = image.calculate_average(reader)
else:
# Get the frame dimensions from a block header, and use zeros
# for the dark sample.
reader = io.reader(files, version=reader_version)
frame_dimensions = reader.read().header.frame_dimensions
dark = np.zeros(frame_dimensions)
reader = io.reader(files, version=reader_version)
data = image.electron_count(reader, dark, scan_dimensions=scan_dimensions)
frame_dimensions = data.frame_dimensions
inner_radii = [0, inner_radius]
outer_radii = [outer_radius, outer_radius]
names = ['bright', 'dark']
reader.reset()
imgs = image.create_stem_images(reader,
inner_radii,
outer_radii,
scan_dimensions=scan_dimensions)
io.save_electron_counts(output, data)
io.save_stem_images(output, imgs, names)
if save_raw:
reader.reset()
# In order to avoid two copies of the data, we must allocate
# space for the large numpy array first.
raw_data = np.zeros((np.prod(scan_dimensions), frame_dimensions[1],
frame_dimensions[0]),
dtype=np.uint16)
# Make sure the frames are sorted in order
for block in reader:
for i in range(len(block.header.image_numbers)):
num = block.header.image_numbers[i]
raw_data[num] = block.data[i]
io.save_raw_data(output, raw_data, zip_data=zip_raw)
def make_stem_hdf5(files, dark_sample, width, height, inner_radius,
outer_radius, reader_version, dark_reader_version, save_raw,
zip_raw, output):
"""Make an HDF5 file containing a STEM image
Example: "python create_hdf5.py -d darksample.dat /path/to/data/data*.dat"
"""
if len(files) == 0:
sys.exit('No files found')
if reader_version == 1:
reader_version = io.FileVersion.VERSION1
elif reader_version == 2:
reader_version = io.FileVersion.VERSION2
else:
sys.exit('Unknown reader version:', reader_version)
if dark_reader_version == 1:
dark_reader_version = io.FileVersion.VERSION1
elif dark_reader_version == 2:
dark_reader_version = io.FileVersion.VERSION2
else:
sys.exit('Unknown dark reader version:', dark_reader_version)
reader = io.reader(dark_sample, version=dark_reader_version)
dark = image.calculate_average(reader)
reader = io.reader(files, version=reader_version)
frame_events = image.electron_count(reader,
dark,
scan_width=width,
scan_height=height)
# Read one block in to get the detector frames
reader.reset()
block = reader.read()
detector_nx = block.header.frame_width
detector_ny = block.header.frame_height
inner_radii = [0, inner_radius]
outer_radii = [outer_radius, outer_radius]
names = ['bright', 'dark']
reader.reset()
imgs = image.create_stem_images(reader,
inner_radii,
outer_radii,
width=width,
height=height)
io.save_electron_counts(output, frame_events, width, height, detector_nx,
detector_ny)
io.save_stem_images(output, imgs, names)
if save_raw:
reader.reset()
blocks = [block for block in reader]
raw_data = np.concatenate([block.data for block in blocks])
io.save_raw_data(output, raw_data, zip_raw)
def main(files, dark_file, center, inner_radii, outer_radii, output_file,
generate_sparse):
center = center.split(',')
if len(center) != 2:
msg = 'Center must be of the form: center_x,center_y.'
raise click.ClickException(msg)
center = tuple(int(x) for x in center)
inner_radii = inner_radii.split(',')
outer_radii = outer_radii.split(',')
if len(inner_radii) != len(outer_radii):
msg = 'Number of inner and outer radii must match'
raise click.ClickException(msg)
inner_radii = [int(x) for x in inner_radii]
outer_radii = [int(x) for x in outer_radii]
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
world_size = comm.Get_size()
if (world_size > len(files)):
if rank == 0:
print('Error: number of MPI processes,', world_size, ', exceeds',
'the number of files:', len(files))
return
comm.Barrier()
start = MPI.Wtime()
if dark_file is not None:
# Every process will do the dark field reference average for now
reader = io.reader(dark_file, version=io.FileVersion.VERSION3)
dark = image.calculate_average(reader)
else:
dark = np.zeros((576, 576))
# Split up the files among processes
files = get_files(files)
# Create local electron count
reader = io.reader(files, version=io.FileVersion.VERSION3)
electron_counted_data = image.electron_count(reader, dark, verbose=True)
local_frame_events = electron_counted_data.data
# Now reduce to root
global_frame_events = reduce_to_root_method1(local_frame_events)
# global_frame_events = reduce_to_root_method2(local_frame_events)
comm.Barrier()
end = MPI.Wtime()
if rank == 0:
print('time: %s' % (end - start))
if rank == 0:
# Create new electron counted data with the global frame events
data = namedtuple('ElectronCountedData',
['data', 'scan_dimensions', 'frame_dimensions'])
data.data = global_frame_events
data.scan_dimensions = electron_counted_data.scan_dimensions
data.frame_dimensions = electron_counted_data.frame_dimensions
# Write out the HDF5 file
io.save_electron_counts(output_file, data)
if generate_sparse:
# Save out the sparse image
stem_imgs = image.create_stem_images(data,
inner_radii,
outer_radii,
center=center)
for i, img in enumerate(stem_imgs):
fig, ax = plt.subplots(figsize=(12, 12))
ax.matshow(img)
name = 'sparse_stem_image_' + str(i) + '.png'
plt.savefig(name, dpi=300)