def _make_data(self):
self.th = np.linspace(0, 2*np.pi, self.n_pts)
self.scale = np.arange(self.N*self.M)
with h5py.File(self.filename, 'w') as f:
# create a group for maps to hold the data
mapsGrp = f.create_group('MAPS')
# now set a comment
mapsGrp.attrs['comments'] = 'MAPS group'
entryname = 'mca_arr'
comment = 'These are raw spectrum data.'
sn = np.sin(self.th).reshape(self.n_pts, 1, 1)
XY = self.scale.reshape(1, self.N, self.M)
data = XY * sn
ds_data = mapsGrp.create_dataset(entryname, data=data)
ds_data.attrs['comments'] = comment
# insert spectrum-wise resource and datum
resource_id = insert_resource('hdf5_maps', self.filename,
{'dset_path': 'mca_arr'})
self.eids_spectrum = [str(uuid.uuid4()) for j in range(self.N*self.M)]
for uid, (i, j) in zip(self.eids_spectrum,
product(range(self.N), range(self.M))):
insert_datum(resource_id, uid, {'x': i, 'y': j})
# insert plane-wise resource and datum
resource_id = insert_resource('hdf5_planes', self.filename,
{'dset_path': 'mca_arr'})
self.eids_planes = [str(uuid.uuid4()) for j in range(self.n_pts)]
for uid, n in zip(self.eids_planes, range(self.n_pts)):
insert_datum(resource_id, uid, {'e_index': n})
def get_data(ind_v, ind_h):
"""
Get data for given x, y index.
Parameters
----------
ind_v : int
vertical index
ind_h : int
horizontal index
Returns
-------
unicode:
id number of event
"""
uid = str(uuid.uuid1())
# generate 3D random number with a given shape
syn_data = np.random.randn(20, 1, 10)
file_path = save_syn_data(uid, syn_data)
custom = {'dset_path': 'mca_arr'}
fb = insert_resource('hdf_maps', file_path, resource_kwargs=custom)
evl = insert_datum(fb, uid, datum_kwargs={'x': ind_v, 'y': ind_h})
return evl.datum_id
def stage(self):
staged = super().stage()
res_kwargs = {'frame_per_point': 1}
logger.debug("Inserting resource with filename %s", self._fn)
self._resource = fsapi.insert_resource(self._spec, self._fn,
res_kwargs)
return staged
def test_non_unique_fail():
shape = (25, 32)
fb = insert_resource('syn-mod', None, {'shape': shape})
r_id = str(uuid.uuid4())
insert_datum(str(fb['id']), r_id, {'n': 0})
with pytest.raises(pymongo.errors.DuplicateKeyError):
insert_datum(str(fb['id']), r_id, {'n': 1})
def _insert_syn_data_bulk(f_type, shape, count):
fb = insert_resource(f_type, None, {'shape': shape})
d_uid = [str(uuid.uuid4()) for k in range(count)]
d_kwargs = [{'n': k + 1} for k in range(count)]
bulk_insert_datum(fb, d_uid, d_kwargs)
return d_uid
def _insert_syn_data(f_type, shape, count):
fb = insert_resource(f_type, None, {'shape': shape})
ret = []
for k in range(count):
r_id = str(uuid.uuid4())
insert_datum(str(fb.id), r_id, {'n': k + 1})
ret.append(r_id)
return ret
def _make_data(self):
N = 15
filename = self.filename
data = np.ones((N, 9, 8)) * np.arange(N).reshape(N, 1, 1)
np.save(filename, data)
# Insert the data records.
resource_id = insert_resource('npy_FRAMEWISE', filename + '.npy', {})
self.datum_ids = [str(uuid.uuid4()) for i in range(N)]
for i, datum_id in enumerate(self.datum_ids):
insert_datum(resource_id, datum_id, dict(frame_no=i))
def _insert_syn_data(f_type, shape, count):
fb = insert_resource(f_type, None, {'shape': shape})
ret = []
res_map_cycle = itertools.cycle((lambda x: x,
lambda x: x['id'],
lambda x: str(x['id'])))
for k, rmap in zip(range(count), res_map_cycle):
r_id = str(uuid.uuid4())
insert_datum(rmap(fb), r_id, {'n': k + 1})
ret.append(r_id)
return ret
def _make_data(self):
filename = self.filename
with h5py.File(filename) as f:
N = 5
# Write the data.
data = np.multiply.outer(np.arange(N), np.ones((2, 2)))
f.create_dataset('/entry/data/data', data=data)
# Insert the data records.
resource_id = insert_resource(self.spec, filename)
self.datum_ids = [str(uuid.uuid4()) for i in range(N)]
for i, datum_id in enumerate(self.datum_ids):
insert_datum(resource_id, datum_id, dict(point_number=i))
def test_overwrite_global():
h_cache = fsa._FS_SINGLETON._handler_cache
mock_base = dict(spec='syn-mod',
resource_path='',
resource_kwargs={'shape': (5, 7)})
res = fsa.insert_resource(**mock_base)
cache_key = (str(res['id']), SynHandlerMod.__name__)
with fsa.handler_context({'syn-mod': SynHandlerMod}):
fsa.get_spec_handler(res['id'])
assert cache_key in h_cache
fsa.register_handler('syn-mod', SynHandlerEcho, overwrite=True)
assert cache_key not in h_cache
def stage(self):
cam = getattr(self.parent, self._cam_name)
proc = getattr(self.parent, self._proc_name)
images_per_set = getattr(self.parent, self._ips_name).get()
num_sets = getattr(self.parent, self._num_sets_name).get()
self.stage_sigs.update([(proc.num_filter, images_per_set),
(cam.num_images, images_per_set * num_sets)])
super().stage()
res_kwargs = {'template': self.file_template.get(),
'filename': self.file_name.get(),
'frame_per_point': self.get_frames_per_point()}
self._resource = fs.insert_resource('AD_TIFF', self._fp, res_kwargs)
def test_get_handler_global():
h_cache = fsa._FS_SINGLETON._handler_cache
mock_base = dict(spec='syn-mod',
resource_path='',
resource_kwargs={'shape': (5, 7)})
res = fsa.insert_resource(**mock_base)
cache_key = (str(res['id']), SynHandlerMod.__name__)
with fsa.handler_context({'syn-mod': SynHandlerMod}):
handle = fsa.get_spec_handler(res['id'])
assert isinstance(handle, SynHandlerMod)
assert cache_key in h_cache
assert cache_key not in h_cache
def _make_data(self):
filename = self.filename
with h5py.File(filename) as f:
N = 5
# Write the data.
data = np.arange(N, dtype=np.float64)
f.create_dataset(
'/entry/instrument/NDAttributes/NDArrayEpicsTSSec',
data=data)
f.create_dataset(
'/entry/instrument/NDAttributes/NDArrayEpicsTSnSec',
data=data * 1e9)
# Insert the data records.
resource_id = insert_resource(self.spec, filename)
self.datum_ids = [str(uuid.uuid4()) for i in range(N)]
for i, datum_id in enumerate(self.datum_ids):
insert_datum(resource_id, datum_id, dict(point_number=i))
def stage(self):
global proposal_id
global run_id
global current_sample
set_and_wait(self.file_template, '%s%s_%6.6d_'+self.parent.detector_id+'.cbf', timeout=99999)
if self.reset_file_number.get() == 1:
set_and_wait(self.file_number, 1, timeout=99999)
path = '/GPFS/xf16id/exp_path/'
rpath = str(proposal_id)+"/"+str(run_id)+"/"
fpath = path + rpath
makedirs(fpath)
set_and_wait(self.file_path, fpath, timeout=99999)
set_and_wait(self.file_name, current_sample, timeout=99999)
super().stage()
res_kwargs = {'template': self.file_template.get(),
'filename': self.file_name.get(),
'frame_per_point': self.get_frames_per_point(),
'initial_number': self.file_number.get()}
self._resource = fs.insert_resource('AD_CBF', rpath, res_kwargs, root=path)
def stage(self):
global proposal_id
global run_id
global current_sample
global data_path
global collection_lock_file
set_and_wait(self.file_template, '%s%s_%6.6d_'+self.parent.detector_id+'.cbf', timeout=99999)
if self.reset_file_number.get() == 1:
set_and_wait(self.file_number, 1, timeout=99999)
# original code by Hugo
# this is done now when login()
#path = '/GPFS/xf16id/exp_path/'
#rpath = str(proposal_id)+"/"+str(run_id)+"/"
#fpath = path + rpath
#makedirs(fpath)
# modified by LY
# camserver saves data to the local ramdisk, a background process then move them to data_path
# interesting to note that camserver saves the data to filename.tmp, then rename it filename after done writing
# must have the '/' at the end, since camserver will add it to the RBV
# this should done only once for all Pilatus detectors
if self.parent.name == first_Pilatus():
#print("first Pilatus is %s" % self.parent.name)
change_path()
#set_and_wait(self.file_path, "/ramdisk/", timeout=99999)
set_and_wait(self.file_path, data_path, timeout=99999)
set_and_wait(self.file_name, current_sample, timeout=99999)
super().stage()
res_kwargs = {'template': self.file_template.get(),
'filename': self.file_name.get(),
'frame_per_point': self.get_frames_per_point(),
'initial_number': self.file_number.get()}
#self._resource = fs.insert_resource('AD_CBF', rpath, res_kwargs, root=path)
self._resource = fs.insert_resource('AD_CBF', data_path, res_kwargs, root="/")
if self.parent.name == first_Pilatus():
caput("XF:16IDC-ES:Sol{ctrl}ready", 1)
def add_data(self, data, uid=None, resource_kwargs=None):
"""
Parameters
----------
data : ndarray
The data to save
uid : str, optional
The uid to be used for this entry,
if not given use uuid1 to generate one
resource_kwargs : None, optional
Currently raises if not 'falsy' and is ignored.
Returns
-------
uid : str
The uid used to register this data with filestore, can
be used to retrieve it
"""
if not self._writable:
raise RuntimeError("This writer can only write one data entry "
"and has already been used")
if resource_kwargs:
raise ValueError("This writer does not support resource_kwargs")
if op.exists(self._fpath):
raise IOError("the requested file {fpath} "
"already exist".format(fpath=self._fpath))
if uid is None:
uid = str(uuid.uuid1())
np.save(self._fpath, np.asanyarray(data))
self._writable = False
fb = fsc.insert_resource(self.SPEC_NAME, self._fpath, self._f_custom)
evl = fsc.insert_datum(fb, uid, {})
return evl.datum_id
def stage(self):
"Set the filename and record it in a 'resource' document in the filestore database."
print(self.name, 'stage')
DIRECTORY = '/GPFS/xf08id/'
rpath = 'pizza_box_data'
filename = 'en_' + str(uuid.uuid4())[:6]
full_path = os.path.join(rpath, filename)
self._full_path = os.path.join(DIRECTORY,
full_path) # stash for future reference
if len(self._full_path) > 40:
raise RuntimeError(
"Stupidly, EPICS limits the file path to 80 characters. "
"Choose a different DIRECTORY with a shorter path. (I know....)"
)
self._full_path = os.path.join(DIRECTORY,
full_path) # stash for future reference
self.filepath.put(self._full_path)
self.resource_uid = fs.insert_resource('PIZZABOX_ENC_FILE_TXT',
full_path,
{'chunk_size': self.chunk_size},
root=DIRECTORY)
super().stage()
def stage(self):
# if should external trigger
ext_trig = self.parent.external_trig.get()
logger.debug('Stopping xspress3 acquisition')
# really force it to stop acquiring
self.settings.acquire.put(0, wait=True)
total_points = self.parent.total_points.get()
spec_per_point = self.parent.spectra_per_point.get()
total_capture = total_points * spec_per_point
# stop previous acquisition
self.stage_sigs[self.settings.acquire] = 0
# re-order the stage signals and disable the calc record which is
# interfering with the capture count
self.stage_sigs.pop(self.num_capture, None)
self.stage_sigs.pop(self.settings.num_images, None)
self.stage_sigs[self.num_capture_calc_disable] = 1
if ext_trig:
logger.debug('Setting up external triggering')
self.stage_sigs[self.settings.trigger_mode] = 'TTL Veto Only'
self.stage_sigs[self.settings.num_images] = total_capture
else:
logger.debug('Setting up internal triggering')
# self.settings.trigger_mode.put('Internal')
# self.settings.num_images.put(1)
self.stage_sigs[self.settings.trigger_mode] = 'Internal'
self.stage_sigs[self.settings.num_images] = spec_per_point
self.stage_sigs[self.auto_save] = 'No'
logger.debug('Configuring other filestore stuff')
logger.debug('Making the filename')
filename, read_path, write_path = self.make_filename()
logger.debug('Setting up hdf5 plugin: ioc path: %s filename: %s',
write_path, filename)
logger.debug('Erasing old spectra')
self.settings.erase.put(1, wait=True)
# this must be set after self.settings.num_images because at the Epics
# layer there is a helpful link that sets this equal to that (but
# not the other way)
self.stage_sigs[self.num_capture] = total_capture
# actually apply the stage_sigs
ret = super().stage()
self._fn = self.file_template.get() % (self._fp,
self.file_name.get(),
self.file_number.get())
if not self.file_path_exists.value:
raise IOError("Path {} does not exits on IOC!! Please Check"
.format(self.file_path.value))
logger.debug('Inserting the filestore resource: %s', self._fn)
self._filestore_res = fs_api.insert_resource(
Xspress3HDF5Handler.HANDLER_NAME, self._fn, {})
# this gets auto turned off at the end
self.capture.put(1)
# Xspress3 needs a bit of time to configure itself...
# this does not play nice with the event loop :/
time.sleep(self._config_time)
return ret
def stage(self):
super().stage()
res_kwargs = {'frame_per_point': self.get_frames_per_point()}
logger.debug("Inserting resource with filename %s", self._fn)
self._resource = fs.insert_resource('AD_HDF5', self._fn, res_kwargs)
def _insert_fs_resource(self):
return fs_api.insert_resource(Xspress3HDF5Handler.HANDLER_NAME,
self.store_filename, {})
def stage(self):
super().stage()
res_kwargs = {'template': self.file_template.get(),
'filename': self.file_name.get(),
'frame_per_point': self.get_frames_per_point()}
self._resource = fs.insert_resource('AD_TIFF', self._fp, res_kwargs)
def _insert_fs_resource(self):
return fs.insert_resource('AD_TIFF', self._store_file_path,
{'template': self._file_template.value,
'filename': self._filename,
'frame_per_point': 1})
def _insert_fs_resource(self):
return fs.insert_resource('AD_SPE', self._store_file_path,
{'template': self._file_template.value,
'filename': self._filename,
'frame_per_point': self._num_images.value})
def _insert_fs_resource(self):
return fs.insert_resource('AD_HDF5',
self._store_filename,
{'frame_per_point':
self._num_images.value})
