def _setup_beginruns(self): """ Determines if there is a next run as 1) New run found in the same smalldata files 2) New run found in the new smalldata files """ while True: if nodetype == 'smd0': dgrams = self.smdr_man.get_next_dgrams() nbytes = np.zeros(len(self.smd_files), dtype='i') if dgrams is not None: nbytes = np.array([memoryview(d).shape[0] for d in dgrams], dtype='i') else: dgrams = None nbytes = np.empty(len(self.smd_files), dtype='i') self.comms.psana_comm.Bcast(nbytes, root=0) if np.sum(nbytes) == 0: return False if nodetype != 'smd0': dgrams = [np.empty(nbyte, dtype='b') for nbyte in nbytes] for i in range(len(dgrams)): self.comms.psana_comm.Bcast([dgrams[i], nbytes[i], MPI.BYTE], root=0) if nodetype != 'smd0': dgrams = [dgram.Dgram(view=d, config=config, offset=0) \ for d, config in zip(dgrams,self._configs)] if dgrams[0].service() == TransitionId.BeginRun: self.beginruns = dgrams return True
def events(self, view): pf = PacketFooter(view=view) views = pf.split_packets() # Keeps offset, size, & timestamp for all events in the batch # for batch reading (if filter_fn is not given). ofsz_batch = np.zeros((pf.n_packets, self.n_smd_files, 2), dtype=np.intp) has_offset = True for i, event_bytes in enumerate(views): if event_bytes: evt = Event._from_bytes(self.smd_configs, event_bytes) if not evt._has_offset: has_offset = False yield evt # no offset info in the smalldata event else: segment = 0 # "d.info" "detectors" have only one segment ofsz = np.asarray([[d.info[segment].offsetAlg.intOffset, d.info[segment].offsetAlg.intDgramSize] \ for d in evt]) ofsz_batch[i, :, :] = ofsz # Only get big data one event at a time when filter is off if self.filter_fn: bd_evt = self.dm.jump(ofsz[:, 0], ofsz[:, 1]) yield bd_evt if self.filter_fn == 0 and has_offset: # Read chunks of 'size' bytes and store them in views views = [None] * self.n_smd_files view_sizes = np.zeros(self.n_smd_files) for i in range(self.n_smd_files): # If no data were filtered, we can assume that all bigdata # dgrams starting from the first offset are stored consecutively # in the file. We read a chunk of sum(all dgram sizes) and # store in a view. offset = ofsz_batch[0, i, 0] size = np.sum(ofsz_batch[:, i, 1]) view_sizes[i] = size os.lseek(self.dm.fds[i], offset, 0) views[i] = os.read(self.dm.fds[i], size) # Build each event from these views dgrams = [None] * self.n_smd_files offsets = [0] * self.n_smd_files for i in range(pf.n_packets): for j in range(self.n_smd_files): if offsets[j] >= view_sizes[j]: continue size = ofsz_batch[i, j, 1] if size: dgrams[j] = dgram.Dgram(view=views[j], config=self.dm.configs[j], offset=offsets[j]) offsets[j] += size bd_evt = Event(dgrams) yield bd_evt
def __next__(self): if self.cn_events == self.n_events: raise StopIteration if len(self.dm.xtc_files) == 0: smd_evt = Event._from_bytes(self.smd_configs, self.smd_events[self.cn_events], run=self.dm.run()) self.cn_events += 1 return smd_evt if self.filter_fn: smd_evt = Event._from_bytes(self.smd_configs, self.smd_events[self.cn_events], run=self.dm.run()) self.cn_events += 1 ofsz = np.asarray([[d.smdinfo[0].offsetAlg.intOffset, \ d.smdinfo[0].offsetAlg.intDgramSize] for d in smd_evt]) bd_evt = self.dm.jump(ofsz[:, 0], ofsz[:, 1]) return bd_evt dgrams = [None] * self.n_smd_files ofsz = self.ofsz_batch[self.cn_events, :, :] for j in range(self.n_smd_files): if ofsz[j, 1]: dgrams[j] = dgram.Dgram(view=self.bigdata[j], config=self.dm.configs[j], offset=ofsz[j, 0]) bd_evt = Event(dgrams, run=self.dm.run()) self.cn_events += 1 return bd_evt
def __init__(self, xtc_files, configs=[], tag=None, run=None): """ Opens xtc_files and stores configs.""" self.xtc_files = [] self.shmem_cli = None self.shmem_kwargs = {'index':-1,'size':0,'cli_cptr':None} self.configs = [] self.fds = [] self._timestamps = [] # built when iterating self._run = run if isinstance(xtc_files, (str)): self.xtc_files = np.array([xtc_files], dtype='U%s'%FN_L) assert len(self.xtc_files) > 0 elif isinstance(xtc_files, (list, np.ndarray)): if len(xtc_files) > 0: # handles smalldata-only case if xtc_files[0] == 'shmem': self.shmem_cli = PyShmemClient() #establish connection to available server - blocking status = int(self.shmem_cli.connect(tag,0)) assert not status,'shmem connect failure %d' % status #wait for first configure datagram - blocking view = self.shmem_cli.get(self.shmem_kwargs) assert view d = dgram.Dgram(view=view, \ shmem_index=self.shmem_kwargs['index'], \ shmem_size=self.shmem_kwargs['size'], \ shmem_cli_cptr=self.shmem_kwargs['cli_cptr'], \ shmem_cli_pyobj=self.shmem_cli) self.configs += [d] else: self.xtc_files = np.asarray(xtc_files, dtype='U%s'%FN_L) assert len(self.xtc_files) > 0 given_configs = True if len(configs) > 0 else False if given_configs: self.configs = configs for i, xtcdata_filename in enumerate(self.xtc_files): self.fds.append(os.open(xtcdata_filename, os.O_RDONLY)) if not given_configs: d = dgram.Dgram(file_descriptor=self.fds[-1]) self.configs += [d] self.det_class_table, self.xtc_info = self.get_det_class_table() self.calibs = {} # initialize to empty dict - will be populated by run class
def get_next_dgrams(self, configs=None): dgrams = None if not self.smdr.is_complete(): self.smdr.get() if self.smdr.is_complete(): mmrv_bufs, _ = self.smdr.view(batch_size=1) # For configs, we need to copy data from smdreader's buffers # This prevents it from getting overwritten by other dgrams. bytearray_bufs = [bytearray(mmrv_buf) for mmrv_buf in mmrv_bufs] if configs is None: dgrams = [dgram.Dgram(view=ba_buf, offset=0) for ba_buf in bytearray_bufs] else: dgrams = [dgram.Dgram(view=ba_buf, config=config, offset=0) for ba_buf, config in zip(bytearray_bufs, configs)] return dgrams
def _check_missing_endrun(self, beginruns=None): fake_endruns = None if not self.found_endrun: # there's no previous EndRun sec = (self._timestamps[-1] >> 32) & 0xffffffff usec = int((self._timestamps[-1] & 0xffffffff) * 1e3 + 1) if beginruns: self.buffered_beginruns = [ dgram.Dgram(config=config, view=d, offset=0, size=d._size) for d, config in zip(beginruns, self.configs) ] fake_endruns = [dgram.Dgram(config=config, fake_endrun=1, \ fake_endrun_sec=sec, fake_endrun_usec=usec) \ for config in self.configs] self.found_endrun = True else: self.found_endrun = False return fake_endruns
def from_bytes(configs, event_bytes): dgrams = [] if event_bytes: dgrams_bytes = event_bytes.split(b'eod') assert len(configs) == len(dgrams_bytes) dgrams = [dgram.Dgram(config=configs[i], view=dgrams_bytes[i]) \ for i in range(len(configs))] evt = Event(dgrams=dgrams) return evt
def __init__(self, comms, exp, run_no, run_src, **kwargs): """ Parallel read requires that rank 0 does the file system works. Configs and calib constants are sent to other ranks by MPI. Note that destination callback only works with RunParallel. """ super(RunParallel, self).__init__(exp, run_no, max_events=kwargs['max_events'], \ batch_size=kwargs['batch_size'], filter_callback=kwargs['filter_callback'], \ destination=kwargs['destination']) xtc_files, smd_files, other_files = run_src self.comms = comms psana_comm = comms.psana_comm # TODO tjl and cpo to review rank = psana_comm.Get_rank() size = psana_comm.Get_size() if rank == 0: self.smd_dm = DgramManager(smd_files, run=self) self.dm = DgramManager(xtc_files, configs=self.smd_dm.configs, run=self) self.configs = self.dm.configs nbytes = np.array([memoryview(config).shape[0] for config in self.configs], \ dtype='i') self.calibs = {} for det_name in self.detnames: self.calibs[det_name] = super(RunParallel, self)._get_calib(det_name) else: self.smd_dm = None self.dm = None self.configs = None self.calibs = None nbytes = np.empty(len(smd_files), dtype='i') psana_comm.Bcast(nbytes, root=0) # no. of bytes is required for mpich # create empty views of known size if rank > 0: self.configs = [np.empty(nbyte, dtype='b') for nbyte in nbytes] for i in range(len(self.configs)): psana_comm.Bcast([self.configs[i], nbytes[i], MPI.BYTE], root=0) self.calibs = psana_comm.bcast(self.calibs, root=0) if rank > 0: # Create dgram objects using views from rank 0 (no disk operation). self.configs = [ dgram.Dgram(view=config, offset=0) for config in self.configs ] self.dm = DgramManager(xtc_files, configs=self.configs, run=self) self.esm = EnvStoreManager(self.configs, 'epics', 'scan')
def _get_next_evt(self): """ Generate bd evt for different cases: 1) No bigdata or is a Transition Event prior to i_first_L1 create dgrams from smd_view 2) L1Accept event create dgrams from bd_bufs 3) L1Accept with some smd files replaced by bigdata files create dgram from smd_view if use_smds[i_smd] is set otherwise create dgram from bd_bufs """ dgrams = [None] * self.n_smd_files for i_smd in range(self.n_smd_files): # Check in case we need to switch to the next bigdata chunk file if self.services[self.i_evt] != TransitionId.L1Accept: if self.new_chunk_id_array[self.i_evt, i_smd] != 0: print(f'open_new_bd_file i_smd={i_smd} chunk_id={self.new_chunk_id_array[self.i_evt, i_smd]}') self._open_new_bd_file(i_smd, self.new_chunk_id_array[self.i_evt, i_smd]) view, offset, size = (bytearray(),0,0) # Try to create dgram from smd view if self.dm.n_files == 0 or self.use_smds[i_smd] \ or self.i_evt < self.i_first_L1s[i_smd]: view = self.smd_view offset = self.smd_offset_array[self.i_evt, i_smd] size = self.smd_size_array[self.i_evt, i_smd] # Non L1 dgram prior to i_first_L1 are counted as a new "chunk" # because their cutoff flag is set (data coming from smd view # instead of bd chunk). We'll need to update chunk index for # this smd when we see non L1. self.chunk_indices[i_smd] += 1 else: # Check if we need to fill bd buf if this dgram doesn't fit in the current view if self.bd_buf_offsets[i_smd] + self.bd_size_array[self.i_evt, i_smd] \ > memoryview(self.bd_bufs[i_smd]).nbytes: self._fill_bd_chunk(i_smd) self.chunk_indices[i_smd] += 1 # This is the offset of bd buffer! and not what stored in smd dgram, # which in contrast points to the location of disk. offset = self.bd_buf_offsets[i_smd] size = self.bd_size_array[self.i_evt, i_smd] view = self.bd_bufs[i_smd] self.bd_buf_offsets[i_smd] += size if size: # handles missing dgram dgrams[i_smd] = dgram.Dgram(config=self.dm.configs[i_smd], view=view, offset=offset) self.i_evt += 1 self._inc_prometheus_counter('evts') evt = Event(dgrams=dgrams, run=self.dm.get_run()) print(f'YIELD ts={evt.timestamp} service={evt.service()}') return evt
def next(self): """ only support sequential read - no event building""" if self.shmem: view = self.shmem.get(self.shmem_kwargs) if view: # use the most recent configure datagram config = self.configs[len(self.configs) - 1] d = dgram.Dgram(config=config,view=view, \ shmem_index=self.shmem_kwargs['index'], \ shmem_size=self.shmem_kwargs['size'], \ shmem_cli=self.shmem_kwargs['cli']) dgrams = [d] else: raise StopIteration else: dgrams = [dgram.Dgram(config=config) for config in self.configs] evt = Event(dgrams) self._timestamps += [evt.timestamp] return evt
def _get_next_evt(self): """ Generate bd evt for different cases: 1) No bigdata or Transition Event create dgrams from smd_view 2) L1Accept event create dgrams from bd_bufs 3) L1Accept with some smd files replaced by bigdata files create dgram from smd_view if use_smds[i_smd] is set otherwise create dgram from bd_bufs """ dgrams = [None] * self.n_smd_files for i_smd in range(self.n_smd_files): if self.dm.n_files == 0 or \ self.services[self.i_evt] != TransitionId.L1Accept or \ self.use_smds[i_smd]: view = self.smd_view offset = self.smd_offset_array[self.i_evt, i_smd] size = self.smd_size_array[self.i_evt, i_smd] # Non L1 always are counted as a new "chunk" since they # ther cutoff flag is set (data coming from smd view # instead of bd chunk. We'll need to update chunk index for # this smd when we see non L1. self.chunk_indices[i_smd] += 1 # Check in case we need to switch to the next bigdata chunk file if self.services[self.i_evt] != TransitionId.L1Accept: if self.new_chunk_id_array[self.i_evt, i_smd] != 0: self._open_new_bd_file( i_smd, self.new_chunk_id_array[self.i_evt, i_smd]) else: # Fill up bd buf if this dgram doesn't fit in the current view if self.bd_buf_offsets[i_smd] + self.bd_size_array[self.i_evt, i_smd] \ > memoryview(self.bd_bufs[i_smd]).nbytes: self._fill_bd_chunk(i_smd) self.chunk_indices[i_smd] += 1 # This is the offset of bd buffer! and not what stored in smd dgram, # which in contrast points to the location of disk. offset = self.bd_buf_offsets[i_smd] size = self.bd_size_array[self.i_evt, i_smd] view = self.bd_bufs[i_smd] self.bd_buf_offsets[i_smd] += size if size > 0: # handles missing dgram dgrams[i_smd] = dgram.Dgram(config=self.dm.configs[i_smd], view=view, offset=offset) self.i_evt += 1 self._inc_prometheus_counter('evts') evt = Event(dgrams=dgrams, run=self.dm.get_run()) return evt
def jump(self, offsets, sizes): """ Jumps to the offset and reads out dgram on each xtc file. This is used in normal mode (multiple detectors with MPI). """ assert len(offsets) > 0 and len(sizes) > 0 dgrams = [] for fd, config, offset, size in zip(self.fds, self.configs, offsets, sizes): d = dgram.Dgram(file_descriptor=fd, config=config, offset=offset, size=size) dgrams += [d] evt = Event(dgrams, run=self.run()) return evt
def get_next_dgrams(self): """ Returns list of dgrams as appeared in the current offset of the smd chunks. Currently used to retrieve Configure and BeginRun. This allows read with wait for these two types of dgram. """ if self.dsparms.max_events > 0 and \ self.processed_events >= self.dsparms.max_events: logger.debug(f'max_events={self.dsparms.max_events} reached') return None dgrams = None if not self.smdr.is_complete(): self._get() if self.smdr.is_complete(): # Get chunks with only one dgram each. There's no need to set # integrating stream id here since Configure and BeginRun # must exist in this stream too. self.smdr.view(batch_size=1) # For configs, we need to copy data from smdreader's buffers # This prevents it from getting overwritten by other dgrams. bytearray_bufs = [ bytearray(self.smdr.show(i)) for i in range(self.n_files) ] if self.configs is None: dgrams = [ dgram.Dgram(view=ba_buf, offset=0) for ba_buf in bytearray_bufs ] self.configs = dgrams self.smdr.set_configs(self.configs) else: dgrams = [ dgram.Dgram(view=ba_buf, config=config, offset=0) for ba_buf, config in zip(bytearray_bufs, self.configs) ] return dgrams
def jumps(self, dgram_i, offset, size): if offset == 0 and size == 0: d = None else: try: d = dgram.Dgram(file_descriptor=self.fds[dgram_i], config=self.configs[dgram_i], offset=offset, size=size, max_retries=self.max_retries) except StopIteration: d = None return d
def next(self, offsets=[], sizes=[], read_chunk=True): assert len(self.offsets) > 0 or len(offsets) > 0 if len(offsets) == 0: offsets = self.offsets if len(sizes) == 0: sizes = [0] * len(offsets) dgrams = [] for fd, config, offset, size in zip(self.fds, self.configs, offsets, sizes): if (read_chunk): d = dgram.Dgram(config=config, offset=offset) else: assert size > 0 d = dgram.Dgram(file_descriptor=fd, config=config, offset=offset, size=size) dgrams += [d] evt = Event(dgrams=dgrams) self.offsets = evt.offsets return evt
def main(): fd = os.open('/reg/d/psdm/xpp/xpptut15/scratch/mona/smd.xtc', os.O_RDONLY) config = dgram.Dgram(file_descriptor=fd) print(config) # Iterate chunks of n_events n_events = 10000 dchunk = dgramchunk.DgramChunk(fd) displacement = memoryview(config).shape[0] view = dchunk.get(displacement, n_events) while view != 0: do_chunk(bytes(view)) # FIXME: Find a way to pickle without copying displacement += view.nbytes view = dchunk.get(displacement, n_events)
def _from_bytes(cls, configs, event_bytes, run=None): dgrams = [] if event_bytes: pf = PacketFooter(view=event_bytes) views = pf.split_packets() assert len(configs) == pf.n_packets dgrams = [None]*pf.n_packets # make sure that dgrams are arranged # according to the smd files. for i in range(pf.n_packets): if views[i].shape[0] > 0: # do not include any missing dgram dgrams[i] = dgram.Dgram(config=configs[i], view=views[i]) evt = cls(dgrams, run=run) return evt
def smd_client(config): while True: comm.Send(np.array([rank], dtype='i'), dest=0) info = MPI.Status() comm.Probe(MPI.ANY_SOURCE, MPI.ANY_TAG, info) count = info.Get_elements(MPI.BYTE) view = bytearray(count) comm.Recv(view, source=0) if view.startswith(b'end'): break offset = 0 while offset < count: d = dgram.Dgram(config=config, view=view, offset=offset) offset += memoryview(d).shape[0]
def _set_configinfo(self): """ From configs, we generate a dictionary lookup with det_name as a key. The information stored the value field contains: - configs specific to that detector - sorted_segment_ids used by Detector cls for checking if an event has correct no. of segments - detid_dict has segment_id as a key - dettype - uniqueid """ self.configinfo_dict = {} for _, det_class in self.dm.det_classes.items( ): # det_class is either normal or envstore for (det_name, _), _ in det_class.items(): # Create a copy of list of configs for this detector det_configs = [dgram.Dgram(view=config) for config in self.dm.configs \ if hasattr(config.software, det_name)] sorted_segment_ids = [] # a dictionary of the ids (a.k.a. serial-number) of each segment detid_dict = {} dettype = "" uniqueid = "" for config in det_configs: seg_dict = getattr(config.software, det_name) sorted_segment_ids += list(seg_dict.keys()) for segment, det in seg_dict.items(): detid_dict[segment] = det.detid dettype = det.dettype sorted_segment_ids.sort() uniqueid = dettype for segid in sorted_segment_ids: uniqueid += '_' + detid_dict[segid] self.configinfo_dict[det_name] = type("ConfigInfo", (), {\ "configs": det_configs, \ "sorted_segment_ids": sorted_segment_ids, \ "detid_dict": detid_dict, \ "dettype": dettype, \ "uniqueid": uniqueid})
def __next__(self): if self.cn_events == self.n_events: raise StopIteration smd_evt = Event._from_bytes(self.smd_configs, self.smd_events[self.cn_events], run=self.dm.get_run()) if len(self.dm.xtc_files ) == 0 or smd_evt.service() != TransitionId.L1Accept: self.cn_events += 1 self._inc_prometheus_counter('evts') return smd_evt if self.filter_fn: bd_dgrams = [] read_size = 0 for smd_i, smd_dgram in enumerate(smd_evt._dgrams): if self.use_smds[smd_i]: bd_dgrams.append(smd_dgram) else: offset_and_size = smd_evt.get_offset_and_size(smd_i) read_size += offset_and_size[0, 1] bd_dgrams.append( self._read_dgram_from_disk(smd_i, offset_and_size)) bd_evt = Event(dgrams=bd_dgrams, run=self.dm.get_run()) self.cn_events += 1 self._inc_prometheus_counter('evts') return bd_evt dgrams = [None] * self.n_smd_files ofsz = self.ofsz_batch[self.cn_events, :, :] for i_smd in range(self.n_smd_files): d_offset, d_size = ofsz[i_smd] if d_size and d_offset + d_size <= \ memoryview(self.bigdata[i_smd]).nbytes: dgrams[i_smd] = dgram.Dgram(view=self.bigdata[i_smd], config=self.dm.configs[i_smd], offset=d_offset) bd_evt = Event(dgrams, run=self.dm.get_run()) self.cn_events += 1 self._inc_prometheus_counter('evts') return bd_evt
def __next__(self): if self.cn_events == self.n_events: raise StopIteration if len(self.dm.xtc_files) == 0: smd_evt = Event._from_bytes(self.smd_configs, self.smd_events[self.cn_events], run=self.dm.run()) self.cn_events += 1 self._inc_prometheus_counter('evts') return smd_evt if self.filter_fn: smd_evt = Event._from_bytes(self.smd_configs, self.smd_events[self.cn_events], run=self.dm.run()) self.cn_events += 1 if smd_evt.service() == TransitionId.L1Accept: offset_and_size_array = smd_evt.get_offsets_and_sizes() bd_evt = self._read_event_from_disk( offset_and_size_array[:, 0], offset_and_size_array[:, 1]) self._inc_prometheus_counter( 'MB', np.sum(offset_and_size_array[:, 1]) / 1e6) else: bd_evt = smd_evt self._inc_prometheus_counter('evts') return bd_evt dgrams = [None] * self.n_smd_files ofsz = self.ofsz_batch[self.cn_events, :, :] for j in range(self.n_smd_files): d_offset, d_size = ofsz[j] if d_size and d_offset + d_size <= \ memoryview(self.bigdata[j]).nbytes: dgrams[j] = dgram.Dgram(view=self.bigdata[j], config=self.dm.configs[j], offset=d_offset) bd_evt = Event(dgrams, run=self.dm.run()) self.cn_events += 1 self._inc_prometheus_counter('evts') return bd_evt
def _setup_configs(self): """ Creates and broadcasts configs only called by _setup_run() """ g_ts = self.prom_man.get_metric("psana_timestamp") if nodetype == 'smd0': super()._close_opened_smd_files() self.smd_fds = np.array([ os.open(smd_file, os.O_RDONLY) for smd_file in self.smd_files ], dtype=np.int32) logging.info(f'mpi_ds: smd0 opened smd_fds: {self.smd_fds}') self.smdr_man = SmdReaderManager(self.smd_fds, self.dsparms) self._configs = self.smdr_man.get_next_dgrams() super()._apply_detector_selection() super()._setup_det_class_table() super()._set_configinfo() g_ts.labels("first_event").set(time.time()) nbytes = np.array([memoryview(config).shape[0] for config in self._configs], \ dtype='i') else: self._configs = None nbytes = np.empty(len(self.smd_files), dtype='i') self.comms.psana_comm.Bcast( nbytes, root=0) # no. of bytes is required for mpich if nodetype != 'smd0': self._configs = [np.empty(nbyte, dtype='b') for nbyte in nbytes] for i in range(len(self._configs)): self.comms.psana_comm.Bcast( [self._configs[i], nbytes[i], MPI.BYTE], root=0) if nodetype != 'smd0': self._configs = [ dgram.Dgram(view=config, offset=0) for config in self._configs ] g_ts.labels("first_event").set(time.time()) self._setup_det_class_table() self._set_configinfo()
def run_smd0(): filenames = glob.glob( '/reg/d/psdm/xpp/xpptut15/scratch/mona/test/smalldata/*.smd.xtc') #filenames = glob.glob('/u1/mona/smalldata/*.smd.xtc') fds = [os.open(filename, os.O_RDONLY) for filename in filenames] # read configs so that fileptr is in the right place configs = [dgram.Dgram(file_descriptor=fd) for fd in fds] limit = int(sys.argv[1]) st = time.time() smdr = SmdReader(fds[:limit]) got_events = -1 n_events = 10000 processed_events = 0 while got_events != 0: smdr.get(n_events) got_events = smdr.got_events processed_events += got_events #if processed_events >= 100: break #print("processed_events: %d"%processed_events) en = time.time() print("Elapsed Time (s): %f #Events: %d Rate: %f" % ((en - st), processed_events, processed_events / ((en - st) * 1e6)))
def jump(self, offsets, sizes): """ Jumps to the offset and reads out dgram on each xtc file. This is used in normal mode (multiple detectors with MPI). """ assert len(offsets) > 0 and len(sizes) > 0 dgrams = [] for fd, config, offset, size in zip(self.fds, self.configs, offsets, sizes): if offset == 0 and size == 0: d = None else: try: d = dgram.Dgram(file_descriptor=fd, config=config, offset=offset, size=size, max_retries=self.max_retries) except StopIteration: d = None dgrams += [d] evt = Event(dgrams, run=self.run()) return evt
def __init__(self, xtc_files, configs=[]): if isinstance(xtc_files, (str)): self.xtc_files = np.array([xtc_files], dtype='U%s' % FN_L) elif isinstance(xtc_files, (list, np.ndarray)): self.xtc_files = np.asarray(xtc_files, dtype='U%s' % FN_L) assert len(self.xtc_files) > 0 given_configs = True if len(configs) > 0 else False self.configs = [] if given_configs: self.configs = configs for i in range(len(self.configs)): self.configs[i]._assign_dict() self.fds = [] for i, xtcdata_filename in enumerate(self.xtc_files): self.fds.append( os.open(xtcdata_filename, os.O_RDONLY | os.O_LARGEFILE)) if not given_configs: d = dgram.Dgram(file_descriptor=self.fds[-1]) self.configs += [d] self.offsets = [_config._offset for _config in self.configs]
if __name__ == "__main__": comm.Barrier() ts0 = MPI.Wtime() nfiles = 2 # broadcast smd files if rank == 0: smd_files = np.array(['/reg/d/psdm/xpp/xpptut15/scratch/mona/smd-00.xtc', '/reg/d/psdm/xpp/xpptut15/scratch/mona/smd-01.xtc'], dtype='U%s'%FN_L) else: smd_files = np.empty(nfiles, dtype='U%s'%FN_L) comm.Bcast([smd_files, MPI.CHAR], root=0) # broadcast configs if rank == 0: fds = [os.open(smd_file, os.O_RDONLY) for smd_file in smd_files] configs = [dgram.Dgram(file_descriptor=fd) for fd in fds] nbytes = np.array([memoryview(config).shape[0] for config in configs], dtype='i') else: configs = [dgram.Dgram()] * nfiles nbytes = np.empty(nfiles, dtype='i') comm.Bcast(nbytes, root=0) for i in range(nfiles): comm.Bcast([configs[i], nbytes[i], MPI.BYTE], root=0) comm.Barrier() ts1 = MPI.Wtime() # start smd-bd nodes n_smd_nodes = int(round((size - 1) * .25)) n_bd_nodes = size - 1 - n_smd_nodes if rank == 0:
import dgram from psana import dgram as psana_dgram from psana.smdreader import SmdReader import os fd = os.open('data-r0001-s00.smd.xtc2', os.O_RDONLY) config = psana_dgram.Dgram(file_descriptor=fd) smdr = SmdReader([fd]) smdr.get(1) print(smdr.got_events) dgram.get(smdr.view(0))
def __init__(self, expstr, filter=filter, batch_size=1): """Initializes datasource. Keyword arguments: expstr -- experiment string (eg. exp=xpptut13:run=1) or a file or list of files (eg. 'data.xtc' or ['data0.xtc','dataN.xtc']) batch_size -- length of batched offsets Supports: reading file(s) -- DgramManager handles those files (single core) reading an exp -- Single core owns both smd and xtc DgramManagers Multiple cores: rank 0 owns smd DgramManager rank 1 owns xtc DgramManager """ assert batch_size > 0 self.batch_size = batch_size self.filter = filter self.nodetype = 'bd' # Check if we are reading file(s) or an experiment self.read_files = False if isinstance(expstr, (str)): if expstr.find("exp") == -1: self.xtc_files = np.array([expstr], dtype='U%s' % FN_L) self.smd_files = None else: self.read_files = True elif isinstance(expstr, (list, np.ndarray)): self.xtc_files = np.asarray(expstr, dtype='U%s' % FN_L) self.smd_files = None if not self.read_files: self.dm = DgramManager(self.xtc_files) self.configs = self.dm.configs else: # Fetch xtc files if rank == 0: opts = expstr.split(':') exp = {} for opt in opts: items = opt.split('=') assert len(items) == 2 exp[items[0]] = items[1] run = '' if 'dir' in exp: xtc_path = exp['dir'] else: xtc_dir = os.environ.get('SIT_PSDM_DATA', '/reg/d/psdm') xtc_path = os.path.join(xtc_dir, exp['exp'][:3], exp['exp'], 'xtc') if 'run' in exp: run = exp['run'] if run: self.xtc_files = np.array(glob.glob( os.path.join(xtc_path, '*r%s*.xtc' % (run.zfill(4)))), dtype='U%s' % FN_L) else: self.xtc_files = np.array(glob.glob( os.path.join(xtc_path, '*.xtc')), dtype='U%s' % FN_L) self.xtc_files.sort() self.smd_files = np.empty(len(self.xtc_files), dtype='U%s' % FN_L) smd_dir = os.path.join(xtc_path, 'smalldata') for i, xtc_file in enumerate(self.xtc_files): smd_file = os.path.join( smd_dir, os.path.splitext(os.path.basename(xtc_file))[0] + '.smd.xtc') if os.path.isfile(smd_file): self.smd_files[i] = smd_file else: raise InputError(smd_file, "File not found.") self.nfiles = np.array([len(self.xtc_files)], dtype='i') assert self.nfiles[0] > 0 else: self.nfiles = np.zeros(1, dtype='i') if size == 1: self.smd_dm = DgramManager(self.smd_files) self.dm = DgramManager(self.xtc_files, configs=self.smd_dm.configs) self.configs = self.smd_dm.configs else: # Send filenames comm.Bcast(self.nfiles, root=0) if rank > 0: self.xtc_files = np.empty(self.nfiles[0], dtype='U%s' % FN_L) self.smd_files = np.empty(self.nfiles[0], dtype='U%s' % FN_L) comm.Bcast([self.xtc_files, MPI.CHAR], root=0) comm.Bcast([self.smd_files, MPI.CHAR], root=0) # Send configs if rank == 0: self.dm = DgramManager(self.smd_files) self.configs = self.dm.configs nbytes = np.array([memoryview(config).shape[0] for config in self.configs], \ dtype='i') else: self.dm = None self.configs = [ dgram.Dgram() for i in range(self.nfiles[0]) ] nbytes = np.empty(self.nfiles[0], dtype='i') comm.Bcast(nbytes, root=0) # no. of bytes is required for mpich for i in range(self.nfiles[0]): comm.Bcast([self.configs[i], nbytes[i], MPI.BYTE], root=0) # Assign node types self.nsmds = int( os.environ.get('PS_SMD_NODES', np.ceil((size - 1) * PERCENT_SMD))) if rank == 0: self.nodetype = 'smd0' elif rank < self.nsmds + 1: self.nodetype = 'smd' # Big-data nodes own big data if self.nodetype == 'bd': self.dm = DgramManager(self.xtc_files, configs=self.configs) self.Detector = Detector(self.configs)
def main(): # Process arguments parser = argparse.ArgumentParser() parser.add_argument('pvbase', help='EPICS PV base (e.g. DAQ:LAB2:PART:2)') parser.add_argument('-p', type=int, choices=range(0, 8), default=0, help='platform (default 0)') parser.add_argument('-v', action='store_true', help='be verbose') args = parser.parse_args() if args.v: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') else: logging.basicConfig(level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s') logging.info('control server starting') # CM state yy = ControlStateMachine(args.pvbase) logging.debug("ControlStateMachine state: %s" % yy.state()) # context and sockets ctx = zmq.Context() cmd = ctx.socket(zmq.ROUTER) pull = ctx.socket(zmq.PULL) cmd.bind("tcp://*:%d" % ControlMsg.router_port(args.p)) pull.bind("tcp://*:%d" % ControlMsg.pull_port(args.p)) sequence = 0 poller = zmq.Poller() poller.register(cmd, zmq.POLLIN) poller.register(pull, zmq.POLLIN) try: while True: items = dict(poller.poll(1000)) # Handle pull socket if pull in items: msg = pull.recv() config = dgram.Dgram(view=msg) # now it's in dgram.Dgram object ttt = config.seq.timestamp() print('Timestamp:', ttt) # FIXME # Execute state cmd request if cmd in items: msg = cmd.recv_multipart() identity = msg[0] request = msg[1] logging.debug('Received <%s> from cmd' % request.decode()) if request == ControlMsg.PING: # Send reply to client logging.debug("Sending <PONG> reply") cmd.send(identity, zmq.SNDMORE) cmmsg = ControlMsg(sequence, key=ControlMsg.PONG) cmmsg.send(cmd) continue if request == ControlMsg.PONG: continue if request in [ Transition.configure, Transition.beginrun, Transition.enable, Transition.disable, Transition.endrun, Transition.unconfigure, ControlMsg.GETSTATE ]: if request != ControlMsg.GETSTATE: oldstate = yy.state() # Do transition yy.on_transition(request) newstate = yy.state() if newstate != oldstate: logging.debug("ControlStateMachine state: %s" % newstate) # Send reply to client cmd.send(identity, zmq.SNDMORE) cmmsg = ControlMsg(sequence, key=yy.state().key()) cmmsg.send(cmd) continue else: logging.warning("Unknown msg <%s>" % request.decode()) # Send reply to client logging.debug("Sending <HUH?> reply") cmd.send(identity, zmq.SNDMORE) cmmsg = ControlMsg(sequence, key=ControlMsg.HUH) cmmsg.send(cmd) continue except KeyboardInterrupt: logging.debug("Interrupt received") # Clean up logging.debug("Clean up") time.sleep(.25) # close zmq sockets cmd.close() pull.close() # terminate zmq context ctx.term() logging.info('control server exiting')
def _get_offset_and_size(self): """ Use fast step-through to read off offset and size from smd_view. Format of smd_view [ [[d_bytes][d_bytes]....[evt_footer]] <-- 1 event [[d_bytes][d_bytes]....[evt_footer]] [chunk_footer]] """ offset = 0 i_smd = 0 smd_chunk_pf = PacketFooter(view=self.smd_view) dtype = np.int64 # Row - events, col = smd files self.bd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype) self.bd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype) self.smd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype) self.smd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype) self.new_chunk_id_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype) self.cutoff_flag_array = np.ones((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype) self.services = np.zeros(smd_chunk_pf.n_packets, dtype=dtype) smd_aux_sizes = np.zeros(self.n_smd_files, dtype=dtype) self.i_first_L1s = np.zeros(self.n_smd_files, dtype=dtype) + 0xffffff # For comparing if the next dgram should be in the same read current_bd_offsets = np.zeros(self.n_smd_files, dtype=dtype) # Current chunk size (gets reset at boundary) current_bd_chunk_sizes = np.zeros(self.n_smd_files, dtype=dtype) i_evt = 0 while offset < memoryview(self.smd_view).nbytes - memoryview(smd_chunk_pf.footer).nbytes: if i_smd == 0: smd_evt_size = smd_chunk_pf.get_size(i_evt) smd_evt_pf = PacketFooter(view=self.smd_view[offset: offset+smd_evt_size]) smd_aux_sizes[:] = [smd_evt_pf.get_size(i) for i in range(smd_evt_pf.n_packets)] # Only get offset and size of non-missing dgram # TODO: further optimization by looking for the first L1 and read in a big chunk # anything that comes after. Right now, all transitions mark the cutoff points. if smd_aux_sizes[i_smd] == 0: self.cutoff_flag_array[i_evt, i_smd] = 0 else: d = dgram.Dgram(config=self.smd_configs[i_smd], view=self.smd_view, offset=offset) self.smd_offset_array[i_evt, i_smd] = offset self.smd_size_array[i_evt, i_smd] = d._size self.services[i_evt] = d.service() # Any dgrams after the first L1 (as long as they fit in the chunk size) # will be read in together. if self.dm.n_files > 0: if d.service() == TransitionId.L1Accept: if self.i_first_L1s[i_smd] == 0xffffff: self.i_first_L1s[i_smd] = i_evt print(f'i_smd={i_smd} i_first_L1={self.i_first_L1s[i_smd]}') # For SlowUpdate, we need to check if the next dgram gets cutoff elif d.service() == TransitionId.SlowUpdate and hasattr(d, 'chunkinfo'): stream_id = self.dm.get_stream_id(i_smd) _chunk_ids = [getattr(d.chunkinfo[seg_id].chunkinfo, 'chunkid') for seg_id in d.chunkinfo] # There must be only one unique epics var if _chunk_ids: self.new_chunk_id_array[i_evt, i_smd] = _chunk_ids[0] self._get_bd_offset_and_size(d, current_bd_offsets, current_bd_chunk_sizes, i_evt, i_smd, self.i_first_L1s[i_smd]) """ # For L1 with bigdata files, store offset and size found in smd dgrams. # For SlowUpdate, store new chunk id (if found). TODO: check if # we need to always check for epics for SlowUpdate. if d.service() == TransitionId.L1Accept and self.dm.n_files > 0: if i_first_L1 == -1: i_first_L1 = i_evt print(f'i_smd={i_smd} i_first_L1={i_first_L1}') self._get_bd_offset_and_size(d, current_bd_offsets, current_bd_chunk_sizes, i_evt, i_smd, i_first_L1) elif d.service() == TransitionId.SlowUpdate and hasattr(d, 'chunkinfo'): # We only support chunking on bigdata if self.dm.n_files > 0: stream_id = self.dm.get_stream_id(i_smd) _chunk_ids = [getattr(d.chunkinfo[seg_id].chunkinfo, 'chunkid') for seg_id in d.chunkinfo] # There must be only one unique epics var if _chunk_ids: self.new_chunk_id_array[i_evt, i_smd] = _chunk_ids[0] """ offset += smd_aux_sizes[i_smd] i_smd += 1 if i_smd == self.n_smd_files: offset += PacketFooter.n_bytes * (self.n_smd_files + 1) # skip the footer i_smd = 0 # reset to the first smd file i_evt += 1 # done with this smd event # end while offset # Precalculate cutoff indices self.cutoff_indices = [] self.chunk_indices = np.zeros(self.n_smd_files, dtype=dtype) for i_smd in range(self.n_smd_files): self.cutoff_indices.append(np.where(self.cutoff_flag_array[:, i_smd] == 1)[0]) print(f'i_smd={i_smd} cutoff_index={self.cutoff_indices[i_smd]} services={self.services[self.cutoff_indices[i_smd]]}')