def smd_0(fds, n_smd_nodes): """ Sends blocks of smds to smd_node Identifies limit timestamp of the slowest detector then sends all smds within that timestamp to an smd_node. """ assert len(fds) > 0 smdr = SmdReader(fds) got_events = -1 rankreq = np.empty(1, dtype='i') while got_events != 0: smdr.get(n_events) got_events = smdr.got_events views = bytearray() for i in range(len(fds)): view = smdr.view(i) if view != 0: views.extend(view) if i < len(fds) - 1: views.extend(b'endofstream') if views: comm.Recv(rankreq, source=MPI.ANY_SOURCE) comm.Send(views, dest=rankreq[0], tag=12) for i in range(n_smd_nodes): comm.Recv(rankreq, source=MPI.ANY_SOURCE) comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12)
def run_smd0(fds, max_events=0): n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100)) if max_events: if max_events < n_events: n_events = max_events smdr = SmdReader(fds) got_events = -1 processed_events = 0 while got_events != 0: smdr.get(n_events) got_events = smdr.got_events processed_events += got_events print("processed_events", processed_events, "got_events", got_events) views = bytearray() for i in range(len(fds)): view = smdr.view(i) if view != 0: views.extend(view) if i < len(fds) - 1: views.extend(b'endofstream') if views: #run_smd_task(views, run) print("got views") if max_events: if processed_events >= max_events: break """
class SmdReaderManager(object): def __init__(self, fds, max_events): self.n_files = len(fds) assert self.n_files > 0 self.smdr = SmdReader(fds) self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000)) self.max_events = max_events self.processed_events = 0 if self.max_events: if self.max_events < self.n_events: self.n_events = self.max_events def chunks(self): """ Generates a tuple of smd and update dgrams """ got_events = -1 while got_events != 0: self.smdr.get(self.n_events) got_events = self.smdr.got_events self.processed_events += got_events smd_view = bytearray() smd_pf = PacketFooter(n_packets=self.n_files) update_view = bytearray() update_pf = PacketFooter(n_packets=self.n_files) for i in range(self.n_files): _smd_view = self.smdr.view(i) if _smd_view != 0: smd_view.extend(_smd_view) smd_pf.set_size(i, memoryview(_smd_view).shape[0]) _update_view = self.smdr.view(i, update=True) if _update_view != 0: update_view.extend(_update_view) update_pf.set_size(i, memoryview(_update_view).shape[0]) if smd_view or update_view: if smd_view: smd_view.extend(smd_pf.footer) if update_view: update_view.extend(update_pf.footer) yield (smd_view, update_view) if self.max_events: if self.processed_events >= self.max_events: break @property def min_ts(self): return self.smdr.min_ts @property def max_ts(self): return self.smdr.max_ts
def run_smd0(n_events): filenames = glob.glob(os.path.join(xtc_dir, '.tmp', 'smalldata', '*.xtc2')) fds = [os.open(filename, os.O_RDONLY) for filename in filenames] # Move file ptrs to datagram part configs = [Dgram(file_descriptor=fd) for fd in fds] limit = len(filenames) if len(sys.argv) > 1: limit = int(sys.argv[1]) st = time.time() smdr = SmdReader(fds[:limit]) got_events = -1 processed_events = 0 smdr.get(n_events) got_events = smdr.got_events result = {'each_read': [], 'total_n_events': 0} cn_i = 0 while got_events != 0: step_chunk_nbytes = 0 smd_chunk_nbytes = 0 for i in range(limit): smd_view = smdr.view(i) if smd_view: smd_chunk_nbytes += smd_view.nbytes step_view = smdr.view(i, update=True) if step_view: step_chunk_nbytes += step_view.nbytes result['each_read'].append( [got_events, smd_chunk_nbytes, step_chunk_nbytes]) processed_events += got_events # Read more events smdr.get(n_events) got_events = smdr.got_events cn_i += 1 en = time.time() result['total_n_events'] = processed_events for fd in fds: os.close(fd) return result
def smd_0(fds, n_smd_nodes, max_events=0): """ Sends blocks of smds to smd_node Identifies limit timestamp of the slowest detector then sends all smds within that timestamp to an smd_node. """ assert len(fds) > 0 smdr = SmdReader(fds) got_events = -1 processed_events = 0 rankreq = np.empty(1, dtype='i') n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100)) if max_events: if max_events < n_events: n_events = max_events while got_events != 0: smdr.get(n_events) got_events = smdr.got_events processed_events += got_events views = bytearray() for i in range(len(fds)): view = smdr.view(i) if view != 0: views.extend(view) if i < len(fds) - 1: views.extend(b'endofstream') if views: comm.Recv(rankreq, source=MPI.ANY_SOURCE) comm.Send(views, dest=rankreq[0], tag=12) if max_events: if processed_events == max_events: break for i in range(n_smd_nodes): comm.Recv(rankreq, source=MPI.ANY_SOURCE) comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12)
def run_smd0(): #filenames = glob.glob('/reg/neh/home/monarin/psana-nersc/psana2/.tmp/smalldata/*.xtc2') filenames = glob.glob('/ffb01/mona/.tmp/smalldata/*.xtc2') fds = np.array([os.open(filename, os.O_RDONLY) for filename in filenames], dtype=np.int32) # Move file ptrs to datagram part configs = [Dgram(file_descriptor=fd) for fd in fds] beginRun = [Dgram(config=config) for config in configs] limit = len(filenames) if len(sys.argv) > 1: limit = int(sys.argv[1]) st = time.time() smdr = SmdReader(fds[:limit], chunksize) got_events = -1 processed_events = 0 offsets = np.zeros(limit, dtype=np.uint64) how_many = smd0_batch_size to_be_read = max_events - processed_events if to_be_read < how_many: how_many = to_be_read smdr.get(how_many) while smdr.got_events > 0: for i in range(limit): view = smdr.view(i) """ if view: cn_dgrams = 0 while offsets[i] < view.shape[0]: d = Dgram(config=configs[i], view=view, offset=offsets[i]) print(f' buf{i} d_id: {cn_dgrams} d_ts {d.timestamp() & 0xffffffff}') offsets[i] += d._size cn_dgrams += 1 #print(f'smdr_man got {memoryview(view).nbytes}') else: #print(f' buf[{i} empty') pass """ processed_events += smdr.got_events if processed_events >= max_events: break how_many = smd0_batch_size to_be_read = max_events - processed_events if to_be_read < how_many: how_many = to_be_read smdr.get(how_many) offsets[:] = 0 """ while smdr.got_events != 0: if smdr.got_events > 0: processed_events += smdr.got_events if processed_events >= max_events: break for i in range(limit): smdr.view(i) smdr.get(n_events) print(f'smdr.got_events={smdr.got_events}') """ en = time.time() print("#Events: %d Elapsed Time (s): %f Rate (MHz): %f" % (processed_events, (en - st), processed_events / ((en - st) * 1e6)))
import dgram from psana import dgram as psana_dgram from psana.smdreader import SmdReader import os fd = os.open('data-r0001-s00.smd.xtc2', os.O_RDONLY) config = psana_dgram.Dgram(file_descriptor=fd) smdr = SmdReader([fd]) smdr.get(1) print(smdr.got_events) dgram.get(smdr.view(0))
exit() n_files = int(sys.argv[1]) fds = [os.open(xtc_file, os.O_RDONLY) for xtc_file in xtc_files] # do this to move to dgram part of the xtc # since config doesn't have correct timestamp so the # code won't work - won't happen in real situation configs = [dgram.Dgram(file_descriptor=fd) for fd in fds] st = time.time() smdr = DummyReader(fds[:n_files]) got_events = -1 processed_events = 0 while got_events != 0: smdr.get(1000) got_events = smdr.got_events processed_events += got_events for i in range(n_files): view = smdr.view(i) #if processed_events >= 1000: break en = time.time() print("processed_events %d total elapsed %f s rate %f MHz" % (processed_events, en - st, processed_events / ((en - st) * 1000000))) #print("DeltaT get_init(s): %f"%(smdr.dt_get_init)) #print("DeltaT get_dgram(s): %f"%(smdr.dt_get_dgram)) #print("DeltaT reread(s): %f"%(smdr.dt_reread)) total = en - st print(total)
class SmdReaderManager(object): def __init__(self, smd_fds, dsparms, configs=None): self.n_files = len(smd_fds) self.dsparms = dsparms self.configs = configs assert self.n_files > 0 self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000)) if self.dsparms.max_events: if self.dsparms.max_events < self.smd0_n_events: self.smd0_n_events = self.dsparms.max_events self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000)) self.smdr = SmdReader(smd_fds, self.chunksize, self.dsparms.max_retries) self.processed_events = 0 self.got_events = -1 self._run = None # Collecting Smd0 performance using prometheus self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read') def _get(self): st = time.time() self.smdr.get() en = time.time() logger.debug( f'smdreader_manager: read {self.smdr.got/1e6:.5f} MB took {en-st}s. rate: {self.smdr.got/(1e6*(en-st))} MB/s' ) self.c_read.labels('MB', 'None').inc(self.smdr.got / 1e6) self.c_read.labels('seconds', 'None').inc(en - st) if self.smdr.chunk_overflown > 0: msg = f"SmdReader found dgram ({self.smdr.chunk_overflown} MB) larger than chunksize ({self.chunksize/1e6} MB)" raise ValueError(msg) def get_next_dgrams(self): if self.dsparms.max_events > 0 and \ self.processed_events >= self.dsparms.max_events: logger.debug( f'smdreader_manager: get_next_dgrams max_events={self.dsparms.max_events} reached' ) return None dgrams = None if not self.smdr.is_complete(): self._get() if self.smdr.is_complete(): mmrv_bufs, _ = self.smdr.view(batch_size=1) # For configs, we need to copy data from smdreader's buffers # This prevents it from getting overwritten by other dgrams. bytearray_bufs = [bytearray(mmrv_buf) for mmrv_buf in mmrv_bufs] if self.configs is None: dgrams = [ dgram.Dgram(view=ba_buf, offset=0) for ba_buf in bytearray_bufs ] self.configs = dgrams else: dgrams = [ dgram.Dgram(view=ba_buf, config=config, offset=0) for ba_buf, config in zip(bytearray_bufs, self.configs) ] return dgrams def __iter__(self): return self def __next__(self): """ Returns a batch of events as an iterator object. This is used by non-parallel run. Parallel run uses chunks generator that yields chunks of raw smd data and steps (no event building). The iterator stops reading under two conditions. Either there's no more data or max_events reached. """ if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events: raise StopIteration if not self.smdr.is_complete(): self._get() if not self.smdr.is_complete(): raise StopIteration mmrv_bufs, _ = self.smdr.view(batch_size=self.smd0_n_events) batch_iter = BatchIterator(mmrv_bufs, self.configs, self._run, batch_size=self.dsparms.batch_size, filter_fn=self.dsparms.filter, destination=self.dsparms.destination) self.got_events = self.smdr.view_size self.processed_events += self.got_events return batch_iter def chunks(self): """ Generates a tuple of smd and step dgrams """ is_done = False while not is_done: if self.smdr.is_complete(): mmrv_bufs, mmrv_step_bufs = self.smdr.view( batch_size=self.smd0_n_events) self.got_events = self.smdr.view_size self.processed_events += self.got_events # sending data to prometheus logger.debug('smdreader_manager: smd0 got %d events' % (self.got_events)) if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events: logger.debug( f'smdreader_manager: max_events={self.dsparms.max_events} reached' ) is_done = True smd_view = bytearray() smd_pf = PacketFooter(n_packets=self.n_files) step_view = bytearray() step_pf = PacketFooter(n_packets=self.n_files) for i, (mmrv_buf, mmrv_step_buf) in enumerate( zip(mmrv_bufs, mmrv_step_bufs)): if mmrv_buf != 0: smd_view.extend(mmrv_buf) smd_pf.set_size(i, memoryview(mmrv_buf).nbytes) if mmrv_step_buf != 0: step_view.extend(mmrv_step_buf) step_pf.set_size(i, memoryview(mmrv_step_buf).nbytes) if smd_view or step_view: if smd_view: smd_view.extend(smd_pf.footer) if step_view: step_view.extend(step_pf.footer) yield (smd_view, step_view) else: # if self.smdr.is_complete() self._get() if not self.smdr.is_complete(): is_done = True break @property def min_ts(self): return self.smdr.min_ts @property def max_ts(self): return self.smdr.max_ts def set_run(self, run): self._run = run def get_run(self): return self._run
class SmdReaderManager(object): def __init__(self, run): self.n_files = len(run.smd_dm.fds) assert self.n_files > 0 self.run = run self.smdr = SmdReader(run.smd_dm.fds) self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000)) self.processed_events = 0 if self.run.max_events: if self.run.max_events < self.n_events: self.n_events = self.run.max_events self.got_events = -1 def __iter__(self): return self def __next__(self): if self.got_events == 0: raise StopIteration self.smdr.get(self.n_events) self.got_events = self.smdr.got_events self.processed_events += self.got_events views = [self.smdr.view(i) for i in range(self.n_files)] batch_iter = BatchIterator(views, batch_size=self.run.batch_size, \ filter_fn=self.run.filter_callback, destination=self.run.destination) if self.run.max_events: if self.processed_events >= self.run.max_events: self.got_events = 0 return batch_iter def chunks(self): """ Generates a tuple of smd and step dgrams """ got_events = -1 while got_events != 0: self.smdr.get(self.n_events) got_events = self.smdr.got_events self.processed_events += got_events smd_view = bytearray() smd_pf = PacketFooter(n_packets=self.n_files) step_view = bytearray() step_pf = PacketFooter(n_packets=self.n_files) for i in range(self.n_files): _smd_view = self.smdr.view(i) if _smd_view != 0: smd_view.extend(_smd_view) smd_pf.set_size(i, memoryview(_smd_view).shape[0]) _step_view = self.smdr.view(i, step=True) if _step_view != 0: step_view.extend(_step_view) step_pf.set_size(i, memoryview(_step_view).shape[0]) if smd_view or step_view: if smd_view: smd_view.extend(smd_pf.footer) if step_view: step_view.extend(step_pf.footer) yield (smd_view, step_view) if self.run.max_events: if self.processed_events >= self.run.max_events: break @property def min_ts(self): return self.smdr.min_ts @property def max_ts(self): return self.smdr.max_ts
class SmdReaderManager(object): def __init__(self, smd_fds, dsparms, configs=None): self.n_files = len(smd_fds) self.dsparms = dsparms self.configs = configs assert self.n_files > 0 # Sets no. of events Smd0 sends to each EventBuilder core. This gets # overridden by max_events set by DataSource if max_events is smaller. self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000)) if self.dsparms.max_events: if self.dsparms.max_events < self.smd0_n_events: self.smd0_n_events = self.dsparms.max_events # Sets the memory size for smalldata buffer for each stream file. self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000)) self.smdr = SmdReader(smd_fds, self.chunksize, self.dsparms.max_retries) self.processed_events = 0 self.got_events = -1 self._run = None # Collecting Smd0 performance using prometheus self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read') def _get(self): st = time.monotonic() self.smdr.get(self.dsparms.smd_inprogress_converted) en = time.monotonic() logger.debug( f'read {self.smdr.got/1e6:.3f} MB took {en-st}s. rate: {self.smdr.got/(1e6*(en-st))} MB/s' ) self.c_read.labels('MB', 'None').inc(self.smdr.got / 1e6) self.c_read.labels('seconds', 'None').inc(en - st) if self.smdr.chunk_overflown > 0: msg = f"SmdReader found dgram ({self.smdr.chunk_overflown} MB) larger than chunksize ({self.chunksize/1e6} MB)" raise ValueError(msg) def get_next_dgrams(self): """ Returns list of dgrams as appeared in the current offset of the smd chunks. Currently used to retrieve Configure and BeginRun. This allows read with wait for these two types of dgram. """ if self.dsparms.max_events > 0 and \ self.processed_events >= self.dsparms.max_events: logger.debug(f'max_events={self.dsparms.max_events} reached') return None dgrams = None if not self.smdr.is_complete(): self._get() if self.smdr.is_complete(): # Get chunks with only one dgram each. There's no need to set # integrating stream id here since Configure and BeginRun # must exist in this stream too. self.smdr.view(batch_size=1) # For configs, we need to copy data from smdreader's buffers # This prevents it from getting overwritten by other dgrams. bytearray_bufs = [ bytearray(self.smdr.show(i)) for i in range(self.n_files) ] if self.configs is None: dgrams = [ dgram.Dgram(view=ba_buf, offset=0) for ba_buf in bytearray_bufs ] self.configs = dgrams self.smdr.set_configs(self.configs) else: dgrams = [ dgram.Dgram(view=ba_buf, config=config, offset=0) for ba_buf, config in zip(bytearray_bufs, self.configs) ] return dgrams def __iter__(self): return self def __next__(self): """ Returns a batch of events as an iterator object. This is used by non-parallel run. Parallel run uses chunks generator that yields chunks of raw smd data and steps (no event building). The iterator stops reading under two conditions. Either there's no more data or max_events reached. """ intg_stream_id = self.dsparms.intg_stream_id if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events: raise StopIteration if not self.smdr.is_complete(): self._get() if not self.smdr.is_complete(): raise StopIteration self.smdr.view(batch_size=self.smd0_n_events, intg_stream_id=intg_stream_id) mmrv_bufs = [self.smdr.show(i) for i in range(self.n_files)] batch_iter = BatchIterator(mmrv_bufs, self.configs, self._run, batch_size=self.dsparms.batch_size, filter_fn=self.dsparms.filter, destination=self.dsparms.destination, timestamps=self.dsparms.timestamps, intg_stream_id=intg_stream_id) self.got_events = self.smdr.view_size self.processed_events += self.got_events return batch_iter def chunks(self): """ Generates a tuple of smd and step dgrams """ is_done = False d_view, d_read = 0, 0 cn_chunks = 0 while not is_done: logger.debug(f'SMD0 1. STARTCHUNK {time.monotonic()}') st_view, en_view, st_read, en_read = 0, 0, 0, 0 l1_size = 0 tr_size = 0 got_events = 0 if self.smdr.is_complete(): st_view = time.monotonic() # Gets the next batch of already read-in data. self.smdr.view(batch_size=self.smd0_n_events, intg_stream_id=self.dsparms.intg_stream_id) self.got_events = self.smdr.view_size got_events = self.got_events self.processed_events += self.got_events # sending data to prometheus logger.debug('got %d events' % (self.got_events)) if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events: logger.debug( f'max_events={self.dsparms.max_events} reached') is_done = True en_view = time.monotonic() d_view += en_view - st_view logger.debug(f'SMD0 2. DONECREATEVIEW {time.monotonic()}') if self.got_events: cn_chunks += 1 yield cn_chunks else: # if self.smdr.is_complete() st_read = time.monotonic() self._get() en_read = time.monotonic() logger.debug(f'SMD0 3. DONEREAD {time.monotonic()}') d_read += en_read - st_read if not self.smdr.is_complete(): is_done = True break @property def min_ts(self): return self.smdr.min_ts @property def max_ts(self): return self.smdr.max_ts def set_run(self, run): self._run = run def get_run(self): return self._run
class Smd0(Node): """ Sends blocks of smds to smd_node Identifies limit timestamp of the slowest detector then sends all smds within that timestamp to an smd_node. """ def __init__(self, ds): Node.__init__(self, ds.mpi) self.fds = ds.smd_dm.fds assert len(self.fds) > 0 self.smdr = SmdReader(self.fds) self.n_smd_nodes = ds.nsmds self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100)) self.max_events = ds.max_events if self.max_events: if self.max_events < self.n_events: self.n_events = self.max_events if mode == 'mpi': self.run_mpi() elif mode == 'legion': self.run_legion() def run_mpi(self): rank = self.mpi.rank comm = self.mpi.comm rankreq = np.empty(1, dtype='i') for chunk in self.chunks(): comm.Recv(rankreq, source=MPI.ANY_SOURCE) comm.Send(chunk, dest=rankreq[0], tag=12) for i in range(self.n_smd_nodes): comm.Recv(rankreq, source=MPI.ANY_SOURCE) comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12) def run_legion(self, configs, batch_size=1, filter=0): for chunk in self.chunks(): SmdNode.run_legion_task(chunk, configs, batch_size, filter) def chunks(self): processed_events = 0 got_events = -1 while got_events != 0: self.smdr.get(self.n_events) got_events = self.smdr.got_events processed_events += got_events views = bytearray() for i in range(len(self.fds)): view = self.smdr.view(i) if view != 0: views.extend(view) if i < len(self.fds) - 1: views.extend(b'endofstream') if views: yield views if self.max_events: if processed_events == self.max_events: break
class SmdReaderManager(object): def __init__(self, run): self.n_files = len(run.smd_fds) assert self.n_files > 0 self.run = run self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 1000)) if self.run.max_events: if self.run.max_events < self.batch_size: self.batch_size = self.run.max_events self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x100000)) self.smdr = SmdReader(run.smd_fds, self.chunksize) self.processed_events = 0 self.got_events = -1 # Collecting Smd0 performance using prometheus if self.run.prom_man: self.c_read = self.run.prom_man.get_counter('psana_smd0_read') def get_next_dgrams(self, configs=None): dgrams = None if not self.smdr.is_complete(): self.smdr.get() if self.smdr.is_complete(): mmrv_bufs, _ = self.smdr.view(batch_size=1) # For configs, we need to copy data from smdreader's buffers # This prevents it from getting overwritten by other dgrams. bytearray_bufs = [bytearray(mmrv_buf) for mmrv_buf in mmrv_bufs] if configs is None: dgrams = [dgram.Dgram(view=ba_buf, offset=0) for ba_buf in bytearray_bufs] else: dgrams = [dgram.Dgram(view=ba_buf, config=config, offset=0) for ba_buf, config in zip(bytearray_bufs, configs)] return dgrams def __iter__(self): return self def __next__(self): """ Returns a batch of events as an iterator object. This is used by non-parallel run. Parallel run uses chunks generator that yields chunks of raw smd data and steps (no event building). The iterator stops reading under two conditions. Either there's no more data or max_events reached. """ if self.run.max_events and self.processed_events >= self.run.max_events: raise StopIteration if not self.smdr.is_complete(): self.smdr.get() if self.run.prom_man: self.c_read.labels('MB', 'None').inc(self.smdr.got/1e6) if not self.smdr.is_complete(): raise StopIteration mmrv_bufs, _ = self.smdr.view(batch_size=self.batch_size) batch_iter = BatchIterator(mmrv_bufs, batch_size=self.run.batch_size, \ filter_fn=self.run.filter_callback, destination=self.run.destination) self.got_events = self.smdr.view_size self.processed_events += self.got_events # sending data to prometheus if self.run.prom_man: self.c_read.labels('evts', 'None').inc(self.got_events) self.c_read.labels('batches', 'None').inc() return batch_iter def chunks(self): """ Generates a tuple of smd and step dgrams """ is_done = False while not is_done: if self.smdr.is_complete(): mmrv_bufs, mmrv_step_bufs = self.smdr.view(batch_size=self.batch_size) self.got_events = self.smdr.view_size self.processed_events += self.got_events # sending data to prometheus if self.run.prom_man: logging.debug('Smd0 got %d events'%(self.got_events)) self.c_read.labels('evts', 'None').inc(self.got_events) self.c_read.labels('batches', 'None').inc() if self.run.max_events and self.processed_events >= self.run.max_events: is_done = True smd_view = bytearray() smd_pf = PacketFooter(n_packets=self.n_files) step_view = bytearray() step_pf = PacketFooter(n_packets=self.n_files) for i, (mmrv_buf, mmrv_step_buf) in enumerate(zip(mmrv_bufs, mmrv_step_bufs)): if mmrv_buf != 0: smd_view.extend(mmrv_buf) smd_pf.set_size(i, memoryview(mmrv_buf).nbytes) if mmrv_step_buf != 0: step_view.extend(mmrv_step_buf) step_pf.set_size(i, memoryview(mmrv_step_buf).nbytes) if smd_view or step_view: if smd_view: smd_view.extend(smd_pf.footer) if step_view: step_view.extend(step_pf.footer) yield (smd_view, step_view) else: self.smdr.get() if self.run.prom_man: logging.debug('Smd0 read %.2f MB'%(self.smdr.got/1e6)) self.c_read.labels('MB', 'None').inc(self.smdr.got/1e6) if not self.smdr.is_complete(): is_done = True break @property def min_ts(self): return self.smdr.min_ts @property def max_ts(self): return self.smdr.max_ts
class SmdReaderManager(object): def __init__(self, run): self.n_files = len(run.smd_dm.fds) assert self.n_files > 0 self.run = run self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 13500 * 16)) if self.run.max_events: if self.run.max_events < self.batch_size: self.batch_size = self.run.max_events self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000)) self.smdr = SmdReader(run.smd_dm.fds, self.chunksize) self.processed_events = 0 self.got_events = -1 def __iter__(self): return self def _read(self): """ Reads 'batch_size' no. of events. If they don't fit in the 'chunksize', returns only events that fit. If user specifies max_events when creating DataSource, only asks SmdReader to read this amount of events (see how_many is being set below). """ max_retries = int(os.environ.get('PS_SMD_MAX_RETRIES', '5')) sleep_secs = int(os.environ.get('PS_SMD_SLEEP_SECS', '1')) how_many = self.batch_size if self.run.max_events: to_be_read = self.run.max_events - self.processed_events if to_be_read < how_many: how_many = to_be_read self.smdr.get(how_many) cn_retries = 0 while self.smdr.got_events == 0: self.smdr.retry() cn_retries += 1 if cn_retries == max_retries: break time.sleep(sleep_secs) self.got_events = self.smdr.got_events self.processed_events += self.got_events def __next__(self): """ Returns a batch of events as an iterator object. This is used by non-parallel run. Parallel run uses chunks generator that yields chunks of raw smd data and steps (no event building). The iterator stops reading under two conditions. Either there's nothing to read (_read() comes back with got_events=0) or no. of processed_events = max_events (specified at DataSource). """ if self.run.max_events: if self.processed_events >= self.run.max_events: raise StopIteration self._read() if self.got_events == 0: raise StopIteration views = [] for i in range(self.n_files): view = self.smdr.view(i) if view: views.append(view) else: views.append(memoryview(bytearray())) batch_iter = BatchIterator(views, batch_size=self.run.batch_size, \ filter_fn=self.run.filter_callback, destination=self.run.destination) return batch_iter def chunks(self): """ Generates a tuple of smd and step dgrams """ self._read() while self.got_events > 0: smd_view = bytearray() smd_pf = PacketFooter(n_packets=self.n_files) step_view = bytearray() step_pf = PacketFooter(n_packets=self.n_files) for i in range(self.n_files): _smd_view = self.smdr.view(i) if _smd_view != 0: smd_view.extend(_smd_view) smd_pf.set_size(i, memoryview(_smd_view).shape[0]) _step_view = self.smdr.view(i, step=True) if _step_view != 0: step_view.extend(_step_view) step_pf.set_size(i, memoryview(_step_view).shape[0]) if smd_view or step_view: if smd_view: smd_view.extend(smd_pf.footer) if step_view: step_view.extend(step_pf.footer) yield (smd_view, step_view) if self.run.max_events: if self.processed_events >= self.run.max_events: break self._read() @property def min_ts(self): return self.smdr.min_ts @property def max_ts(self): return self.smdr.max_ts