def run_smd0_task(run): global_procs = legion.Tunable.select(legion.Tunable.GLOBAL_PYS).get() smdr_man = SmdReaderManager(run.smd_dm.fds, run.max_events) for i, (smd_chunk, update_chunk) in enumerate(smdr_man.chunks()): run_smd_task(smd_chunk, run, point=i) # Block before returning so that the caller can use this task's future for synchronization legion.execution_fence(block=True)
class Smd0(object): """ Sends blocks of smds to smd_node Identifies limit timestamp of the slowest detector then sends all smds within that timestamp to an smd_node. """ def __init__(self, run): self.smdr_man = SmdReaderManager(run) self.run = run self.step_hist = StepHistory(self.run.comms.smd_size, len(self.run.configs)) self.run_mpi() def run_mpi(self): rankreq = np.empty(1, dtype='i') for (smd_chunk, step_chunk) in self.smdr_man.chunks(): # Creates a chunk from smd and epics data to send to SmdNode # Anatomy of a chunk (pf=packet_footer): # [ [smd0][smd1][smd2][pf] ][ [epics0][epics1][epics2][pf] ][ pf ] # ----- smd_chunk ------ ---------epics_chunk------- # -------------------------- chunk ------------------------------ # Read new epics data as available in the queue # then send only unseen portion of data to the evtbuilder rank. if not smd_chunk: break self.run.comms.smd_comm.Recv(rankreq, source=MPI.ANY_SOURCE) # Check missing steps for the current client missing_step_views = self.step_hist.get_buffer(rankreq[0]) # Update step buffers (after getting the missing steps step_pf = PacketFooter(view=step_chunk) step_views = step_pf.split_packets() self.step_hist.extend_buffers(step_views, rankreq[0]) smd_extended = repack_for_eb(smd_chunk, missing_step_views, self.run.configs) self.run.comms.smd_comm.Send(smd_extended, dest=rankreq[0]) for i in range(self.run.comms.n_smd_nodes): self.run.comms.smd_comm.Recv(rankreq, source=MPI.ANY_SOURCE) self.run.comms.smd_comm.Send(bytearray(), dest=rankreq[0])
def steps(self): current_step_pos = 0 """ Generates events between steps. """ smdr_man = SmdReaderManager(self.smd_dm.fds, self.max_events) for i, (smd_chunk, step_chunk) in enumerate(smdr_man.chunks()): # Update step stores step_pf = PacketFooter(view=step_chunk) step_views = step_pf.split_packets() self.epics_store.update(step_views) self.step_store.update(step_views) eb_man = EventBuilderManager(smd_chunk, self.configs, \ batch_size=self.batch_size, filter_fn=self.filter_callback) for i, step_dgram in enumerate( self.step_store.dgrams(from_pos=current_step_pos + 1)): if step_dgram: limit_ts = step_dgram.seq.timestamp() current_step_pos += 1 else: limit_ts = -1 yield Step(self, eb_man=eb_man, limit_ts=limit_ts)
class Smd0(object): """ Sends blocks of smds to smd_node Identifies limit timestamp of the slowest detector then sends all smds within that timestamp to an smd_node. """ def __init__(self, run): self.smdr_man = SmdReaderManager(run.smd_dm.fds, run.max_events) self.run = run self.epics_man = UpdateManager(smd_size, self.run.epics_store.n_files) self.run_mpi() def run_mpi(self): rankreq = np.empty(1, dtype='i') for (smd_chunk, update_chunk) in self.smdr_man.chunks(): # Creates a chunk from smd and epics data to send to SmdNode # Anatomy of a chunk (pf=packet_footer): # [ [smd0][smd1][smd2][pf] ][ [epics0][epics1][epics2][pf] ][ pf ] # ----- smd_chunk ------ ---------epics_chunk------- # -------------------------- chunk ------------------------------ # Read new epics data as available in the queue # then send only unseen portion of data to the evtbuilder rank. update_pf = PacketFooter(view=update_chunk) self.epics_man.extend_buffers(update_pf.split_packets()) smd_comm.Recv(rankreq, source=MPI.ANY_SOURCE) epics_chunk = self.epics_man.get_buffer(rankreq[0]) pf = PacketFooter(2) pf.set_size(0, memoryview(smd_chunk).shape[0]) pf.set_size(1, memoryview(epics_chunk).shape[0]) chunk = smd_chunk + epics_chunk + pf.footer smd_comm.Send(chunk, dest=rankreq[0]) for i in range(PS_SMD_NODES): smd_comm.Recv(rankreq, source=MPI.ANY_SOURCE) smd_comm.Send(bytearray(), dest=rankreq[0])
def events(self): ev_man = EventManager(self.configs, self.dm, \ filter_fn=self.filter_callback) #get smd chunks smdr_man = SmdReaderManager(self.smd_dm.fds, self.max_events) for (smd_chunk, step_chunk) in smdr_man.chunks(): # Update epics_store for each chunk step_pf = PacketFooter(view=step_chunk) step_views = step_pf.split_packets() self.epics_store.update(step_views) eb_man = EventBuilderManager(smd_chunk, self.configs, batch_size=self.batch_size, filter_fn=self.filter_callback) for batch_dict in eb_man.batches(): batch, _ = batch_dict[ 0] # there's only 1 dest_rank for serial run for evt in ev_man.events(batch): if evt._dgrams[0].seq.service() != 12: continue yield evt
def smd_chunks(run): smdr_man = SmdReaderManager(run) for smd_chunk, update_chunk in smdr_man.chunks(): yield smd_chunk
import os, glob from psana.psexp.smdreader_manager import SmdReaderManager class Container(object): pass class Run(object): def __init__(self): filenames = glob.glob( '/reg/neh/home/monarin/lcls2/psana/psana/tests/.tmp_smd0/.tmp/smalldata/*.xtc2' ) fds = [os.open(f, os.O_RDONLY) for f in filenames] self.smd_dm = Container() setattr(self.smd_dm, 'fds', fds) self.max_events = 1000 if __name__ == "__main__": run = Run() os.environ['PS_SMD_N_EVENTS'] = "1" smdr_man = SmdReaderManager(run) for chunk in smdr_man.chunks(): smd_chunk, step_chunk = chunk print( f'{memoryview(smd_chunk).nbytes} {memoryview(step_chunk).nbytes}')