def __init__(self, name, app, wait_for_initial_tokens=False): super().__init__(name, app) log.debug("initialize new dataflow runtime process (%s)", self.full_name) app_trace = app.trace self._channels = {} self._current_segment = None self._remaining_compute_cycles = None # a seekable iterator over all segments in the process trace self._trace = more_itertools.seekable(app_trace.get_trace(name), maxlen=16) self._current_segment = None self._remaining_compute_cycles = None # keep track of the total cycles to process and the sum of cycles # already processed self._total_cycles = app_trace.accumulate_processor_cycles(name) self._total_cycles_processed = { p: 0 for p in self._total_cycles.keys() } # lets the workload method know whether it is run for the first time # or whether it is resumed self._is_running = False self._wait_for_initial_tokens = wait_for_initial_tokens
def __init__(self, iterable: Iterable, is_cached: bool = True, is_error_suppress: bool = True): iterable = always_iterable(iterable) self.is_cached = is_cached self.is_error_suppress = is_error_suppress self.iterable = seekable(iterable) if self.is_cached else iterable
def process(codes): it = seekable(enumerate(codes)) for pos, [opcode_num] in it: func, params, offset = parse_opcode_num(codes, opcode_num, pos) try: result = func(codes, *params) or {} it.seek(result.get('seek_to', pos + offset)) except Halt: return
def _run(self): it = seekable(enumerate(self.codes)) for pos, [opcode_num] in it: func, params, offset, opcode = self._parse_opcode_num( self.codes, opcode_num, pos) yield {'opcode': opcode} try: result = func(self.codes, *params) or {} it.seek(result.get('seek_to', pos + offset)) except self.Halt: return
def test_workload_wait_for_initial_token( self, env, dataflow_process, processor, empty_channel ): dataflow_process._wait_for_initial_tokens = True dataflow_process._trace = more_itertools.seekable( self.initial_read_trace_generator(), maxlen=16 ) env.run() dataflow_process._channels["chan"] = weakref.ref(empty_channel) dataflow_process.start() env.run() assert dataflow_process._state == ProcessState.BLOCKED
def read(instance, limit=8): data = [] try: rows = mit.seekable(instance.data.iter(size=limit, sort="row_no")) rows.seek(0) column_names = instance.data.headers_map.keys() for count, row in enumerate(rows, start=1): data.append(make_row(column_names, row)) if count == limit: break except Exception: pass return data
def test_workload_read_block( self, env, dataflow_process, processor, empty_channel ): dataflow_process._trace = more_itertools.seekable( self.read_trace_generator(), maxlen=16 ) env.run() dataflow_process._channels["chan"] = weakref.ref(empty_channel) dataflow_process.start() env.run() dataflow_process.activate(processor) env.run() finished = env.process(dataflow_process.workload()) env.run(finished) assert dataflow_process._state == ProcessState.BLOCKED assert env.now == 1
def test_preemption(self, env, dataflow_process, processor, processor2): # monkey patch the process to add a trace dataflow_process._trace = more_itertools.seekable( self.preemption_trace_generator(), maxlen=16 ) dataflow_process._total_cycles = {"Test": 10, "Test2": 20} dataflow_process._remaining_compute_cycles = {"Test": 0, "Test2": 0} env.run() dataflow_process.start() env.run() dataflow_process.activate(processor) assert dataflow_process.processor is processor env.run() env.process(dataflow_process.workload()) env.run(5) dataflow_process.preempt() env.run(10) assert dataflow_process.processor is None assert dataflow_process._remaining_compute_cycles["Test"] == 5 assert dataflow_process._remaining_compute_cycles["Test2"] == 10 assert dataflow_process.get_progress() == 0.5 # continue execution on processor2 for 5 cycles (10 ticks) dataflow_process.activate(processor2) env.run(15) env.process(dataflow_process.workload()) assert dataflow_process.processor is processor2 env.run(25) dataflow_process.preempt() env.run(26) assert dataflow_process.processor is None assert dataflow_process._remaining_compute_cycles["Test"] == 3 assert dataflow_process._remaining_compute_cycles["Test2"] == 5 assert dataflow_process.get_progress() == 0.725 dataflow_process.activate(processor2) env.run(30) finished = env.process(dataflow_process.workload()) env.run(finished) assert dataflow_process._remaining_compute_cycles is None assert dataflow_process._remaining_compute_cycles is None assert dataflow_process.get_progress() == 1.0
def _run( self, env, dataflow_process, processor, trace=None, channel=None, ): dataflow_process._trace = more_itertools.seekable(trace, maxlen=16) env.run() dataflow_process._channels["chan"] = ( weakref.ref(channel) if channel else None ) dataflow_process.start() env.run() dataflow_process.activate(processor) env.run() finished = env.process(dataflow_process.workload()) env.run(finished)
def loc_tz_getter() -> Iterator[DayWithZone]: # seekable makes it cache the emitted values return seekable(_iter_tzs())
from more_itertools import seekable data = "This is example sentence for seeking back and forth".split() it = seekable(data) for word in it: ... next(it) # StopIteration it.seek(3) next(it) # "sentence"
def __init__(self, iterable, filename=None): self._iterable = more_itertools.seekable(iterable) self._filename = filename self._ptr = 0 # pointer to the next item self._len = None