def process_bundle(self, request, instruction_id): bundle_processor.BundleProcessor( self.fns[request.process_bundle_descriptor_reference], self.state_handler, self.data_channel_factory).process_bundle(instruction_id) return beam_fn_api_pb2.ProcessBundleResponse()
def get(self, instruction_id, bundle_descriptor_id): try: # pop() is threadsafe processor = self.cached_bundle_processors[bundle_descriptor_id].pop() except IndexError: processor = bundle_processor.BundleProcessor( self.fns[bundle_descriptor_id], self.state_handler_factory.create_state_handler( self.fns[bundle_descriptor_id].state_api_service_descriptor), self.data_channel_factory) self.active_bundle_processors[ instruction_id] = bundle_descriptor_id, processor return processor
def process_bundle(self, request, instruction_id): self.bundle_processors[ instruction_id] = processor = bundle_processor.BundleProcessor( self.fns[request.process_bundle_descriptor_reference], self.state_handler, self.data_channel_factory) try: processor.process_bundle(instruction_id) finally: del self.bundle_processors[instruction_id] return beam_fn_api_pb2.InstructionResponse( instruction_id=instruction_id, process_bundle=beam_fn_api_pb2.ProcessBundleResponse( metrics=processor.metrics()))
def process_bundle(self, request, instruction_id): process_bundle_desc = self.fns[ request.process_bundle_descriptor_reference] state_handler = self.state_handler_factory.create_state_handler( process_bundle_desc.state_api_service_descriptor) self.bundle_processors[ instruction_id] = processor = bundle_processor.BundleProcessor( process_bundle_desc, state_handler, self.data_channel_factory) try: with state_handler.process_instruction_id(instruction_id): processor.process_bundle(instruction_id) finally: del self.bundle_processors[instruction_id] return beam_fn_api_pb2.InstructionResponse( instruction_id=instruction_id, process_bundle=beam_fn_api_pb2.ProcessBundleResponse( metrics=processor.metrics(), monitoring_infos=processor.monitoring_infos()))
def get(self, instruction_id, bundle_descriptor_id): # type: (str, str) -> bundle_processor.BundleProcessor """ Return the requested ``BundleProcessor``, creating it if necessary. Moves the ``BundleProcessor`` from the inactive to the active cache. """ try: # pop() is threadsafe processor = self.cached_bundle_processors[bundle_descriptor_id].pop() except IndexError: processor = bundle_processor.BundleProcessor( self.fns[bundle_descriptor_id], self.state_handler_factory.create_state_handler( self.fns[bundle_descriptor_id].state_api_service_descriptor), self.data_channel_factory) self.active_bundle_processors[ instruction_id] = bundle_descriptor_id, processor return processor
def get_bundle_processor(self, instruction_id, bundle_descriptor_id): try: # pop() is threadsafe processor = self.cached_bundle_processors[ bundle_descriptor_id].pop() state_handler = processor.state_handler except IndexError: process_bundle_desc = self.fns[bundle_descriptor_id] state_handler = self.state_handler_factory.create_state_handler( process_bundle_desc.state_api_service_descriptor) processor = bundle_processor.BundleProcessor( process_bundle_desc, state_handler, self.data_channel_factory) try: self.active_bundle_processors[instruction_id] = processor with state_handler.process_instruction_id(instruction_id): yield processor finally: del self.active_bundle_processors[instruction_id] # Outside the finally block as we only want to re-use on success. processor.reset() self.cached_bundle_processors[bundle_descriptor_id].append(processor)
def get(self, instruction_id, bundle_descriptor_id): # type: (str, str) -> bundle_processor.BundleProcessor """ Return the requested ``BundleProcessor``, creating it if necessary. Moves the ``BundleProcessor`` from the inactive to the active cache. """ with self._lock: try: # pop() is threadsafe processor = self.cached_bundle_processors[bundle_descriptor_id].pop() self.active_bundle_processors[ instruction_id] = bundle_descriptor_id, processor try: del self.known_not_running_instruction_ids[instruction_id] except KeyError: # The instruction may have not been pre-registered before execution # since activate() may have never been invoked pass return processor except IndexError: pass # Make sure we instantiate the processor while not holding the lock. processor = bundle_processor.BundleProcessor( self.fns[bundle_descriptor_id], self.state_handler_factory.create_state_handler( self.fns[bundle_descriptor_id].state_api_service_descriptor), self.data_channel_factory) with self._lock: self.active_bundle_processors[ instruction_id] = bundle_descriptor_id, processor try: del self.known_not_running_instruction_ids[instruction_id] except KeyError: # The instruction may have not been pre-registered before execution # since activate() may have never been invoked pass return processor