def process_bundle(self, request, instruction_id):
bundle_processor.BundleProcessor(
self.fns[request.process_bundle_descriptor_reference],
self.state_handler,
self.data_channel_factory).process_bundle(instruction_id)
return beam_fn_api_pb2.ProcessBundleResponse()
def get(self, instruction_id, bundle_descriptor_id):
try:
# pop() is threadsafe
processor = self.cached_bundle_processors[bundle_descriptor_id].pop()
except IndexError:
processor = bundle_processor.BundleProcessor(
self.fns[bundle_descriptor_id],
self.state_handler_factory.create_state_handler(
self.fns[bundle_descriptor_id].state_api_service_descriptor),
self.data_channel_factory)
self.active_bundle_processors[
instruction_id] = bundle_descriptor_id, processor
return processor
def process_bundle(self, request, instruction_id):
self.bundle_processors[
instruction_id] = processor = bundle_processor.BundleProcessor(
self.fns[request.process_bundle_descriptor_reference],
self.state_handler, self.data_channel_factory)
try:
processor.process_bundle(instruction_id)
finally:
del self.bundle_processors[instruction_id]
return beam_fn_api_pb2.InstructionResponse(
instruction_id=instruction_id,
process_bundle=beam_fn_api_pb2.ProcessBundleResponse(
metrics=processor.metrics()))
def process_bundle(self, request, instruction_id):
process_bundle_desc = self.fns[
request.process_bundle_descriptor_reference]
state_handler = self.state_handler_factory.create_state_handler(
process_bundle_desc.state_api_service_descriptor)
self.bundle_processors[
instruction_id] = processor = bundle_processor.BundleProcessor(
process_bundle_desc, state_handler, self.data_channel_factory)
try:
with state_handler.process_instruction_id(instruction_id):
processor.process_bundle(instruction_id)
finally:
del self.bundle_processors[instruction_id]
return beam_fn_api_pb2.InstructionResponse(
instruction_id=instruction_id,
process_bundle=beam_fn_api_pb2.ProcessBundleResponse(
metrics=processor.metrics(),
monitoring_infos=processor.monitoring_infos()))
def get(self, instruction_id, bundle_descriptor_id):
# type: (str, str) -> bundle_processor.BundleProcessor
"""
Return the requested ``BundleProcessor``, creating it if necessary.
Moves the ``BundleProcessor`` from the inactive to the active cache.
"""
try:
# pop() is threadsafe
processor = self.cached_bundle_processors[bundle_descriptor_id].pop()
except IndexError:
processor = bundle_processor.BundleProcessor(
self.fns[bundle_descriptor_id],
self.state_handler_factory.create_state_handler(
self.fns[bundle_descriptor_id].state_api_service_descriptor),
self.data_channel_factory)
self.active_bundle_processors[
instruction_id] = bundle_descriptor_id, processor
return processor
def get_bundle_processor(self, instruction_id, bundle_descriptor_id):
try:
# pop() is threadsafe
processor = self.cached_bundle_processors[
bundle_descriptor_id].pop()
state_handler = processor.state_handler
except IndexError:
process_bundle_desc = self.fns[bundle_descriptor_id]
state_handler = self.state_handler_factory.create_state_handler(
process_bundle_desc.state_api_service_descriptor)
processor = bundle_processor.BundleProcessor(
process_bundle_desc, state_handler, self.data_channel_factory)
try:
self.active_bundle_processors[instruction_id] = processor
with state_handler.process_instruction_id(instruction_id):
yield processor
finally:
del self.active_bundle_processors[instruction_id]
# Outside the finally block as we only want to re-use on success.
processor.reset()
self.cached_bundle_processors[bundle_descriptor_id].append(processor)
def get(self, instruction_id, bundle_descriptor_id):
# type: (str, str) -> bundle_processor.BundleProcessor
"""
Return the requested ``BundleProcessor``, creating it if necessary.
Moves the ``BundleProcessor`` from the inactive to the active cache.
"""
with self._lock:
try:
# pop() is threadsafe
processor = self.cached_bundle_processors[bundle_descriptor_id].pop()
self.active_bundle_processors[
instruction_id] = bundle_descriptor_id, processor
try:
del self.known_not_running_instruction_ids[instruction_id]
except KeyError:
# The instruction may have not been pre-registered before execution
# since activate() may have never been invoked
pass
return processor
except IndexError:
pass
# Make sure we instantiate the processor while not holding the lock.
processor = bundle_processor.BundleProcessor(
self.fns[bundle_descriptor_id],
self.state_handler_factory.create_state_handler(
self.fns[bundle_descriptor_id].state_api_service_descriptor),
self.data_channel_factory)
with self._lock:
self.active_bundle_processors[
instruction_id] = bundle_descriptor_id, processor
try:
del self.known_not_running_instruction_ids[instruction_id]
except KeyError:
# The instruction may have not been pre-registered before execution
# since activate() may have never been invoked
pass
return processor