def on_incoming_records(self, connection: ConnectionInterface) -> None:
"""Process records in batches."""
input_df = connection.record_containers[0].build_dataframe()
self.output_anchor.push_records(
generate_records_from_df(input_df, self.output_anchor.record_info))
connection.clear_records()
def test_connection_interface_ii_close(
connection_interface: ConnectionInterface):
events = []
connection_interface.subscribe(ConnectionEvents.CONNECTION_CLOSED,
lambda **kwargs: events.append(kwargs))
assert connection_interface.status == ConnectionStatus.CREATED
connection_interface.ii_close()
assert connection_interface.status == ConnectionStatus.CLOSED
assert len(events) == 1
def on_incoming_records(self, connection: ConnectionInterface) -> None:
"""Process records in batches."""
self.generate_metadata()
df = connection.record_containers[0].build_dataframe()
df["optional_value"] = self.workflow_config["Value"]
self.output_anchor.push_records(
generate_records_from_df(df, self.output_anchor.record_info))
connection.clear_records()
def test_connection_interface_ii_init(
connection_interface: ConnectionInterface, record_collection):
record_info = record_collection[0]
assert connection_interface.record_info is None
events = []
connection_interface.subscribe(ConnectionEvents.CONNECTION_INITIALIZED,
lambda **kwargs: events.append(kwargs))
connection_interface.ii_init(record_info=record_info)
assert connection_interface.record_info is record_info
assert len(events) == 1
def connection_interface() -> ConnectionInterface:
connection_name = "#1"
anchor = InputAnchor("Input", False)
return ConnectionInterface(plugin=MockPlugin(),
connection_name=connection_name,
anchor=anchor)
def test_plugin_initialization_callback():
plugin = MockPlugin()
connection_interface = ConnectionInterface(
plugin=plugin,
connection_name="Connection",
anchor=InputAnchor(name="Input", optional=False),
)
assert not connection_interface.plugin_failed
plugin.notify_topic(PluginEvents.PLUGIN_FAILURE)
assert connection_interface.plugin_failed
def test_connection_interface_record_handling(
connection_interface: ConnectionInterface, record_collection):
record_info, records = record_collection
container = MockRecordContainer()
connection_interface.add_record_container(container)
events = []
connection_interface.subscribe(ConnectionEvents.RECORD_RECEIVED,
lambda **kwargs: events.append(kwargs))
for record in records:
connection_interface.ii_push_record(record)
assert len(events) == len(records)
assert len(container.records) == len(records)
connection_interface.clear_records()
assert len(container.records) == 0
def test_connection_interface_ii_update_progress(
connection_interface: ConnectionInterface):
events = []
connection_interface.subscribe(ConnectionEvents.PROGRESS_UPDATE,
lambda **kwargs: events.append(kwargs))
assert connection_interface.progress_percentage == 0
connection_interface.ii_update_progress(60)
assert len(events) == 1
assert connection_interface.progress_percentage == 60
connection_interface.ii_update_progress(70)
assert len(events) == 2
assert connection_interface.progress_percentage == 70
connection_interface.ii_update_progress(-20)
assert len(events) == 3
assert connection_interface.progress_percentage == 0
def on_incoming_records(self, connection: ConnectionInterface) -> None:
"""Process records in batches."""
# Do nothing with records for now, this is an output tool
connection.clear_records()