def setUpClass(cls):
super(Test1ToNTest, cls).setUpClass()
print("Start of tests: preparing nifi objects")
config.nifi_config.host = 'http://192.168.56.5:8080/nifi-api'
flow_name = "Test1ToNTest"
nav = CanvasNavigator()
# Delete all leftovers from previous (failed?) tests
pgs_to_be_deleted = nav.groups(flow_name)
for pg in pgs_to_be_deleted:
canvas.delete_process_group(pg, force=True)
# Create new process group in root
Test1ToNTest.pg_test = canvas.create_process_group(nav.current, flow_name, (0, 0))
# Create simple flow to test
Test1ToNTest.proc_start = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("GenerateFlowFile"),
CANVAS_CENTER,
"Start")
Test1ToNTest.proc_2 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"Processor 2")
Test1ToNTest.proc_3 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"Processor 3")
Test1ToNTest.proc_end_1 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"End 1")
Test1ToNTest.proc_end_2 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"End 2")
canvas.update_processor(Test1ToNTest.proc_end_1,
nifi.ProcessorConfigDTO(auto_terminated_relationships=["success", "failure"]))
canvas.update_processor(Test1ToNTest.proc_end_2,
nifi.ProcessorConfigDTO(auto_terminated_relationships=["success", "failure"]))
Test1ToNTest.conn_1 = canvas.create_connection(Test1ToNTest.proc_start, Test1ToNTest.proc_2, ["success"])
Test1ToNTest.conn_2 = canvas.create_connection(Test1ToNTest.proc_2, Test1ToNTest.proc_3, ["success", "failure"])
Test1ToNTest.conn_3 = canvas.create_connection(Test1ToNTest.proc_3, Test1ToNTest.proc_end_1, ["success", "failure"])
Test1ToNTest.conn_4 = canvas.create_connection(Test1ToNTest.proc_3, Test1ToNTest.proc_end_2, ["success", "failure"])
canvas.schedule_process_group(Test1ToNTest.pg_test.component.id, scheduled=True)
def test_update_processor(regress_nifi, fix_proc):
# TODO: Add way more tests to this
f_p1 = fix_proc.generate()
update = nifi.ProcessorConfigDTO(scheduling_period='3s')
r1 = canvas.update_processor(f_p1, update)
with pytest.raises(ValueError, match='update param is not an instance'):
_ = canvas.update_processor(f_p1, 'FakeNews')
def run(self, input_name, flowfile, output_attributes=None, timeout=5):
"""
Runs the actual test with the flowfile provided.
Builds the test components on the nifi canvas
Starts the base process group
Post flowfile via http to initiate test
Destroys all test components on the nifi canvas
Returns output: flowfile in output.text, attributes in output.headers
Args:
input_name (str): The input to post the message to
flowfile (FlowFile): The flowfile and attributes to post
output_attributes (collections.Iterable of str): List of attributes to capture in the
test output
timeout (integer): Timeout in seconds. Will throw requests.exceptions.ReadTimeout
when timeout expires
Returns:
(FlowFile)
"""
assert isinstance(flowfile, FlowFile)
# Set up testing infrastructure. This will stop the base
self.__build()
# Adding requested attributes as header parameters
if output_attributes is not None:
for attr in output_attributes:
canvas.update_processor(
self.http_out,
nifi.ProcessorConfigDTO(
properties={attr: "${" + attr + "}"}))
# Start complete process group
self.__start_base()
# Prepare request
parsed_url = urlparse(config.nifi_config.host)
url = parsed_url.scheme + '://' + parsed_url.hostname + ':' + str(
self.port) + '/' + self.name
headers = flowfile.attributes
headers["test_input_name"] = input_name
# Perform actual request
response = requests.post(url,
data=flowfile.content,
headers=headers,
timeout=timeout)
# Clean up testing infrastructure
self.__destroy()
# Should always be 200
assert response.status_code == 200
response.headers.pop('Date')
response.headers.pop('Transfer-Encoding')
response.headers.pop('Server')
return FlowFile(response.text, dict(response.headers))
def test_update_processor(fixture_processor, regress):
test_proc = fixture_processor.generate()
update = nifi.ProcessorConfigDTO(
scheduling_period='3s'
)
r1 = canvas.update_processor(test_proc, update)
with pytest.raises(ValueError, match='update param is not an instance'):
_ = canvas.update_processor(test_proc, 'FakeNews')
def create_output_replacetext(parent_pg, location, name):
"""
Creates a ReplaceText to write attributes into the flowfile. When
we later merge the contents, we would otherwise lose the
attributes
Args:
parent_pg (ProcessGroupEntity): Target process group to place
processor
location (Location): x,y coordinated to place the processor
name (string): Name of the test input
Returns:
(ProcessorEntity)
"""
assert isinstance(parent_pg, nifi.ProcessGroupEntity)
assert isinstance(location, Location)
assert isinstance(name, str)
return nifi.ProcessGroupsApi().create_processor(
id=parent_pg.component.id,
body=nifi.ProcessorEntity(
revision=nifi.RevisionDTO(version=0),
component=nifi.ProcessorDTO(
type="org.apache.nifi.processors.standard.ReplaceText",
name="Merge flowfile and attributes",
position=nifi.PositionDTO(
x=location.x,
y=location.y
),
config=nifi.ProcessorConfigDTO(
properties={
# "Search Value": "^(.*)$",
"Replacement Value": "{\"flowfile\":\"$1\",\"attributes\":{" +
"\"test_start_time\":\"${test_start_time}\"," +
"\"test_end_time\":\"${test_end_time}\"," +
"\"test_duration\":\"${test_duration}\"," +
"\"test_input_name\":\"${http.headers.test_input_name}\"," +
"\"test_output_name\":\"${test_output_name}\"" +
"}}"
},
auto_terminated_relationships=["failure"]
)
)
)
)
def create_response_handler(parent_pg, location, http_context):
"""
Creates a HandleHttpResponse and connects it to the Http controller service
Args:
parent_pg (ProcessGroupEntity): Target process group to place
processor
location (Location): x,y coordinated to place the processor
http_context (ControllerServiceEntity): StandardHttpContextMap to
connect the processor to
Returns:
(ProcessorEntity)
"""
assert isinstance(parent_pg, nifi.ProcessGroupEntity)
assert isinstance(location, Location)
assert isinstance(http_context, nifi.ControllerServiceEntity)
return nifi.ProcessGroupsApi().create_processor(
id=parent_pg.component.id,
body=nifi.ProcessorEntity(
revision=nifi.RevisionDTO(version=0),
component=nifi.ProcessorDTO(
type="org.apache.nifi.processors.standard.HandleHttpResponse",
name="Return test result",
position=nifi.PositionDTO(
x=location.x,
y=location.y
),
config=nifi.ProcessorConfigDTO(
properties={
"HTTP Context Map": http_context.component.id,
"HTTP Status Code": "200",
"test_input_name": "${http.headers.test_input_name}",
"test_output_name": "${test_output_name}",
"test_start_time": "${test_start_time}",
"test_end_time": "${test_end_time}",
"test_duration": "${test_duration}"
},
auto_terminated_relationships=["failure", "success"]
)
)
)
)
def __init__(self, process_group, processor_name, processor_location,
processor_config):
self.group = process_group
self.name = processor_name
self.location = processor_location
self.config = nifi.ProcessorConfigDTO(
penalty_duration=processor_config["penalty_duration"],
yield_duration=processor_config["yield_duration"],
bulletin_level=processor_config["bulletin_level"],
auto_terminated_relationships=processor_config[
"auto_terminated_relationships"],
scheduling_strategy=processor_config["scheduling_strategy"],
concurrently_schedulable_task_count=processor_config[
"concurrently_schedulable_task_count"],
scheduling_period=processor_config["scheduling_period"],
execution_node=processor_config["execution_node"],
run_duration_millis=processor_config["run_duration_millis"],
properties=None)
def create_input_router(parent_pg, location, test_inputs):
"""
Creates a RouteOnAttribute, routing each different input
Only flow files with a matching http.headers.test_input_name attribute
will be routed to an (corresponding) input
Args:
parent_pg (ProcessGroupEntity): Target process group to place
processor
location (Location): x,y coordinated to place the processor
test_inputs (array of ProcessorEntity, InputPortEntity and OutputPortEntity):
array of outputs for the router, routed to the different inputs
Returns:
(ProcessorEntity)
"""
assert isinstance(parent_pg, nifi.ProcessGroupEntity)
assert isinstance(location, Location)
# assert isinstance(test_inputs, array)
route_properties = {}
for test_input in test_inputs:
route_properties[test_input.component.name] = "${http.headers.test_input_name:equals('" \
+ test_input.component.name + "')}"
return nifi.ProcessGroupsApi().create_processor(
id=parent_pg.component.id,
body=nifi.ProcessorEntity(
revision=nifi.RevisionDTO(version=0),
component=nifi.ProcessorDTO(
type="org.apache.nifi.processors.standard.RouteOnAttribute",
name="Route to correct input",
position=nifi.PositionDTO(
x=location.x,
y=location.y
),
config=nifi.ProcessorConfigDTO(
properties=route_properties,
auto_terminated_relationships=["unmatched"]
)
)
)
)
def create_request_handler(parent_pg, location, http_context, port):
"""
Creates a HandleHttpRequest and connects it to the Http controller service
Args:
parent_pg (ProcessGroupEntity): Target process group to place
processor
location (Location): x,y coordinated to place the processor
http_context (ControllerServiceEntity): StandardHttpContextMap to
connect the processor to
port (int): port number to use
Returns:
(ProcessorEntity)
"""
assert isinstance(parent_pg, nifi.ProcessGroupEntity)
assert isinstance(location, Location)
assert isinstance(http_context, nifi.ControllerServiceEntity)
assert isinstance(port, int)
return nifi.ProcessGroupsApi().create_processor(
id=parent_pg.component.id,
body=nifi.ProcessorEntity(
revision=nifi.RevisionDTO(version=0),
component=nifi.ProcessorDTO(
type="org.apache.nifi.processors.standard.HandleHttpRequest",
name="Receive test message",
position=nifi.PositionDTO(
x=location.x,
y=location.y
),
config=nifi.ProcessorConfigDTO(
properties={
"HTTP Context Map": http_context.component.id,
"Listening Port": port
}
)
)
)
)
def create_output_attribute(parent_pg, location, name):
"""
Creates a UpdateAttribute to register the test_output_name to the
response
Args:
parent_pg (ProcessGroupEntity): Target process group to place
processor
location (Location): x,y coordinated to place the processor
name (string): Name of the test output
Returns:
(ProcessorEntity)
"""
assert isinstance(parent_pg, nifi.ProcessGroupEntity)
assert isinstance(location, Location)
assert isinstance(name, str)
return nifi.ProcessGroupsApi().create_processor(
id=parent_pg.component.id,
body=nifi.ProcessorEntity(
revision=nifi.RevisionDTO(version=0),
component=nifi.ProcessorDTO(
type="org.apache.nifi.processors.attributes.UpdateAttribute",
name="Set output name '" + name + "'",
position=nifi.PositionDTO(
x=location.x,
y=location.y
),
config=nifi.ProcessorConfigDTO(
properties={
"test_output_name": name,
"test_end_time": "${now():toNumber()}",
"test_duration": "${now():toNumber():minus(${test_start_time})}"
}
)
)
)
)
def create_output_mergecontent(parent_pg, location, name):
"""
Creates a MergeContent to combine flowfiles from different outputs
Args:
parent_pg (ProcessGroupEntity): Target process group to place
processor
location (Location): x,y coordinated to place the processor
name (string): Name of the test input
Returns:
(ProcessorEntity)
"""
assert isinstance(parent_pg, nifi.ProcessGroupEntity)
assert isinstance(location, Location)
assert isinstance(name, str)
return nifi.ProcessGroupsApi().create_processor(
id=parent_pg.component.id,
body=nifi.ProcessorEntity(
revision=nifi.RevisionDTO(version=0),
component=nifi.ProcessorDTO(
type="org.apache.nifi.processors.standard.MergeContent",
name="Merge outputs",
position=nifi.PositionDTO(
x=location.x,
y=location.y
),
config=nifi.ProcessorConfigDTO(
properties={
"Header": "[",
"Footer": "]",
"Demarcator": ","
},
auto_terminated_relationships=["failure", "original"]
)
)
)
)
def create_output_router(parent_pg, location, name):
"""
Creates a RouteOnAttribute to filter all incoming messages from
the actual test results
Args:
parent_pg (ProcessGroupEntity): Target process group to place
processor
location (Location): x,y coordinated to place the processor
name (string): Name of the test input
Returns:
(ProcessorEntity)
"""
assert isinstance(parent_pg, nifi.ProcessGroupEntity)
assert isinstance(location, Location)
assert isinstance(name, str)
return nifi.ProcessGroupsApi().create_processor(
id=parent_pg.component.id,
body=nifi.ProcessorEntity(
revision=nifi.RevisionDTO(version=0),
component=nifi.ProcessorDTO(
type="org.apache.nifi.processors.standard.RouteOnAttribute",
name="Filter only test messages",
position=nifi.PositionDTO(
x=location.x,
y=location.y
),
config=nifi.ProcessorConfigDTO(
properties={"test": "${http.request.uri:equals('/" + name + "')}"},
auto_terminated_relationships=["unmatched"]
)
)
)
)
