def setUpClass(cls): super(Test1ToNTest, cls).setUpClass() print("Start of tests: preparing nifi objects") config.nifi_config.host = 'http://192.168.56.5:8080/nifi-api' flow_name = "Test1ToNTest" nav = CanvasNavigator() # Delete all leftovers from previous (failed?) tests pgs_to_be_deleted = nav.groups(flow_name) for pg in pgs_to_be_deleted: canvas.delete_process_group(pg, force=True) # Create new process group in root Test1ToNTest.pg_test = canvas.create_process_group(nav.current, flow_name, (0, 0)) # Create simple flow to test Test1ToNTest.proc_start = canvas.create_processor( Test1ToNTest.pg_test, canvas.get_processor_type("GenerateFlowFile"), CANVAS_CENTER, "Start") Test1ToNTest.proc_2 = canvas.create_processor( Test1ToNTest.pg_test, canvas.get_processor_type("DebugFlow"), CANVAS_CENTER, "Processor 2") Test1ToNTest.proc_3 = canvas.create_processor( Test1ToNTest.pg_test, canvas.get_processor_type("DebugFlow"), CANVAS_CENTER, "Processor 3") Test1ToNTest.proc_end_1 = canvas.create_processor( Test1ToNTest.pg_test, canvas.get_processor_type("DebugFlow"), CANVAS_CENTER, "End 1") Test1ToNTest.proc_end_2 = canvas.create_processor( Test1ToNTest.pg_test, canvas.get_processor_type("DebugFlow"), CANVAS_CENTER, "End 2") canvas.update_processor(Test1ToNTest.proc_end_1, nifi.ProcessorConfigDTO(auto_terminated_relationships=["success", "failure"])) canvas.update_processor(Test1ToNTest.proc_end_2, nifi.ProcessorConfigDTO(auto_terminated_relationships=["success", "failure"])) Test1ToNTest.conn_1 = canvas.create_connection(Test1ToNTest.proc_start, Test1ToNTest.proc_2, ["success"]) Test1ToNTest.conn_2 = canvas.create_connection(Test1ToNTest.proc_2, Test1ToNTest.proc_3, ["success", "failure"]) Test1ToNTest.conn_3 = canvas.create_connection(Test1ToNTest.proc_3, Test1ToNTest.proc_end_1, ["success", "failure"]) Test1ToNTest.conn_4 = canvas.create_connection(Test1ToNTest.proc_3, Test1ToNTest.proc_end_2, ["success", "failure"]) canvas.schedule_process_group(Test1ToNTest.pg_test.component.id, scheduled=True)
def test_update_processor(regress_nifi, fix_proc): # TODO: Add way more tests to this f_p1 = fix_proc.generate() update = nifi.ProcessorConfigDTO(scheduling_period='3s') r1 = canvas.update_processor(f_p1, update) with pytest.raises(ValueError, match='update param is not an instance'): _ = canvas.update_processor(f_p1, 'FakeNews')
def run(self, input_name, flowfile, output_attributes=None, timeout=5): """ Runs the actual test with the flowfile provided. Builds the test components on the nifi canvas Starts the base process group Post flowfile via http to initiate test Destroys all test components on the nifi canvas Returns output: flowfile in output.text, attributes in output.headers Args: input_name (str): The input to post the message to flowfile (FlowFile): The flowfile and attributes to post output_attributes (collections.Iterable of str): List of attributes to capture in the test output timeout (integer): Timeout in seconds. Will throw requests.exceptions.ReadTimeout when timeout expires Returns: (FlowFile) """ assert isinstance(flowfile, FlowFile) # Set up testing infrastructure. This will stop the base self.__build() # Adding requested attributes as header parameters if output_attributes is not None: for attr in output_attributes: canvas.update_processor( self.http_out, nifi.ProcessorConfigDTO( properties={attr: "${" + attr + "}"})) # Start complete process group self.__start_base() # Prepare request parsed_url = urlparse(config.nifi_config.host) url = parsed_url.scheme + '://' + parsed_url.hostname + ':' + str( self.port) + '/' + self.name headers = flowfile.attributes headers["test_input_name"] = input_name # Perform actual request response = requests.post(url, data=flowfile.content, headers=headers, timeout=timeout) # Clean up testing infrastructure self.__destroy() # Should always be 200 assert response.status_code == 200 response.headers.pop('Date') response.headers.pop('Transfer-Encoding') response.headers.pop('Server') return FlowFile(response.text, dict(response.headers))
def test_update_processor(fixture_processor, regress): test_proc = fixture_processor.generate() update = nifi.ProcessorConfigDTO( scheduling_period='3s' ) r1 = canvas.update_processor(test_proc, update) with pytest.raises(ValueError, match='update param is not an instance'): _ = canvas.update_processor(test_proc, 'FakeNews')
def create_output_replacetext(parent_pg, location, name): """ Creates a ReplaceText to write attributes into the flowfile. When we later merge the contents, we would otherwise lose the attributes Args: parent_pg (ProcessGroupEntity): Target process group to place processor location (Location): x,y coordinated to place the processor name (string): Name of the test input Returns: (ProcessorEntity) """ assert isinstance(parent_pg, nifi.ProcessGroupEntity) assert isinstance(location, Location) assert isinstance(name, str) return nifi.ProcessGroupsApi().create_processor( id=parent_pg.component.id, body=nifi.ProcessorEntity( revision=nifi.RevisionDTO(version=0), component=nifi.ProcessorDTO( type="org.apache.nifi.processors.standard.ReplaceText", name="Merge flowfile and attributes", position=nifi.PositionDTO( x=location.x, y=location.y ), config=nifi.ProcessorConfigDTO( properties={ # "Search Value": "^(.*)$", "Replacement Value": "{\"flowfile\":\"$1\",\"attributes\":{" + "\"test_start_time\":\"${test_start_time}\"," + "\"test_end_time\":\"${test_end_time}\"," + "\"test_duration\":\"${test_duration}\"," + "\"test_input_name\":\"${http.headers.test_input_name}\"," + "\"test_output_name\":\"${test_output_name}\"" + "}}" }, auto_terminated_relationships=["failure"] ) ) ) )
def create_response_handler(parent_pg, location, http_context): """ Creates a HandleHttpResponse and connects it to the Http controller service Args: parent_pg (ProcessGroupEntity): Target process group to place processor location (Location): x,y coordinated to place the processor http_context (ControllerServiceEntity): StandardHttpContextMap to connect the processor to Returns: (ProcessorEntity) """ assert isinstance(parent_pg, nifi.ProcessGroupEntity) assert isinstance(location, Location) assert isinstance(http_context, nifi.ControllerServiceEntity) return nifi.ProcessGroupsApi().create_processor( id=parent_pg.component.id, body=nifi.ProcessorEntity( revision=nifi.RevisionDTO(version=0), component=nifi.ProcessorDTO( type="org.apache.nifi.processors.standard.HandleHttpResponse", name="Return test result", position=nifi.PositionDTO( x=location.x, y=location.y ), config=nifi.ProcessorConfigDTO( properties={ "HTTP Context Map": http_context.component.id, "HTTP Status Code": "200", "test_input_name": "${http.headers.test_input_name}", "test_output_name": "${test_output_name}", "test_start_time": "${test_start_time}", "test_end_time": "${test_end_time}", "test_duration": "${test_duration}" }, auto_terminated_relationships=["failure", "success"] ) ) ) )
def __init__(self, process_group, processor_name, processor_location, processor_config): self.group = process_group self.name = processor_name self.location = processor_location self.config = nifi.ProcessorConfigDTO( penalty_duration=processor_config["penalty_duration"], yield_duration=processor_config["yield_duration"], bulletin_level=processor_config["bulletin_level"], auto_terminated_relationships=processor_config[ "auto_terminated_relationships"], scheduling_strategy=processor_config["scheduling_strategy"], concurrently_schedulable_task_count=processor_config[ "concurrently_schedulable_task_count"], scheduling_period=processor_config["scheduling_period"], execution_node=processor_config["execution_node"], run_duration_millis=processor_config["run_duration_millis"], properties=None)
def create_input_router(parent_pg, location, test_inputs): """ Creates a RouteOnAttribute, routing each different input Only flow files with a matching http.headers.test_input_name attribute will be routed to an (corresponding) input Args: parent_pg (ProcessGroupEntity): Target process group to place processor location (Location): x,y coordinated to place the processor test_inputs (array of ProcessorEntity, InputPortEntity and OutputPortEntity): array of outputs for the router, routed to the different inputs Returns: (ProcessorEntity) """ assert isinstance(parent_pg, nifi.ProcessGroupEntity) assert isinstance(location, Location) # assert isinstance(test_inputs, array) route_properties = {} for test_input in test_inputs: route_properties[test_input.component.name] = "${http.headers.test_input_name:equals('" \ + test_input.component.name + "')}" return nifi.ProcessGroupsApi().create_processor( id=parent_pg.component.id, body=nifi.ProcessorEntity( revision=nifi.RevisionDTO(version=0), component=nifi.ProcessorDTO( type="org.apache.nifi.processors.standard.RouteOnAttribute", name="Route to correct input", position=nifi.PositionDTO( x=location.x, y=location.y ), config=nifi.ProcessorConfigDTO( properties=route_properties, auto_terminated_relationships=["unmatched"] ) ) ) )
def create_request_handler(parent_pg, location, http_context, port): """ Creates a HandleHttpRequest and connects it to the Http controller service Args: parent_pg (ProcessGroupEntity): Target process group to place processor location (Location): x,y coordinated to place the processor http_context (ControllerServiceEntity): StandardHttpContextMap to connect the processor to port (int): port number to use Returns: (ProcessorEntity) """ assert isinstance(parent_pg, nifi.ProcessGroupEntity) assert isinstance(location, Location) assert isinstance(http_context, nifi.ControllerServiceEntity) assert isinstance(port, int) return nifi.ProcessGroupsApi().create_processor( id=parent_pg.component.id, body=nifi.ProcessorEntity( revision=nifi.RevisionDTO(version=0), component=nifi.ProcessorDTO( type="org.apache.nifi.processors.standard.HandleHttpRequest", name="Receive test message", position=nifi.PositionDTO( x=location.x, y=location.y ), config=nifi.ProcessorConfigDTO( properties={ "HTTP Context Map": http_context.component.id, "Listening Port": port } ) ) ) )
def create_output_attribute(parent_pg, location, name): """ Creates a UpdateAttribute to register the test_output_name to the response Args: parent_pg (ProcessGroupEntity): Target process group to place processor location (Location): x,y coordinated to place the processor name (string): Name of the test output Returns: (ProcessorEntity) """ assert isinstance(parent_pg, nifi.ProcessGroupEntity) assert isinstance(location, Location) assert isinstance(name, str) return nifi.ProcessGroupsApi().create_processor( id=parent_pg.component.id, body=nifi.ProcessorEntity( revision=nifi.RevisionDTO(version=0), component=nifi.ProcessorDTO( type="org.apache.nifi.processors.attributes.UpdateAttribute", name="Set output name '" + name + "'", position=nifi.PositionDTO( x=location.x, y=location.y ), config=nifi.ProcessorConfigDTO( properties={ "test_output_name": name, "test_end_time": "${now():toNumber()}", "test_duration": "${now():toNumber():minus(${test_start_time})}" } ) ) ) )
def create_output_mergecontent(parent_pg, location, name): """ Creates a MergeContent to combine flowfiles from different outputs Args: parent_pg (ProcessGroupEntity): Target process group to place processor location (Location): x,y coordinated to place the processor name (string): Name of the test input Returns: (ProcessorEntity) """ assert isinstance(parent_pg, nifi.ProcessGroupEntity) assert isinstance(location, Location) assert isinstance(name, str) return nifi.ProcessGroupsApi().create_processor( id=parent_pg.component.id, body=nifi.ProcessorEntity( revision=nifi.RevisionDTO(version=0), component=nifi.ProcessorDTO( type="org.apache.nifi.processors.standard.MergeContent", name="Merge outputs", position=nifi.PositionDTO( x=location.x, y=location.y ), config=nifi.ProcessorConfigDTO( properties={ "Header": "[", "Footer": "]", "Demarcator": "," }, auto_terminated_relationships=["failure", "original"] ) ) ) )
def create_output_router(parent_pg, location, name): """ Creates a RouteOnAttribute to filter all incoming messages from the actual test results Args: parent_pg (ProcessGroupEntity): Target process group to place processor location (Location): x,y coordinated to place the processor name (string): Name of the test input Returns: (ProcessorEntity) """ assert isinstance(parent_pg, nifi.ProcessGroupEntity) assert isinstance(location, Location) assert isinstance(name, str) return nifi.ProcessGroupsApi().create_processor( id=parent_pg.component.id, body=nifi.ProcessorEntity( revision=nifi.RevisionDTO(version=0), component=nifi.ProcessorDTO( type="org.apache.nifi.processors.standard.RouteOnAttribute", name="Filter only test messages", position=nifi.PositionDTO( x=location.x, y=location.y ), config=nifi.ProcessorConfigDTO( properties={"test": "${http.request.uri:equals('/" + name + "')}"}, auto_terminated_relationships=["unmatched"] ) ) ) )