def test_create_process_group(regress_nifi): r = canvas.create_process_group(parent_pg=canvas.get_process_group( canvas.get_root_pg_id(), 'id'), new_pg_name=conftest.test_pg_name, location=(400.0, 400.0), comment='some comment') assert r.component.name == conftest.test_pg_name assert r.position.x == r.position.y == 400 assert r.component.parent_group_id == canvas.get_root_pg_id() assert isinstance(r, nifi.ProcessGroupEntity) # Test process group creation on other than root process group. s = canvas.create_process_group(parent_pg=canvas.get_process_group( conftest.test_pg_name), location=(200.0, 200.0), new_pg_name=conftest.test_another_pg_name) assert s.component.name == conftest.test_another_pg_name assert s.position.x == s.position.y == 200 assert s.component.parent_group_id == canvas.get_process_group( conftest.test_pg_name, "name").id assert isinstance(s, nifi.ProcessGroupEntity) with pytest.raises(ValueError): parent_pg = canvas.get_process_group('NiFi Flow') parent_pg.id = 'invalid' _ = canvas.create_process_group(parent_pg, 'irrelevant', (0, 0))
def test_create_process_group(regress): r = canvas.create_process_group(canvas.get_process_group( canvas.get_root_pg_id(), 'id'), config.test_pg_name, location=(400.0, 400.0)) assert r.component.name == config.test_pg_name assert r.position.x == r.position.y == 400 assert r.component.parent_group_id == canvas.get_root_pg_id() with pytest.raises(ApiException): parent_pg = canvas.get_process_group('NiFi Flow') parent_pg.id = 'invalid' _ = canvas.create_process_group(parent_pg, 'irrelevant', (0, 0))
def test_get_process_group(regress_nifi, fix_pg): with pytest.raises(AssertionError): _ = canvas.get_process_group('nipyapi_test', 'invalid') f_pg = fix_pg.generate() pg1 = canvas.get_process_group(f_pg.id, 'id') assert isinstance(pg1, ProcessGroupEntity) duplicate_pg = fix_pg.generate() pg2 = canvas.get_process_group(duplicate_pg.id, 'id') assert pg2.id != pg1.id pg_list = canvas.get_process_group(f_pg.status.name) assert isinstance(pg_list, list) # the two duplicates, and root = 3 assert len(pg_list) == 3
def test_schedule_process_group(fix_proc, fix_pg): f_pg = fix_pg.generate() _ = fix_proc.generate(parent_pg=f_pg) r1 = canvas.schedule_process_group(f_pg.id, True) status = canvas.get_process_group(f_pg.id, 'id') assert r1 is True assert status.running_count == 1 r2 = canvas.schedule_process_group(f_pg.id, False) assert r2 is True status = canvas.get_process_group(f_pg.id, 'id') assert status.running_count == 0 assert status.stopped_count == 1 with pytest.raises(AssertionError): _ = canvas.schedule_process_group(f_pg.id, 'BANANA')
def setUpClass(cls): super(CanvasNavigatorTest, cls).setUpClass() print("Start of tests: preparing nifi objects") config.nifi_config.host = 'http://192.168.56.5:8080/nifi-api' root = canvas.get_process_group(canvas.get_root_pg_id(), 'id') # Create new process group in root CanvasNavigatorTest.pg_parent = canvas.create_process_group( root, "parent", CANVAS_CENTER) # Create new child process groups in unittest CanvasNavigatorTest.pg_child1 = canvas.create_process_group( CanvasNavigatorTest.pg_parent, "child1", CANVAS_CENTER) CanvasNavigatorTest.pg_child2 = canvas.create_process_group( CanvasNavigatorTest.pg_parent, "child2", CANVAS_CENTER) CanvasNavigatorTest.pg_child2_2 = canvas.create_process_group( CanvasNavigatorTest.pg_parent, "child2", CANVAS_CENTER) CanvasNavigatorTest.pg_grandchild1 = canvas.create_process_group( CanvasNavigatorTest.pg_child1, "grandchild1", CANVAS_CENTER) # Create other objects as well CanvasNavigatorTest.proc = canvas.create_processor( CanvasNavigatorTest.pg_parent, canvas.get_processor_type("GenerateFlowFile"), CANVAS_CENTER, "proc") CanvasNavigatorTest.input_port = canvas.create_port( CanvasNavigatorTest.pg_parent.component.id, "INPUT_PORT", "input_port", "STOPPED", CANVAS_CENTER) CanvasNavigatorTest.output_port = canvas.create_port( CanvasNavigatorTest.pg_parent.component.id, "OUTPUT_PORT", "output_port", "STOPPED", CANVAS_CENTER) CanvasNavigatorTest.controller = canvas.create_controller( CanvasNavigatorTest.pg_parent, canvas.list_all_controller_types()[0], "controller")
def test_complex_template_versioning(regress_flow_reg, fix_ctv): # There is a complex bug where a new flow version cannot be switched to # and generates a NiFi NPE if attempted when create_flow_version is used # BUG FIXED: issue with variable name found in Swagger definition # https://github.com/apache/nifi/pull/2479#issuecomment-366582829 # Create a new flow version vers = versioning.list_flow_versions(fix_ctv.bucket.identifier, fix_ctv.flow.identifier) assert vers[0].version == 2 # create_flow_version is the problem new_ss = versioning.create_flow_version( flow=fix_ctv.flow, flow_snapshot=fix_ctv.snapshot_w_template, refresh=True) assert isinstance(new_ss, registry.VersionedFlowSnapshot) vers = versioning.list_flow_versions(fix_ctv.bucket.identifier, fix_ctv.flow.identifier) assert vers[0].version == 3 new_ver_info = versioning.get_version_info(fix_ctv.pg) r1 = versioning.update_flow_ver(fix_ctv.pg) assert r1.request.complete is True assert r1.request.failure_reason is None r2 = canvas.schedule_process_group(fix_ctv.pg.id, True) status = canvas.get_process_group(fix_ctv.pg.id, 'id') assert status.running_count >= 5 with pytest.raises(ValueError): _ = versioning.update_flow_ver(fix_ctv.pg, 'bob') with pytest.raises(ValueError): _ = versioning.update_flow_ver(fix_ctv.pg, '9999999')
def test_save_flow_ver(fixture_registry_bucket, fixture_pg, fixture_processor): test_pg = fixture_pg.generate() test_b, test_rc = fixture_registry_bucket r1 = versioning.save_flow_ver(process_group=test_pg, registry_client=test_rc, bucket=test_b, flow_name=config.test_versioned_flow_name, comment='a test comment', desc='a test description') assert isinstance(r1, nifi.VersionControlInformationEntity) # Next we validate you can't duplicate a flow name in a bucket with pytest.raises(ValueError): _ = versioning.save_flow_ver(process_group=test_pg, registry_client=test_rc, bucket=test_b, flow_name=config.test_versioned_flow_name, comment='NiPyApi Test', desc='NiPyApi Test') # Add a processor, refresh status, and save a new version fixture_processor.generate(parent_pg=test_pg) test_pg = canvas.get_process_group(test_pg.id, 'id') r2 = versioning.save_flow_ver( process_group=test_pg, registry_client=test_rc, bucket=test_b, flow_id=r1.version_control_information.flow_id, comment='a test comment') assert isinstance(r2, nifi.VersionControlInformationEntity) assert r2.version_control_information.version > \ r1.version_control_information.version
def test_list_nested_processors(regress_nifi, fix_pg, fix_proc): pg_1 = fix_pg.generate( parent_pg=canvas.get_process_group(canvas.get_root_pg_id(), 'id')) pg_2 = fix_pg.generate(parent_pg=pg_1) root_proc_1 = fix_proc.generate() pg_1_proc_1 = fix_proc.generate(parent_pg=pg_1) pg_1_proc_2 = fix_proc.generate(parent_pg=pg_1) pg_2_proc_1 = fix_proc.generate(parent_pg=pg_2) pg_2_proc_2 = fix_proc.generate(parent_pg=pg_2) pg_2_proc_3 = fix_proc.generate(parent_pg=pg_2) pg_2_proc_4 = fix_proc.generate(parent_pg=pg_2) r1 = [ x for x in canvas.list_all_processors('root') if conftest.test_basename in x.status.name ] assert len(r1) == 7 r2 = [ x for x in canvas.list_all_processors(pg_2.id) if conftest.test_basename in x.status.name ] assert len(r2) == 4 r3 = [ x for x in canvas.list_all_processors(pg_1.id) if conftest.test_basename in x.status.name ] assert len(r3) == 6
def test_current_name(self): print("Testing current name") self.assertEqual( self.nav.current_name(), canvas.get_process_group(canvas.get_root_pg_id(), 'id').component.name, 'incorrect current name')
def set_environment(run_id): if not run_id: run_id = str(int(time.time())) # Initialize NiFi API config.nifi_config.host = _get_nifi_api_url() config.registry_config.host = _get_nifireg_api_url() if _IS_TLS_ENABLED: security.set_service_ssl_context(service='nifi', ca_file=_TRUSTSTORE) security.set_service_ssl_context(service='registry', ca_file=_TRUSTSTORE) security.service_login(service='nifi', username='******', password='******') security.service_login(service='registry', username='******', password='******') # Get NiFi root PG root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id') # Get EFM flow flow_id, efm_pg_id = efm_get_flow('iot-1') return (run_id, root_pg, efm_pg_id, flow_id)
def test_create_template(self): from nipyapi.nifi import TemplateEntity t_id = canvas.get_process_group('nipyapi_test_0').id r = templates.create_template(pg_id=t_id, name='nipyapi_testTemplate_01', desc='Nothing Here') assert isinstance(r, TemplateEntity)
def test_create_controller(regress_nifi, fix_cont): root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id') cont_type = canvas.list_all_controller_types()[0] r1 = canvas.create_controller(parent_pg=root_pg, controller=cont_type) assert isinstance(r1, nifi.ControllerServiceEntity) with pytest.raises(AssertionError): _ = canvas.create_controller('pie', cont_type) with pytest.raises(AssertionError): _ = canvas.create_controller(root_pg, 'pie')
def to_process_group(json_process_group): try: process_group = canvas.get_process_group( json_process_group["parent_group_id"], 'id') process_group.revision.version = 0 return process_group except Exception as e: return str(e)
def wait_for_data(timeout_secs=120): while timeout_secs: bytes_in = canvas.get_process_group( PG_NAME, 'name').status.aggregate_snapshot.bytes_in if bytes_in > 0: break timeout_secs -= 1 time.sleep(1) # wait a few more seconds just to let the pipes to be primed time.sleep(10)
def wait_for_data(timeout_secs=120): LOG.info("Setup complete, waiting for data to flow in NiFi") while timeout_secs: bytes_in = canvas.get_process_group(PG_NAME, 'name').status.aggregate_snapshot.bytes_in if bytes_in > 0: break timeout_secs -= 1 LOG.info("Data not Flowing yet, sleeping for 3") time.sleep(3) # wait a few more seconds just to let the pipes to be primed time.sleep(10)
def test_client_recursion_limit(fix_pg, fix_funnel, target=450): # https://github.com/Chaffelson/nipyapi/issues/147 parent_pg = canvas.get_process_group('root') for i in range(0, target): parent_pg = fix_pg.generate(parent_pg, str(i)) fix_funnel.generate(parent_pg) start = time.time() r1 = canvas.list_all_process_groups(canvas.get_root_pg_id()) end = time.time() assert len(r1) == target + 1 # +1 to allow for root PG print("Len {0} Set {1}".format(len(r1), len(set([x.id for x in r1])))) print("Elapsed r1: {0}".format((end - start)))
def test_save_flow_ver(regress_flow_reg, fix_bucket, fix_pg, fix_proc): f_reg_client = conftest.ensure_registry_client( config.registry_local_name ) f_bucket = fix_bucket() f_pg = fix_pg.generate() test_bucket = versioning.get_registry_bucket(f_bucket.identifier, 'id') assert test_bucket.name == conftest.test_bucket_name r1 = versioning.save_flow_ver( process_group=f_pg, registry_client=f_reg_client, bucket=test_bucket, flow_name=conftest.test_versioned_flow_name, comment='a test comment', desc='a test description' ) assert isinstance(r1, nifi.VersionControlInformationEntity) # Next we validate you can't duplicate a flow name in a bucket with pytest.raises(ValueError): _ = versioning.save_flow_ver( process_group=f_pg, registry_client=f_reg_client, bucket=f_bucket, flow_name=conftest.test_versioned_flow_name, comment='NiPyApi Test', desc='NiPyApi Test' ) # Add a processor, refresh status, and save a new version fix_proc.generate(parent_pg=f_pg) f_pg = canvas.get_process_group(f_pg.id, 'id') r2 = versioning.save_flow_ver( process_group=f_pg, registry_client=f_reg_client, bucket=f_bucket, flow_id=r1.version_control_information.flow_id, comment='a test comment' ) assert isinstance(r2, nifi.VersionControlInformationEntity) assert r2.version_control_information.version > \ r1.version_control_information.version with pytest.raises(ValueError): _ = versioning.save_flow_ver( process_group=f_pg, registry_client=f_reg_client, bucket=f_bucket, flow_name=conftest.test_versioned_flow_name, comment='a test comment', desc='a test description', refresh=False ) # shortcut to clean up the test objects when not using the fixture conftest.cleanup_reg()
def wait_for_data(pg_name, timeout_secs=120): while timeout_secs: pg = canvas.get_process_group(pg_name, 'name') if pg is None: break bytes_in = pg.status.aggregate_snapshot.bytes_in if bytes_in > 0: break timeout_secs -= 1 LOG.info("Data not Flowing yet, sleeping for 3") time.sleep(3) # wait a few more seconds just to let the pipes to be primed time.sleep(10)
def set_environment(run_id): if not run_id: run_id = str(int(time.time())) # Initialize NiFi API config.nifi_config.host = _NIFI_API_URL config.registry_config.host = _NIFIREG_API_URL # Get NiFi root PG root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id') # Get EFM flow flow_id, efm_pg_id = efm_get_flow('iot-1') return (run_id, root_pg, efm_pg_id, flow_id)
def test_get_component_connections(regress_nifi, fix_proc): f_p1 = fix_proc.generate() f_p2 = fix_proc.generate() f_p3 = canvas.create_processor( parent_pg=canvas.get_process_group(canvas.get_root_pg_id(), 'id'), processor=canvas.get_processor_type('AttributesToJSON'), location=(400.0, 425.0), name=conftest.test_processor_name + '_inbound') canvas.create_connection(f_p1, f_p3, name=conftest.test_basename) canvas.create_connection(f_p2, f_p3, name=conftest.test_basename) r1 = canvas.get_component_connections(f_p1) assert len(r1) == 1 assert r1[0].source_id == f_p1.id r2 = canvas.get_component_connections(f_p3) assert len(r2) == 2 assert r2[0].destination_id == f_p3.id assert r2[1].source_id in [f_p1.id, f_p2.id]
def create_pg_snippet(pg_id): """ Creates a snippet of the targetted process group, and returns the object :param pg_id: ID of the process Group to snippet :return: Snippet Object """ target_pg = get_process_group(pg_id, 'id') new_snippet_req = nifi.SnippetEntity( snippet={ 'processGroups': { target_pg.id: target_pg.revision }, 'parentGroupId': target_pg.nipyapi_extended.process_group_flow.parent_group_id }) snippet_resp = nifi.SnippetsApi().create_snippet(new_snippet_req) return snippet_resp
def set_environment(): # Initialize NiFi API config.nifi_config.host = _get_api_url() config.registry_config.host = nifireg.get_api_url() if is_tls_enabled(): security.set_service_ssl_context(service='nifi', ca_file=get_truststore_path()) security.set_service_ssl_context(service='registry', ca_file=get_truststore_path()) security.service_login(service='nifi', username='******', password=get_the_pwd()) security.service_login(service='registry', username='******', password=get_the_pwd()) # Get NiFi root PG root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id') # Get EFM flow flow_id, efm_pg_id = efm.get_flow('iot-1') return root_pg, efm_pg_id, flow_id
def test_get_snippet(self): from nipyapi.nifi import SnippetEntity t_id = canvas.get_process_group('nipyapi_test_0').id r = templates.create_pg_snippet(t_id) assert isinstance(r, SnippetEntity)
args = parser.parse_args() nifi_host = args.nifi_host instance = args.instance zookeeper_list = args.zookeeper_list if not nifi_host.endswith("/"): nifi_host = nifi_host + "/" nipyapi.config.nifi_config.host = nifi_host + "nifi-api" root_pg_id = nipyapi.canvas.get_root_pg_id() print(root_pg_id) root_pg = canvas.get_process_group(root_pg_id, 'id') location_x = 2000 location_y = 2000 location = (location_x, location_y) template_entity = nipyapi.templates.upload_template(root_pg_id, 'jangowave_demo.xml') flow = nipyapi.templates.deploy_template(root_pg_id, template_entity.id, 2000, 2000) jd = canvas.get_process_group('jangowave_demo') for cs in canvas.list_all_processors(jd.id): if cs.status.run_status != "ENABLED" and cs.component.name == "RecordIngest":