def test_create_process_group(regress_nifi):
r = canvas.create_process_group(parent_pg=canvas.get_process_group(
canvas.get_root_pg_id(), 'id'),
new_pg_name=conftest.test_pg_name,
location=(400.0, 400.0),
comment='some comment')
assert r.component.name == conftest.test_pg_name
assert r.position.x == r.position.y == 400
assert r.component.parent_group_id == canvas.get_root_pg_id()
assert isinstance(r, nifi.ProcessGroupEntity)
# Test process group creation on other than root process group.
s = canvas.create_process_group(parent_pg=canvas.get_process_group(
conftest.test_pg_name),
location=(200.0, 200.0),
new_pg_name=conftest.test_another_pg_name)
assert s.component.name == conftest.test_another_pg_name
assert s.position.x == s.position.y == 200
assert s.component.parent_group_id == canvas.get_process_group(
conftest.test_pg_name, "name").id
assert isinstance(s, nifi.ProcessGroupEntity)
with pytest.raises(ValueError):
parent_pg = canvas.get_process_group('NiFi Flow')
parent_pg.id = 'invalid'
_ = canvas.create_process_group(parent_pg, 'irrelevant', (0, 0))
def test_create_process_group(regress):
r = canvas.create_process_group(canvas.get_process_group(
canvas.get_root_pg_id(), 'id'),
config.test_pg_name,
location=(400.0, 400.0))
assert r.component.name == config.test_pg_name
assert r.position.x == r.position.y == 400
assert r.component.parent_group_id == canvas.get_root_pg_id()
with pytest.raises(ApiException):
parent_pg = canvas.get_process_group('NiFi Flow')
parent_pg.id = 'invalid'
_ = canvas.create_process_group(parent_pg, 'irrelevant', (0, 0))
def test_get_process_group(regress_nifi, fix_pg):
with pytest.raises(AssertionError):
_ = canvas.get_process_group('nipyapi_test', 'invalid')
f_pg = fix_pg.generate()
pg1 = canvas.get_process_group(f_pg.id, 'id')
assert isinstance(pg1, ProcessGroupEntity)
duplicate_pg = fix_pg.generate()
pg2 = canvas.get_process_group(duplicate_pg.id, 'id')
assert pg2.id != pg1.id
pg_list = canvas.get_process_group(f_pg.status.name)
assert isinstance(pg_list, list)
# the two duplicates, and root = 3
assert len(pg_list) == 3
def test_schedule_process_group(fix_proc, fix_pg):
f_pg = fix_pg.generate()
_ = fix_proc.generate(parent_pg=f_pg)
r1 = canvas.schedule_process_group(f_pg.id, True)
status = canvas.get_process_group(f_pg.id, 'id')
assert r1 is True
assert status.running_count == 1
r2 = canvas.schedule_process_group(f_pg.id, False)
assert r2 is True
status = canvas.get_process_group(f_pg.id, 'id')
assert status.running_count == 0
assert status.stopped_count == 1
with pytest.raises(AssertionError):
_ = canvas.schedule_process_group(f_pg.id, 'BANANA')
def setUpClass(cls):
super(CanvasNavigatorTest, cls).setUpClass()
print("Start of tests: preparing nifi objects")
config.nifi_config.host = 'http://192.168.56.5:8080/nifi-api'
root = canvas.get_process_group(canvas.get_root_pg_id(), 'id')
# Create new process group in root
CanvasNavigatorTest.pg_parent = canvas.create_process_group(
root, "parent", CANVAS_CENTER)
# Create new child process groups in unittest
CanvasNavigatorTest.pg_child1 = canvas.create_process_group(
CanvasNavigatorTest.pg_parent, "child1", CANVAS_CENTER)
CanvasNavigatorTest.pg_child2 = canvas.create_process_group(
CanvasNavigatorTest.pg_parent, "child2", CANVAS_CENTER)
CanvasNavigatorTest.pg_child2_2 = canvas.create_process_group(
CanvasNavigatorTest.pg_parent, "child2", CANVAS_CENTER)
CanvasNavigatorTest.pg_grandchild1 = canvas.create_process_group(
CanvasNavigatorTest.pg_child1, "grandchild1", CANVAS_CENTER)
# Create other objects as well
CanvasNavigatorTest.proc = canvas.create_processor(
CanvasNavigatorTest.pg_parent,
canvas.get_processor_type("GenerateFlowFile"), CANVAS_CENTER,
"proc")
CanvasNavigatorTest.input_port = canvas.create_port(
CanvasNavigatorTest.pg_parent.component.id, "INPUT_PORT",
"input_port", "STOPPED", CANVAS_CENTER)
CanvasNavigatorTest.output_port = canvas.create_port(
CanvasNavigatorTest.pg_parent.component.id, "OUTPUT_PORT",
"output_port", "STOPPED", CANVAS_CENTER)
CanvasNavigatorTest.controller = canvas.create_controller(
CanvasNavigatorTest.pg_parent,
canvas.list_all_controller_types()[0], "controller")
def test_complex_template_versioning(regress_flow_reg, fix_ctv):
# There is a complex bug where a new flow version cannot be switched to
# and generates a NiFi NPE if attempted when create_flow_version is used
# BUG FIXED: issue with variable name found in Swagger definition
# https://github.com/apache/nifi/pull/2479#issuecomment-366582829
# Create a new flow version
vers = versioning.list_flow_versions(fix_ctv.bucket.identifier,
fix_ctv.flow.identifier)
assert vers[0].version == 2
# create_flow_version is the problem
new_ss = versioning.create_flow_version(
flow=fix_ctv.flow,
flow_snapshot=fix_ctv.snapshot_w_template,
refresh=True)
assert isinstance(new_ss, registry.VersionedFlowSnapshot)
vers = versioning.list_flow_versions(fix_ctv.bucket.identifier,
fix_ctv.flow.identifier)
assert vers[0].version == 3
new_ver_info = versioning.get_version_info(fix_ctv.pg)
r1 = versioning.update_flow_ver(fix_ctv.pg)
assert r1.request.complete is True
assert r1.request.failure_reason is None
r2 = canvas.schedule_process_group(fix_ctv.pg.id, True)
status = canvas.get_process_group(fix_ctv.pg.id, 'id')
assert status.running_count >= 5
with pytest.raises(ValueError):
_ = versioning.update_flow_ver(fix_ctv.pg, 'bob')
with pytest.raises(ValueError):
_ = versioning.update_flow_ver(fix_ctv.pg, '9999999')
def test_save_flow_ver(fixture_registry_bucket, fixture_pg, fixture_processor):
test_pg = fixture_pg.generate()
test_b, test_rc = fixture_registry_bucket
r1 = versioning.save_flow_ver(process_group=test_pg,
registry_client=test_rc,
bucket=test_b,
flow_name=config.test_versioned_flow_name,
comment='a test comment',
desc='a test description')
assert isinstance(r1, nifi.VersionControlInformationEntity)
# Next we validate you can't duplicate a flow name in a bucket
with pytest.raises(ValueError):
_ = versioning.save_flow_ver(process_group=test_pg,
registry_client=test_rc,
bucket=test_b,
flow_name=config.test_versioned_flow_name,
comment='NiPyApi Test',
desc='NiPyApi Test')
# Add a processor, refresh status, and save a new version
fixture_processor.generate(parent_pg=test_pg)
test_pg = canvas.get_process_group(test_pg.id, 'id')
r2 = versioning.save_flow_ver(
process_group=test_pg,
registry_client=test_rc,
bucket=test_b,
flow_id=r1.version_control_information.flow_id,
comment='a test comment')
assert isinstance(r2, nifi.VersionControlInformationEntity)
assert r2.version_control_information.version > \
r1.version_control_information.version
def test_list_nested_processors(regress_nifi, fix_pg, fix_proc):
pg_1 = fix_pg.generate(
parent_pg=canvas.get_process_group(canvas.get_root_pg_id(), 'id'))
pg_2 = fix_pg.generate(parent_pg=pg_1)
root_proc_1 = fix_proc.generate()
pg_1_proc_1 = fix_proc.generate(parent_pg=pg_1)
pg_1_proc_2 = fix_proc.generate(parent_pg=pg_1)
pg_2_proc_1 = fix_proc.generate(parent_pg=pg_2)
pg_2_proc_2 = fix_proc.generate(parent_pg=pg_2)
pg_2_proc_3 = fix_proc.generate(parent_pg=pg_2)
pg_2_proc_4 = fix_proc.generate(parent_pg=pg_2)
r1 = [
x for x in canvas.list_all_processors('root')
if conftest.test_basename in x.status.name
]
assert len(r1) == 7
r2 = [
x for x in canvas.list_all_processors(pg_2.id)
if conftest.test_basename in x.status.name
]
assert len(r2) == 4
r3 = [
x for x in canvas.list_all_processors(pg_1.id)
if conftest.test_basename in x.status.name
]
assert len(r3) == 6
def test_current_name(self):
print("Testing current name")
self.assertEqual(
self.nav.current_name(),
canvas.get_process_group(canvas.get_root_pg_id(),
'id').component.name,
'incorrect current name')
def set_environment(run_id):
if not run_id:
run_id = str(int(time.time()))
# Initialize NiFi API
config.nifi_config.host = _get_nifi_api_url()
config.registry_config.host = _get_nifireg_api_url()
if _IS_TLS_ENABLED:
security.set_service_ssl_context(service='nifi', ca_file=_TRUSTSTORE)
security.set_service_ssl_context(service='registry',
ca_file=_TRUSTSTORE)
security.service_login(service='nifi',
username='******',
password='******')
security.service_login(service='registry',
username='******',
password='******')
# Get NiFi root PG
root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id')
# Get EFM flow
flow_id, efm_pg_id = efm_get_flow('iot-1')
return (run_id, root_pg, efm_pg_id, flow_id)
def test_create_template(self):
from nipyapi.nifi import TemplateEntity
t_id = canvas.get_process_group('nipyapi_test_0').id
r = templates.create_template(pg_id=t_id,
name='nipyapi_testTemplate_01',
desc='Nothing Here')
assert isinstance(r, TemplateEntity)
def test_create_controller(regress_nifi, fix_cont):
root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id')
cont_type = canvas.list_all_controller_types()[0]
r1 = canvas.create_controller(parent_pg=root_pg, controller=cont_type)
assert isinstance(r1, nifi.ControllerServiceEntity)
with pytest.raises(AssertionError):
_ = canvas.create_controller('pie', cont_type)
with pytest.raises(AssertionError):
_ = canvas.create_controller(root_pg, 'pie')
def to_process_group(json_process_group):
try:
process_group = canvas.get_process_group(
json_process_group["parent_group_id"], 'id')
process_group.revision.version = 0
return process_group
except Exception as e:
return str(e)
def wait_for_data(timeout_secs=120):
while timeout_secs:
bytes_in = canvas.get_process_group(
PG_NAME, 'name').status.aggregate_snapshot.bytes_in
if bytes_in > 0:
break
timeout_secs -= 1
time.sleep(1)
# wait a few more seconds just to let the pipes to be primed
time.sleep(10)
def wait_for_data(timeout_secs=120):
LOG.info("Setup complete, waiting for data to flow in NiFi")
while timeout_secs:
bytes_in = canvas.get_process_group(PG_NAME, 'name').status.aggregate_snapshot.bytes_in
if bytes_in > 0:
break
timeout_secs -= 1
LOG.info("Data not Flowing yet, sleeping for 3")
time.sleep(3)
# wait a few more seconds just to let the pipes to be primed
time.sleep(10)
def test_client_recursion_limit(fix_pg, fix_funnel, target=450):
# https://github.com/Chaffelson/nipyapi/issues/147
parent_pg = canvas.get_process_group('root')
for i in range(0, target):
parent_pg = fix_pg.generate(parent_pg, str(i))
fix_funnel.generate(parent_pg)
start = time.time()
r1 = canvas.list_all_process_groups(canvas.get_root_pg_id())
end = time.time()
assert len(r1) == target + 1 # +1 to allow for root PG
print("Len {0} Set {1}".format(len(r1), len(set([x.id for x in r1]))))
print("Elapsed r1: {0}".format((end - start)))
def test_save_flow_ver(regress_flow_reg, fix_bucket, fix_pg, fix_proc):
f_reg_client = conftest.ensure_registry_client(
config.registry_local_name
)
f_bucket = fix_bucket()
f_pg = fix_pg.generate()
test_bucket = versioning.get_registry_bucket(f_bucket.identifier, 'id')
assert test_bucket.name == conftest.test_bucket_name
r1 = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=test_bucket,
flow_name=conftest.test_versioned_flow_name,
comment='a test comment',
desc='a test description'
)
assert isinstance(r1, nifi.VersionControlInformationEntity)
# Next we validate you can't duplicate a flow name in a bucket
with pytest.raises(ValueError):
_ = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=f_bucket,
flow_name=conftest.test_versioned_flow_name,
comment='NiPyApi Test',
desc='NiPyApi Test'
)
# Add a processor, refresh status, and save a new version
fix_proc.generate(parent_pg=f_pg)
f_pg = canvas.get_process_group(f_pg.id, 'id')
r2 = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=f_bucket,
flow_id=r1.version_control_information.flow_id,
comment='a test comment'
)
assert isinstance(r2, nifi.VersionControlInformationEntity)
assert r2.version_control_information.version > \
r1.version_control_information.version
with pytest.raises(ValueError):
_ = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=f_bucket,
flow_name=conftest.test_versioned_flow_name,
comment='a test comment',
desc='a test description',
refresh=False
)
# shortcut to clean up the test objects when not using the fixture
conftest.cleanup_reg()
def wait_for_data(pg_name, timeout_secs=120):
while timeout_secs:
pg = canvas.get_process_group(pg_name, 'name')
if pg is None:
break
bytes_in = pg.status.aggregate_snapshot.bytes_in
if bytes_in > 0:
break
timeout_secs -= 1
LOG.info("Data not Flowing yet, sleeping for 3")
time.sleep(3)
# wait a few more seconds just to let the pipes to be primed
time.sleep(10)
def set_environment(run_id):
if not run_id:
run_id = str(int(time.time()))
# Initialize NiFi API
config.nifi_config.host = _NIFI_API_URL
config.registry_config.host = _NIFIREG_API_URL
# Get NiFi root PG
root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id')
# Get EFM flow
flow_id, efm_pg_id = efm_get_flow('iot-1')
return (run_id, root_pg, efm_pg_id, flow_id)
def test_get_component_connections(regress_nifi, fix_proc):
f_p1 = fix_proc.generate()
f_p2 = fix_proc.generate()
f_p3 = canvas.create_processor(
parent_pg=canvas.get_process_group(canvas.get_root_pg_id(), 'id'),
processor=canvas.get_processor_type('AttributesToJSON'),
location=(400.0, 425.0),
name=conftest.test_processor_name + '_inbound')
canvas.create_connection(f_p1, f_p3, name=conftest.test_basename)
canvas.create_connection(f_p2, f_p3, name=conftest.test_basename)
r1 = canvas.get_component_connections(f_p1)
assert len(r1) == 1
assert r1[0].source_id == f_p1.id
r2 = canvas.get_component_connections(f_p3)
assert len(r2) == 2
assert r2[0].destination_id == f_p3.id
assert r2[1].source_id in [f_p1.id, f_p2.id]
def create_pg_snippet(pg_id):
"""
Creates a snippet of the targetted process group, and returns the object
:param pg_id: ID of the process Group to snippet
:return: Snippet Object
"""
target_pg = get_process_group(pg_id, 'id')
new_snippet_req = nifi.SnippetEntity(
snippet={
'processGroups': {
target_pg.id: target_pg.revision
},
'parentGroupId':
target_pg.nipyapi_extended.process_group_flow.parent_group_id
})
snippet_resp = nifi.SnippetsApi().create_snippet(new_snippet_req)
return snippet_resp
def set_environment():
# Initialize NiFi API
config.nifi_config.host = _get_api_url()
config.registry_config.host = nifireg.get_api_url()
if is_tls_enabled():
security.set_service_ssl_context(service='nifi',
ca_file=get_truststore_path())
security.set_service_ssl_context(service='registry',
ca_file=get_truststore_path())
security.service_login(service='nifi',
username='******',
password=get_the_pwd())
security.service_login(service='registry',
username='******',
password=get_the_pwd())
# Get NiFi root PG
root_pg = canvas.get_process_group(canvas.get_root_pg_id(), 'id')
# Get EFM flow
flow_id, efm_pg_id = efm.get_flow('iot-1')
return root_pg, efm_pg_id, flow_id
def test_get_snippet(self):
from nipyapi.nifi import SnippetEntity
t_id = canvas.get_process_group('nipyapi_test_0').id
r = templates.create_pg_snippet(t_id)
assert isinstance(r, SnippetEntity)
args = parser.parse_args()
nifi_host = args.nifi_host
instance = args.instance
zookeeper_list = args.zookeeper_list
if not nifi_host.endswith("/"):
nifi_host = nifi_host + "/"
nipyapi.config.nifi_config.host = nifi_host + "nifi-api"
root_pg_id = nipyapi.canvas.get_root_pg_id()
print(root_pg_id)
root_pg = canvas.get_process_group(root_pg_id, 'id')
location_x = 2000
location_y = 2000
location = (location_x, location_y)
template_entity = nipyapi.templates.upload_template(root_pg_id,
'jangowave_demo.xml')
flow = nipyapi.templates.deploy_template(root_pg_id, template_entity.id, 2000,
2000)
jd = canvas.get_process_group('jangowave_demo')
for cs in canvas.list_all_processors(jd.id):
if cs.status.run_status != "ENABLED" and cs.component.name == "RecordIngest":