def test_list_all_connections(regress_nifi, fix_proc):
f_p1 = fix_proc.generate()
f_p2 = fix_proc.generate()
r1 = [
x for x in canvas.list_all_connections()
if conftest.test_basename in x.component.name
]
assert not r1
# connect single relationship
c1 = canvas.create_connection(f_p1, f_p2, ['success'],
conftest.test_basename)
r2 = [
x for x in canvas.list_all_connections('root')
if conftest.test_basename in x.component.name
]
assert len(r2) == 1
r3 = [
x for x in canvas.list_all_connections(canvas.get_root_pg_id())
if conftest.test_basename in x.component.name
]
assert len(r3) == 1
assert isinstance(r2[0], nifi.ConnectionEntity)
c2 = canvas.create_connection(f_p1, f_p2, name=conftest.test_basename)
r2 = [
x for x in canvas.list_all_connections('root')
if conftest.test_basename in x.component.name
]
assert len(r2) == 2
_ = canvas.delete_connection(c1)
_ = canvas.delete_connection(c2)
r3 = [
x for x in canvas.list_all_connections('root')
if conftest.test_basename in x.component.name
]
assert not r3
def create(self, destination_type = None, destination_group = None):
if destination_group == None:
destination_group = self.group
if destination_type == None:
destination_type = 'PROCESSOR'
canvas.create_connection(
parent_pg=self.group,
connection=nifi.ConnectionDTO(
source=nifi.ConnectableDTO(
id=self.source.id,
type='PROCESSOR',
group_id=self.group.id
),
destination=nifi.ConnectableDTO(
id=self.destination.id,
type=destination_type,
group_id=destination_group.id
),
selected_relationships=self.selected_relationships,
),
name=self.name
)
return self.destination
def step2_edge_flow(env):
# Create input port and funnel in NiFi
env.from_gw = canvas.create_port(env.root_pg.id, 'INPUT_PORT',
'from Gateway', 'STOPPED', (0, 200))
funnel_position = (96, 350)
env.temp_funnel = create_funnel(env.root_pg.id, (96, 350))
canvas.create_connection(env.from_gw, env.temp_funnel)
# Create flow in EFM
env.consume_mqtt = efm_create_processor(
env.flow_id, env.efm_pg_id, 'ConsumeMQTT',
'org.apache.nifi.processors.mqtt.ConsumeMQTT', (100, 100), {
'Broker URI': 'tcp://edge2ai-1.dim.local:1883',
'Client ID': 'minifi-iot',
'Topic Filter': 'iot/#',
'Max Queue Size': '60',
})
env.nifi_rpg = efm_create_remote_processor_group(env.flow_id,
env.efm_pg_id,
'Remote PG', _NIFI_URL,
'HTTP', (100, 400))
env.consume_conn = efm_create_connection(env.flow_id,
env.efm_pg_id,
env.consume_mqtt,
'PROCESSOR',
env.nifi_rpg,
'REMOTE_INPUT_PORT', ['Message'],
destination_port=env.from_gw.id)
# Create a bucket in NiFi Registry to save the edge flow versions
if not versioning.get_registry_bucket('IoT'):
versioning.create_registry_bucket('IoT')
# Publish/version the flow
efm_publish_flow(env.flow_id, 'First version - ' + env.run_id)
def test_create_connection_funnels(regress_nifi, fix_proc, fix_funnel):
f_p1 = fix_proc.generate()
f_f1 = fix_funnel.generate()
r1 = canvas.create_connection(source=f_p1, target=f_f1)
assert isinstance(r1, nifi.ConnectionEntity)
f_p2 = fix_proc.generate()
r2 = canvas.create_connection(source=f_f1, target=f_p2)
assert isinstance(r2, nifi.ConnectionEntity)
def test_create_connection_processors(regress_nifi, fix_proc):
f_p1 = fix_proc.generate()
f_p2 = fix_proc.generate()
# connect single relationship
r1 = canvas.create_connection(f_p1, f_p2, ['success'],
conftest.test_basename)
assert isinstance(r1, nifi.ConnectionEntity)
# connect all relationships by default
r2 = canvas.create_connection(f_p1, f_p2, name=conftest.test_basename)
assert isinstance(r2, nifi.ConnectionEntity)
with pytest.raises(AssertionError):
_ = canvas.create_connection(f_p1, f_p2, ['not a connection'])
def setUpClass(cls):
super(Test1ToNTest, cls).setUpClass()
print("Start of tests: preparing nifi objects")
config.nifi_config.host = 'http://192.168.56.5:8080/nifi-api'
flow_name = "Test1ToNTest"
nav = CanvasNavigator()
# Delete all leftovers from previous (failed?) tests
pgs_to_be_deleted = nav.groups(flow_name)
for pg in pgs_to_be_deleted:
canvas.delete_process_group(pg, force=True)
# Create new process group in root
Test1ToNTest.pg_test = canvas.create_process_group(nav.current, flow_name, (0, 0))
# Create simple flow to test
Test1ToNTest.proc_start = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("GenerateFlowFile"),
CANVAS_CENTER,
"Start")
Test1ToNTest.proc_2 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"Processor 2")
Test1ToNTest.proc_3 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"Processor 3")
Test1ToNTest.proc_end_1 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"End 1")
Test1ToNTest.proc_end_2 = canvas.create_processor(
Test1ToNTest.pg_test,
canvas.get_processor_type("DebugFlow"),
CANVAS_CENTER,
"End 2")
canvas.update_processor(Test1ToNTest.proc_end_1,
nifi.ProcessorConfigDTO(auto_terminated_relationships=["success", "failure"]))
canvas.update_processor(Test1ToNTest.proc_end_2,
nifi.ProcessorConfigDTO(auto_terminated_relationships=["success", "failure"]))
Test1ToNTest.conn_1 = canvas.create_connection(Test1ToNTest.proc_start, Test1ToNTest.proc_2, ["success"])
Test1ToNTest.conn_2 = canvas.create_connection(Test1ToNTest.proc_2, Test1ToNTest.proc_3, ["success", "failure"])
Test1ToNTest.conn_3 = canvas.create_connection(Test1ToNTest.proc_3, Test1ToNTest.proc_end_1, ["success", "failure"])
Test1ToNTest.conn_4 = canvas.create_connection(Test1ToNTest.proc_3, Test1ToNTest.proc_end_2, ["success", "failure"])
canvas.schedule_process_group(Test1ToNTest.pg_test.component.id, scheduled=True)
def test_list_all_connections(regress_nifi, fix_pg, fix_proc):
f_p1 = fix_proc.generate()
f_p2 = fix_proc.generate()
r1 = [
x for x in canvas.list_all_connections()
if conftest.test_basename in x.component.name
]
assert not r1
# connect single relationship
c1 = canvas.create_connection(f_p1, f_p2, ['success'],
conftest.test_basename)
r2 = [
x for x in canvas.list_all_connections('root')
if conftest.test_basename in x.component.name
]
assert len(r2) == 1
r3 = [
x for x in canvas.list_all_connections(canvas.get_root_pg_id())
if conftest.test_basename in x.component.name
]
assert len(r3) == 1
assert isinstance(r2[0], nifi.ConnectionEntity)
c2 = canvas.create_connection(f_p1, f_p2, name=conftest.test_basename)
r2 = [
x for x in canvas.list_all_connections('root')
if conftest.test_basename in x.component.name
]
assert len(r2) == 2
_ = canvas.delete_connection(c1)
_ = canvas.delete_connection(c2)
r4 = [
x for x in canvas.list_all_connections('root')
if conftest.test_basename in x.component.name
]
assert not r4
# Test Issue #129 - nested PGs with descendents missing nested content
f_pg1 = fix_pg.generate()
f_pg2 = fix_pg.generate(parent_pg=f_pg1)
f_p3 = fix_proc.generate(parent_pg=f_pg2)
f_p4 = fix_proc.generate(parent_pg=f_pg2)
c2 = canvas.create_connection(f_p3, f_p4, ['success'],
conftest.test_basename)
r5 = [
x for x in canvas.list_all_connections(f_pg2.id)
if conftest.test_basename in x.component.name
]
assert len(r5) == 1
assert r5[0].id == c2.id
def test_get_component_connections(regress_nifi, fix_proc):
f_p1 = fix_proc.generate()
f_p2 = fix_proc.generate()
f_p3 = canvas.create_processor(
parent_pg=canvas.get_process_group(canvas.get_root_pg_id(), 'id'),
processor=canvas.get_processor_type('AttributesToJSON'),
location=(400.0, 425.0),
name=conftest.test_processor_name + '_inbound')
canvas.create_connection(f_p1, f_p3, name=conftest.test_basename)
canvas.create_connection(f_p2, f_p3, name=conftest.test_basename)
r1 = canvas.get_component_connections(f_p1)
assert len(r1) == 1
assert r1[0].source_id == f_p1.id
r2 = canvas.get_component_connections(f_p3)
assert len(r2) == 2
assert r2[0].destination_id == f_p3.id
assert r2[1].source_id in [f_p1.id, f_p2.id]
def lab2_edge_flow(self):
# Create input port and funnel in NiFi
self.context.from_gw = canvas.create_port(self.context.root_pg.id,
'INPUT_PORT', 'from Gateway',
'STOPPED', (0, 200))
self.context.temp_funnel = nf.create_funnel(self.context.root_pg.id,
(96, 350))
canvas.create_connection(self.context.from_gw,
self.context.temp_funnel)
canvas.schedule_components(self.context.root_pg.id, True,
[self.context.from_gw])
# Create flow in EFM
self.context.consume_mqtt = efm.create_processor(
self.context.flow_id, self.context.efm_pg_id, 'ConsumeMQTT',
'org.apache.nifi.processors.mqtt.ConsumeMQTT', (100, 100), {
'Broker URI':
'tcp://{hostname}:1883'.format(hostname=get_hostname()),
'Client ID':
'minifi-iot',
'Topic Filter':
'iot/#',
'Max Queue Size':
'60',
})
self.context.nifi_rpg = efm.create_remote_processor_group(
self.context.flow_id, self.context.efm_pg_id, 'Remote PG',
nf.get_url(), 'HTTP', (100, 400))
self.context.consume_conn = efm.create_connection(
self.context.flow_id,
self.context.efm_pg_id,
self.context.consume_mqtt,
'PROCESSOR',
self.context.nifi_rpg,
'REMOTE_INPUT_PORT', ['Message'],
destination_port=self.context.from_gw.id,
name='Sensor data',
flow_file_expiration='60 seconds')
# Create a bucket in NiFi Registry to save the edge flow versions
if not versioning.get_registry_bucket('IoT'):
versioning.create_registry_bucket('IoT')
# Publish/version the flow
efm.publish_flow(self.context.flow_id,
'First version - {}'.format(self.run_id))
def __build_inputs(self):
# Create http context map for communication
self.logger.debug(
"Creating 'StandardHttpContextMap' for communication")
self.http_context = canvas_ext.create_http_context_map(
self.test_group, self.name)
# Keep track of "cursor" location on canvas
location = Location()
# Create http request for starting a test
self.logger.debug("Creating 'HandleHttpRequest' for starting a test")
self.http_in = canvas_ext.create_request_handler(
self.test_group, location, self.http_context, self.port)
location.y += 200
# Set start time
self.logger.debug("Creating 'UpdateAttribute' to set test start time")
in_attribute = canvas_ext.create_input_attribute(
self.test_group, location, "Set test start time")
location.y += 200
self.logger.debug(
"Connecting 'HandleHttpRequest' with 'UpdateAttribute'")
canvas.create_connection(self.http_in, in_attribute, ["success"])
# Route request to correct port
self.logger.debug(
"Creating 'RouteOnAttribute' to send each request to the correct input port"
)
in_route = canvas_ext.create_input_router(self.test_group, location,
self.inputs)
location.y += 200
self.logger.debug(
"Connecting 'UpdateAttribute' with 'RouteOnAttribute'")
canvas.create_connection(in_attribute, in_route, ["success"])
location.y += 100 # Taking some extra vertical space, because we can have so many ports
for test_input in self.inputs:
input_name = test_input.component.name
self.logger.debug("Creating port for input '%s'", input_name)
output_port = canvas_ext.create_test_input(self.test_group,
location, input_name)
location.x += 400
self.logger.debug("Connecting 'RouteOnAttribute' to port '%s'",
input_name)
canvas.create_connection(in_route, output_port, [input_name])
self.logger.debug("Connecting port '%s' to processor '%s'",
input_name, input_name)
canvas.create_connection(output_port, test_input)
def test_delete_connection(regress_nifi, fix_proc):
f_p1 = fix_proc.generate()
f_p2 = fix_proc.generate()
# connect single relationship
c1 = canvas.create_connection(f_p1, f_p2, ['success'],
conftest.test_basename)
r1 = canvas.delete_connection(c1)
assert isinstance(r1, nifi.ConnectionEntity)
assert r1.status is None
def test_connect_output_ports(regress_nifi, fix_pg):
f_pg_1 = fix_pg.generate()
f_pg_2 = fix_pg.generate()
f_pg_1_output = canvas.create_port(f_pg_1.id, 'OUTPUT_PORT',
conftest.test_basename + 'output',
'STOPPED')
f_pg_2_input = canvas.create_port(f_pg_2.id, 'INPUT_PORT',
conftest.test_basename + 'input',
'STOPPED')
r1 = canvas.create_connection(source=f_pg_1_output,
target=f_pg_2_input,
name=conftest.test_basename)
assert isinstance(r1, nifi.ConnectionEntity)
def lab4_nifi_flow(env):
LOG.info("Running step4_nifi_flow")
# Create a bucket in NiFi Registry to save the edge flow versions
env.sensor_bucket = versioning.get_registry_bucket('SensorFlows')
if not env.sensor_bucket:
env.sensor_bucket = versioning.create_registry_bucket('SensorFlows')
# Create NiFi Process Group
env.reg_client = versioning.create_registry_client('NiFi Registry', _NIFIREG_URL, 'The registry...')
env.sensor_pg = canvas.create_process_group(env.root_pg, PG_NAME, (330, 350))
#env.sensor_flow = versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_name='SensorProcessGroup', comment='Enabled version control - ' + env.run_id)
env.sensor_flow = save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_name='SensorProcessGroup', comment='Enabled version control - ' + str(env.run_id))
# Create controller services
env.sr_svc = create_controller(env.sensor_pg, 'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry', {'url': _SCHREG_API_URL}, True)
env.json_reader_svc = create_controller(env.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {'schema-access-strategy': 'schema-name', 'schema-registry': env.sr_svc.id}, True)
env.json_writer_svc = create_controller(env.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter', {'schema-access-strategy': 'schema-name', 'schema-registry': env.sr_svc.id, 'Schema Write Strategy': 'hwx-schema-ref-attributes'}, True)
# Create flow
sensor_port = canvas.create_port(env.sensor_pg.id, 'INPUT_PORT', 'Sensor Data', 'RUNNING', (0, 0))
upd_attr = create_processor(env.sensor_pg, 'Set Schema Name', 'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100),
{
'properties': {
'schema.name': 'SensorReading',
},
}
)
canvas.create_connection(sensor_port, upd_attr)
pub_kafka = create_processor(env.sensor_pg, 'Publish to Kafka topic: iot', 'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0', (0, 300),
{
'properties': {
'bootstrap.servers': 'edge2ai-1.dim.local:9092',
'topic': 'iot',
'record-reader': env.json_reader_svc.id,
'record-writer': env.json_writer_svc.id,
'use-transactions': 'false',
'attribute-name-regex': 'schema.*',
'client.id': PRODUCER_CLIENT_ID,
},
'autoTerminatedRelationships': ['success'],
}
)
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = create_funnel(env.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
#versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='First version - ' + env.run_id)
save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='First version - ' + str(env.run_id))
# Start flow
canvas.schedule_process_group(env.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
update_connection(env.from_gw, env.temp_funnel, sensor_port)
def test_recreate_connection(self):
nav = CanvasNavigator()
loc = Location()
name = "nipytest - unit test - test_recreate_connection"
# Testing incorrect inputs
with self.assertRaises(AssertionError):
canvas_ext.recreate_connection(1)
# Create connection
input_port = canvas_ext.create_test_output(nav.current, loc, name)
output_port = canvas_ext.create_test_input(nav.current, loc, name)
connection = canvas.create_connection(input_port, output_port)
canvas.delete_connection(connection)
# Run function
new_connection = canvas_ext.recreate_connection(connection)
# If output is ok type, function was successful
self.assertIsInstance(new_connection, nifi.ConnectionEntity)
self.assertNotEqual(connection.component.id,
new_connection.component.id)
# Remove temporary created object(s)
canvas.delete_connection(new_connection)
canvas.delete_port(input_port)
canvas.delete_port(output_port)
def __build_outputs(self):
# First creating response, so we can connect all outputs immediately in the loop
location = Location(0, 1200)
self.logger.debug(
"Creating RouteOnAttribute for filtering only test results")
out_route = canvas_ext.create_output_router(self.test_group, location,
self.name)
location.y += 200
self.logger.debug("Creating HandleHttpResponse for test results")
self.http_out = canvas_ext.create_response_handler(
self.test_group, location, self.http_context)
canvas.create_connection(out_route, self.http_out, ["test"])
location.x = 0
for output in self.outputs:
location.y = 800
output_name = output.component.name
self.logger.debug("Creating input_port for output '%s'",
output_name)
input_port = canvas_ext.create_test_output(self.test_group,
location, output_name)
location.y += 200
self.logger.debug("Connecting processor '%s' to input_port '%s'",
output_name, output_name)
canvas.create_connection(output, input_port)
self.logger.debug("Creating UpdateAttribute for output '%s'",
output_name)
update = canvas_ext.create_output_attribute(
self.test_group, location, output_name)
canvas.create_connection(input_port, update)
canvas.create_connection(update, out_route)
location.x += 400
def lab4_rest_and_kudu(self):
# Prepare Impala/Kudu table
kudu.create_table()
kudu_table_name = kudu.get_kudu_table_name('default', 'sensors')
# Set required variables
if not self.context.skip_cdsw:
# Set the variable with the CDSW access key
canvas.update_variable_registry(
self.context.sensor_pg,
[('cdsw.access.key', cdsw.get_model_access_key())])
# Set the variable with the CDSW model API key
canvas.update_variable_registry(
self.context.sensor_pg,
[('cdsw.model.api.key', cdsw.create_model_api_key())])
# Create controllers
self.context.json_reader_with_schema_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'hwx-schema-ref-attributes',
'schema-registry': self.context.sr_svc.id
},
True,
name='JsonTreeReader - With schema identifier')
props = {
'rest-lookup-url': cdsw.get_model_endpoint_url(),
'rest-lookup-record-reader': self.context.json_reader_svc.id,
'rest-lookup-record-path': '/response',
'Authorization': 'Bearer ${cdsw.model.api.key}',
}
if is_tls_enabled():
props.update({
'rest-lookup-ssl-context-service':
self.context.ssl_svc.id,
})
rest_lookup_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.lookup.RestLookupService',
props, True)
# Build flow
fail_funnel = nf.create_funnel(self.context.sensor_pg.id, (1400, 340))
props = {
'topic': 'iot',
'topic_type': 'names',
'record-reader': self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.json_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'consumer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
consume_kafka = nf.create_processor(
self.context.sensor_pg, 'Consume Kafka iot messages', [
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_0'
], (700, 0), {'properties': props})
canvas.create_connection(consume_kafka, fail_funnel, ['parse.failure'])
predict = nf.create_processor(
self.context.sensor_pg, 'Predict machine health',
'org.apache.nifi.processors.standard.LookupRecord', (700, 200), {
'properties': {
'record-reader':
self.context.json_reader_with_schema_svc.id,
'record-writer':
self.context.json_writer_svc.id,
'lookup-service':
rest_lookup_svc.id,
'result-record-path':
'/response',
'routing-strategy':
'route-to-success',
'result-contents':
'insert-entire-record',
'mime.type':
"toString('application/json', 'UTF-8')",
'request.body':
"concat('{\"accessKey\":\"', '${cdsw.access.key}', "
"'\",\"request\":{\"feature\":\"', /sensor_0, ', ', "
"/sensor_1, ', ', /sensor_2, ', ', /sensor_3, ', ', "
"/sensor_4, ', ', /sensor_5, ', ', /sensor_6, ', ', "
"/sensor_7, ', ', /sensor_8, ', ', /sensor_9, ', ', "
"/sensor_10, ', ', /sensor_11, '\"}}')",
'request.method':
"toString('post', 'UTF-8')",
},
})
canvas.create_connection(predict, fail_funnel, ['failure'])
canvas.create_connection(consume_kafka, predict, ['success'])
update_health = nf.create_processor(
self.context.sensor_pg, 'Update health flag',
'org.apache.nifi.processors.standard.UpdateRecord', (700, 400), {
'properties': {
'record-reader':
self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.json_writer_svc.id,
'replacement-value-strategy': 'record-path-value',
'/is_healthy': '/response/result',
},
})
canvas.create_connection(update_health, fail_funnel, ['failure'])
canvas.create_connection(predict, update_health, ['success'])
write_kudu = nf.create_processor(
self.context.sensor_pg, 'Write to Kudu',
'org.apache.nifi.processors.kudu.PutKudu', (700, 600), {
'properties': {
'Kudu Masters':
get_hostname() + ':7051',
'Table Name':
kudu_table_name,
'record-reader':
self.context.json_reader_with_schema_svc.id,
'kerberos-credentials-service':
self.context.keytab_svc.id if is_tls_enabled() else None,
},
})
canvas.create_connection(write_kudu, fail_funnel, ['failure'])
canvas.create_connection(update_health, write_kudu, ['success'])
props = {
'topic': 'iot_enriched',
'record-reader': self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.json_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'producer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
pub_kafka_enriched = nf.create_processor(
self.context.sensor_pg, 'Publish to Kafka topic: iot_enriched', [
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0'
], (300, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched,
['success'])
props = {
'topic': 'iot_enriched_avro',
'record-reader': self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.avro_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'producer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
pub_kafka_enriched_avro = nf.create_processor(
self.context.sensor_pg,
'Publish to Kafka topic: iot_enriched_avro', [
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0'
], (-100, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched_avro,
['success'])
monitor_activity = nf.create_processor(
self.context.sensor_pg, 'Monitor Activity',
'org.apache.nifi.processors.standard.MonitorActivity', (700, 800),
{
'properties': {
'Threshold Duration': '45 secs',
'Continually Send Messages': 'true',
},
'autoTerminatedRelationships':
['activity.restored', 'success'],
})
canvas.create_connection(monitor_activity, fail_funnel, ['inactive'])
canvas.create_connection(write_kudu, monitor_activity, ['success'])
# Version flow
nifireg.save_flow_ver(self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_id=self.context.sensor_flow.
version_control_information.flow_id,
comment='Second version - {}'.format(
self.run_id))
# Start everything
canvas.schedule_process_group(self.context.root_pg.id, True)
def lab2_nifi_flow(self):
# Create a bucket in NiFi Registry to save the edge flow versions
self.context.sensor_bucket = versioning.get_registry_bucket(
'SensorFlows')
if not self.context.sensor_bucket:
self.context.sensor_bucket = versioning.create_registry_bucket(
'SensorFlows')
# Create NiFi Process Group
self.context.reg_client = versioning.create_registry_client(
'NiFi Registry', nifireg.get_url(), 'The registry...')
self.context.sensor_pg = canvas.create_process_group(
self.context.root_pg, PG_NAME, (330, 350))
self.context.sensor_flow = nifireg.save_flow_ver(
self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_name='SensorProcessGroup',
comment='Enabled version control - {}'.format(self.run_id))
# Update default SSL context controller service
ssl_svc_name = 'Default NiFi SSL Context Service'
if is_tls_enabled():
props = {
'SSL Protocol': 'TLS',
'Truststore Type': 'JKS',
'Truststore Filename':
'/opt/cloudera/security/jks/truststore.jks',
'Truststore Password': get_the_pwd(),
'Keystore Type': 'JKS',
'Keystore Filename': '/opt/cloudera/security/jks/keystore.jks',
'Keystore Password': get_the_pwd(),
'key-password': get_the_pwd(),
}
self.context.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
if self.context.ssl_svc:
canvas.schedule_controller(self.context.ssl_svc, False)
self.context.ssl_svc = canvas.get_controller(
ssl_svc_name, 'name')
canvas.update_controller(
self.context.ssl_svc,
nifi.ControllerServiceDTO(properties=props))
self.context.ssl_svc = canvas.get_controller(
ssl_svc_name, 'name')
canvas.schedule_controller(self.context.ssl_svc, True)
else:
self.context.keytab_svc = nf.create_controller(
self.context.root_pg,
'org.apache.nifi.ssl.StandardRestrictedSSLContextService',
props,
True,
name=ssl_svc_name)
# Create controller services
if is_tls_enabled():
self.context.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
props = {
'Kerberos Keytab': '/keytabs/admin.keytab',
'Kerberos Principal': 'admin',
}
self.context.keytab_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.kerberos.KeytabCredentialsService', props,
True)
else:
self.context.ssl_svc = None
self.context.keytab_svc = None
props = {
'url': schreg.get_api_url(),
}
if is_tls_enabled():
props.update({
'kerberos-credentials-service':
self.context.keytab_svc.id,
'ssl-context-service':
self.context.ssl_svc.id,
})
self.context.sr_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry',
props, True)
self.context.json_reader_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id
}, True)
self.context.json_writer_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter',
{
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id,
'Schema Write Strategy': 'hwx-schema-ref-attributes'
}, True)
self.context.avro_writer_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.avro.AvroRecordSetWriter',
{
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id,
'Schema Write Strategy': 'hwx-content-encoded-schema'
}, True)
# Create flow
sensor_port = canvas.create_port(self.context.sensor_pg.id,
'INPUT_PORT', 'Sensor Data',
'STOPPED', (0, 0))
upd_attr = nf.create_processor(
self.context.sensor_pg, 'Set Schema Name',
'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100),
{
'properties': {
'schema.name': 'SensorReading',
},
})
canvas.create_connection(sensor_port, upd_attr)
props = {
'topic': 'iot',
'record-reader': self.context.json_reader_svc.id,
'record-writer': self.context.json_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'producer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
pub_kafka = nf.create_processor(
self.context.sensor_pg, 'Publish to Kafka topic: iot', [
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0'
], (0, 300), {
'properties': props,
'autoTerminatedRelationships': ['success'],
})
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = nf.create_funnel(self.context.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
nifireg.save_flow_ver(self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_id=self.context.sensor_flow.
version_control_information.flow_id,
comment='First version - {}'.format(self.run_id))
# Start flow
canvas.schedule_process_group(self.context.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
nf.update_connection(self.context.from_gw, self.context.temp_funnel,
sensor_port)
canvas.schedule_components(self.context.root_pg.id, True,
[sensor_port])
def lab4_nifi_flow(env):
LOG.info("Running step4_nifi_flow")
# Create a bucket in NiFi Registry to save the edge flow versions
env.sensor_bucket = versioning.get_registry_bucket('SensorFlows')
if not env.sensor_bucket:
env.sensor_bucket = versioning.create_registry_bucket('SensorFlows')
# Create NiFi Process Group
env.reg_client = versioning.create_registry_client('NiFi Registry',
_get_nifireg_url(),
'The registry...')
env.sensor_pg = canvas.create_process_group(env.root_pg, PG_NAME,
(330, 350))
#env.sensor_flow = versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_name='SensorProcessGroup', comment='Enabled version control - ' + env.run_id)
env.sensor_flow = save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_name='SensorProcessGroup',
comment='Enabled version control - ' +
str(env.run_id))
# Update default SSL context controller service
ssl_svc_name = 'Default NiFi SSL Context Service'
if _IS_TLS_ENABLED:
props = {
'SSL Protocol': 'TLS',
'Truststore Type': 'JKS',
'Truststore Filename': '/opt/cloudera/security/jks/truststore.jks',
'Truststore Password': _THE_PWD,
'Keystore Type': 'JKS',
'Keystore Filename': '/opt/cloudera/security/jks/keystore.jks',
'Keystore Password': _THE_PWD,
'key-password': _THE_PWD,
}
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
if env.ssl_svc:
canvas.schedule_controller(env.ssl_svc, False)
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
canvas.update_controller(
env.ssl_svc, nifi.ControllerServiceDTO(properties=props))
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
canvas.schedule_controller(env.ssl_svc, True)
else:
env.keytab_svc = create_controller(
env.root_pg,
'org.apache.nifi.ssl.StandardRestrictedSSLContextService',
props,
True,
name=ssl_svc_name)
# Create controller services
if _IS_TLS_ENABLED:
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
props = {
'Kerberos Keytab': '/keytabs/admin.keytab',
'Kerberos Principal': 'admin',
}
env.keytab_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.kerberos.KeytabCredentialsService',
props, True)
else:
env.ssl_svc = None
env.keytab_svc = None
props = {
'url': _get_schreg_api_url(),
}
if _IS_TLS_ENABLED:
props.update({
'kerberos-credentials-service': env.keytab_svc.id,
'ssl-context-service': env.ssl_svc.id,
})
env.sr_svc = create_controller(
env.sensor_pg,
'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry',
props, True)
env.json_reader_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id
}, True)
env.json_writer_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id,
'Schema Write Strategy': 'hwx-schema-ref-attributes'
}, True)
env.avro_writer_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.avro.AvroRecordSetWriter', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id,
'Schema Write Strategy': 'hwx-content-encoded-schema'
}, True)
# Create flow
sensor_port = canvas.create_port(env.sensor_pg.id, 'INPUT_PORT',
'Sensor Data', 'RUNNING', (0, 0))
upd_attr = create_processor(
env.sensor_pg, 'Set Schema Name',
'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100), {
'properties': {
'schema.name': 'SensorReading',
},
})
canvas.create_connection(sensor_port, upd_attr)
props = {
'topic': 'iot',
'record-reader': env.json_reader_svc.id,
'record-writer': env.json_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'producer'))
pub_kafka = create_processor(
env.sensor_pg, 'Publish to Kafka topic: iot',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0',
(0, 300), {
'properties': props,
'autoTerminatedRelationships': ['success'],
})
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = create_funnel(env.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
#versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='First version - ' + env.run_id)
save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_id=env.sensor_flow.version_control_information.flow_id,
comment='First version - ' + str(env.run_id))
# Start flow
canvas.schedule_process_group(env.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
update_connection(env.from_gw, env.temp_funnel, sensor_port)
def lab7_rest_and_kudu(env):
LOG.info("Running step7_rest_and_kudu")
# Create controllers
env.json_reader_with_schema_svc = create_controller(
env.sensor_pg,
'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'hwx-schema-ref-attributes',
'schema-registry': env.sr_svc.id
},
True,
name='JsonTreeReader - With schema identifier')
props = {
'rest-lookup-url': get_cdsw_altus_api() + '/models/call-model',
'rest-lookup-record-reader': env.json_reader_svc.id,
'rest-lookup-record-path': '/response'
}
if _IS_TLS_ENABLED:
props.update({
'rest-lookup-ssl-context-service': env.ssl_svc.id,
})
rest_lookup_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.lookup.RestLookupService', props, True)
# Build flow
fail_funnel = create_funnel(env.sensor_pg.id, (1400, 340))
props = {
'topic': 'iot',
'topic_type': 'names',
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'consumer'))
consume_kafka = create_processor(
env.sensor_pg, 'Consume Kafka iot messages',
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_0',
(700, 0), {'properties': props})
canvas.create_connection(consume_kafka, fail_funnel, ['parse.failure'])
predict = create_processor(
env.sensor_pg, 'Predict machine health',
'org.apache.nifi.processors.standard.LookupRecord', (700, 200), {
'properties': {
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'lookup-service': rest_lookup_svc.id,
'result-record-path': '/response',
'routing-strategy': 'route-to-success',
'result-contents': 'insert-entire-record',
'mime.type': "toString('application/json', 'UTF-8')",
'request.body':
"concat('{\"accessKey\":\"', '${cdsw.access.key}', '\",\"request\":{\"feature\":\"', /sensor_0, ', ', /sensor_1, ', ', /sensor_2, ', ', /sensor_3, ', ', /sensor_4, ', ', /sensor_5, ', ', /sensor_6, ', ', /sensor_7, ', ', /sensor_8, ', ', /sensor_9, ', ', /sensor_10, ', ', /sensor_11, '\"}}')",
'request.method': "toString('post', 'UTF-8')",
},
})
canvas.create_connection(predict, fail_funnel, ['failure'])
canvas.create_connection(consume_kafka, predict, ['success'])
update_health = create_processor(
env.sensor_pg, 'Update health flag',
'org.apache.nifi.processors.standard.UpdateRecord', (700, 400), {
'properties': {
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'replacement-value-strategy': 'record-path-value',
'/is_healthy': '/response/result',
},
})
canvas.create_connection(update_health, fail_funnel, ['failure'])
canvas.create_connection(predict, update_health, ['success'])
if get_kudu_version() >= (1, 14):
kudu_table_name = 'default.sensors'
else:
kudu_table_name = 'impala::default.sensors'
write_kudu = create_processor(
env.sensor_pg, 'Write to Kudu',
'org.apache.nifi.processors.kudu.PutKudu', (700, 600), {
'properties': {
'Kudu Masters':
_HOSTNAME + ':7051',
'Table Name':
kudu_table_name,
'record-reader':
env.json_reader_with_schema_svc.id,
'kerberos-credentials-service':
env.keytab_svc.id if _IS_TLS_ENABLED else None,
},
})
canvas.create_connection(write_kudu, fail_funnel, ['failure'])
canvas.create_connection(update_health, write_kudu, ['success'])
props = {
'topic': 'iot_enriched',
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'producer'))
pub_kafka_enriched = create_processor(
env.sensor_pg, 'Publish to Kafka topic: iot_enriched',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0',
(300, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched, ['success'])
props = {
'topic': 'iot_enriched_avro',
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.avro_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'producer'))
pub_kafka_enriched_avro = create_processor(
env.sensor_pg, 'Publish to Kafka topic: iot_enriched_avro',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0',
(-100, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched_avro,
['success'])
monitor_activity = create_processor(
env.sensor_pg, 'Monitor Activity',
'org.apache.nifi.processors.standard.MonitorActivity', (700, 800), {
'properties': {
'Threshold Duration': '45 secs',
'Continually Send Messages': 'true',
},
'autoTerminatedRelationships': ['activity.restored', 'success'],
})
canvas.create_connection(monitor_activity, fail_funnel, ['inactive'])
canvas.create_connection(write_kudu, monitor_activity, ['success'])
# Version flow
save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_id=env.sensor_flow.version_control_information.flow_id,
comment='Second version - ' + str(env.run_id))
# Prepare Impala/Kudu table
create_kudu_table()
# Set the variable with the CDSW access key
if env.cdsw_flag:
canvas.update_variable_registry(
env.sensor_pg, [('cdsw.access.key', get_cdsw_model_access_key())])
# Start everything
canvas.schedule_process_group(env.root_pg.id, True)
def step7_rest_and_kudu(env):
# Create controllers
json_reader_with_schema_svc = create_controller(
env.sensor_pg,
'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'hwx-schema-ref-attributes',
'schema-registry': env.sr_svc.id
},
True,
name='JsonTreeReader - With schema identifier')
rest_lookup_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.lookup.RestLookupService', {
'rest-lookup-url': get_cdsw_altus_api() + '/models/call-model',
'rest-lookup-record-reader': env.json_reader_svc.id,
'rest-lookup-record-path': '/response'
}, True)
# Build flow
fail_funnel = create_funnel(env.sensor_pg.id, (1400, 340))
consume_kafka = create_processor(
env.sensor_pg, 'Consume Kafka iot messages',
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_0',
(700, 0), {
'properties': {
'bootstrap.servers': 'edge2ai-1.dim.local:9092',
'topic': 'iot',
'topic_type': 'names',
'record-reader': json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'honor-transactions': 'false',
'group.id': CONSUMER_GROUP_ID,
'auto.offset.reset': 'latest',
'header-name-regex': 'schema.*',
},
})
canvas.create_connection(consume_kafka, fail_funnel, ['parse.failure'])
predict = create_processor(
env.sensor_pg, 'Predict machine health',
'org.apache.nifi.processors.standard.LookupRecord', (700, 200), {
'properties': {
'record-reader': json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'lookup-service': rest_lookup_svc.id,
'result-record-path': '/response',
'routing-strategy': 'route-to-success',
'result-contents': 'insert-entire-record',
'mime.type': "toString('application/json', 'UTF-8')",
'request.body':
"concat('{\"accessKey\":\"', '${cdsw.access.key}', '\",\"request\":{\"feature\":\"', /sensor_0, ', ', /sensor_1, ', ', /sensor_2, ', ', /sensor_3, ', ', /sensor_4, ', ', /sensor_5, ', ', /sensor_6, ', ', /sensor_7, ', ', /sensor_8, ', ', /sensor_9, ', ', /sensor_10, ', ', /sensor_11, '\"}}')",
'request.method': "toString('post', 'UTF-8')",
},
})
canvas.create_connection(predict, fail_funnel, ['failure'])
canvas.create_connection(consume_kafka, predict, ['success'])
update_health = create_processor(
env.sensor_pg, 'Update health flag',
'org.apache.nifi.processors.standard.UpdateRecord', (700, 400), {
'properties': {
'record-reader': json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'replacement-value-strategy': 'record-path-value',
'/is_healthy': '/response/result',
},
})
canvas.create_connection(update_health, fail_funnel, ['failure'])
canvas.create_connection(predict, update_health, ['success'])
write_kudu = create_processor(
env.sensor_pg, 'Write to Kudu',
'org.apache.nifi.processors.kudu.PutKudu', (700, 600), {
'properties': {
'Kudu Masters': 'edge2ai-1.dim.local:7051',
'Table Name': 'impala::default.sensors',
'record-reader': json_reader_with_schema_svc.id,
},
})
canvas.create_connection(write_kudu, fail_funnel, ['failure'])
canvas.create_connection(update_health, write_kudu, ['success'])
monitor_activity = create_processor(
env.sensor_pg, 'Monitor Activity',
'org.apache.nifi.processors.standard.MonitorActivity', (700, 800), {
'properties': {
'Threshold Duration': '45 secs',
'Continually Send Messages': 'true',
},
'autoTerminatedRelationships': ['activity.restored', 'success'],
})
canvas.create_connection(monitor_activity, fail_funnel, ['inactive'])
canvas.create_connection(write_kudu, monitor_activity, ['success'])
# Version flow
#versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='Second version - ' + env.run_id)
save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_id=env.sensor_flow.version_control_information.flow_id,
comment='Second version - ' + env.run_id)
# Prepare Impala/Kudu table
create_kudu_table()
# Set the variable with the CDSW access key
canvas.update_variable_registry(
env.sensor_pg, [('cdsw.access.key', get_cdsw_model_access_key())])
# Start everything
canvas.schedule_process_group(env.root_pg.id, True)
