def test_update_variable_registry(fixture_pg):
test_pg = fixture_pg.generate()
r1 = canvas.update_variable_registry(test_pg,
config.test_variable_registry_entry)
with pytest.raises(ValueError,
match='param update is not a valid list of'):
_ = canvas.update_variable_registry(test_pg, '')
def test_update_variable_registry(fix_pg):
test_pg = fix_pg.generate()
r1 = canvas.update_variable_registry(test_pg,
conftest.test_variable_registry_entry)
assert isinstance(r1, nifi.VariableRegistryEntity)
with pytest.raises(ValueError,
match='param update is not a valid list of'):
_ = canvas.update_variable_registry(test_pg, '')
def test_update_variable_registry(fix_pg):
test_pg = fix_pg.generate()
r1 = canvas.update_variable_registry(test_pg,
conftest.test_variable_registry_entry)
assert isinstance(r1, nifi.VariableRegistryEntity)
with pytest.raises(ValueError, match='not the most up-to-date revision'):
_ = canvas.update_variable_registry(
test_pg, conftest.test_variable_registry_entry, refresh=False)
r2 = canvas.update_variable_registry(test_pg,
conftest.test_variable_registry_entry,
refresh=True)
assert isinstance(r2, nifi.VariableRegistryEntity)
r3 = canvas.update_variable_registry(test_pg, [('key1', 'value1'),
('key2', 'value2')],
refresh=True)
assert isinstance(r3, nifi.VariableRegistryEntity)
with pytest.raises(ValueError,
match='param update is not a valid list of'):
_ = canvas.update_variable_registry(test_pg, '')
def step7_rest_and_kudu(env):
# Create controllers
json_reader_with_schema_svc = create_controller(
env.sensor_pg,
'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'hwx-schema-ref-attributes',
'schema-registry': env.sr_svc.id
},
True,
name='JsonTreeReader - With schema identifier')
rest_lookup_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.lookup.RestLookupService', {
'rest-lookup-url': get_cdsw_altus_api() + '/models/call-model',
'rest-lookup-record-reader': env.json_reader_svc.id,
'rest-lookup-record-path': '/response'
}, True)
# Build flow
fail_funnel = create_funnel(env.sensor_pg.id, (1400, 340))
consume_kafka = create_processor(
env.sensor_pg, 'Consume Kafka iot messages',
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_0',
(700, 0), {
'properties': {
'bootstrap.servers': 'edge2ai-1.dim.local:9092',
'topic': 'iot',
'topic_type': 'names',
'record-reader': json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'honor-transactions': 'false',
'group.id': CONSUMER_GROUP_ID,
'auto.offset.reset': 'latest',
'header-name-regex': 'schema.*',
},
})
canvas.create_connection(consume_kafka, fail_funnel, ['parse.failure'])
predict = create_processor(
env.sensor_pg, 'Predict machine health',
'org.apache.nifi.processors.standard.LookupRecord', (700, 200), {
'properties': {
'record-reader': json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'lookup-service': rest_lookup_svc.id,
'result-record-path': '/response',
'routing-strategy': 'route-to-success',
'result-contents': 'insert-entire-record',
'mime.type': "toString('application/json', 'UTF-8')",
'request.body':
"concat('{\"accessKey\":\"', '${cdsw.access.key}', '\",\"request\":{\"feature\":\"', /sensor_0, ', ', /sensor_1, ', ', /sensor_2, ', ', /sensor_3, ', ', /sensor_4, ', ', /sensor_5, ', ', /sensor_6, ', ', /sensor_7, ', ', /sensor_8, ', ', /sensor_9, ', ', /sensor_10, ', ', /sensor_11, '\"}}')",
'request.method': "toString('post', 'UTF-8')",
},
})
canvas.create_connection(predict, fail_funnel, ['failure'])
canvas.create_connection(consume_kafka, predict, ['success'])
update_health = create_processor(
env.sensor_pg, 'Update health flag',
'org.apache.nifi.processors.standard.UpdateRecord', (700, 400), {
'properties': {
'record-reader': json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'replacement-value-strategy': 'record-path-value',
'/is_healthy': '/response/result',
},
})
canvas.create_connection(update_health, fail_funnel, ['failure'])
canvas.create_connection(predict, update_health, ['success'])
write_kudu = create_processor(
env.sensor_pg, 'Write to Kudu',
'org.apache.nifi.processors.kudu.PutKudu', (700, 600), {
'properties': {
'Kudu Masters': 'edge2ai-1.dim.local:7051',
'Table Name': 'impala::default.sensors',
'record-reader': json_reader_with_schema_svc.id,
},
})
canvas.create_connection(write_kudu, fail_funnel, ['failure'])
canvas.create_connection(update_health, write_kudu, ['success'])
monitor_activity = create_processor(
env.sensor_pg, 'Monitor Activity',
'org.apache.nifi.processors.standard.MonitorActivity', (700, 800), {
'properties': {
'Threshold Duration': '45 secs',
'Continually Send Messages': 'true',
},
'autoTerminatedRelationships': ['activity.restored', 'success'],
})
canvas.create_connection(monitor_activity, fail_funnel, ['inactive'])
canvas.create_connection(write_kudu, monitor_activity, ['success'])
# Version flow
#versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='Second version - ' + env.run_id)
save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_id=env.sensor_flow.version_control_information.flow_id,
comment='Second version - ' + env.run_id)
# Prepare Impala/Kudu table
create_kudu_table()
# Set the variable with the CDSW access key
canvas.update_variable_registry(
env.sensor_pg, [('cdsw.access.key', get_cdsw_model_access_key())])
# Start everything
canvas.schedule_process_group(env.root_pg.id, True)
def lab7_rest_and_kudu(env):
LOG.info("Running step7_rest_and_kudu")
# Create controllers
env.json_reader_with_schema_svc = create_controller(
env.sensor_pg,
'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'hwx-schema-ref-attributes',
'schema-registry': env.sr_svc.id
},
True,
name='JsonTreeReader - With schema identifier')
props = {
'rest-lookup-url': get_cdsw_altus_api() + '/models/call-model',
'rest-lookup-record-reader': env.json_reader_svc.id,
'rest-lookup-record-path': '/response'
}
if _IS_TLS_ENABLED:
props.update({
'rest-lookup-ssl-context-service': env.ssl_svc.id,
})
rest_lookup_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.lookup.RestLookupService', props, True)
# Build flow
fail_funnel = create_funnel(env.sensor_pg.id, (1400, 340))
props = {
'topic': 'iot',
'topic_type': 'names',
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'consumer'))
consume_kafka = create_processor(
env.sensor_pg, 'Consume Kafka iot messages',
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_0',
(700, 0), {'properties': props})
canvas.create_connection(consume_kafka, fail_funnel, ['parse.failure'])
predict = create_processor(
env.sensor_pg, 'Predict machine health',
'org.apache.nifi.processors.standard.LookupRecord', (700, 200), {
'properties': {
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'lookup-service': rest_lookup_svc.id,
'result-record-path': '/response',
'routing-strategy': 'route-to-success',
'result-contents': 'insert-entire-record',
'mime.type': "toString('application/json', 'UTF-8')",
'request.body':
"concat('{\"accessKey\":\"', '${cdsw.access.key}', '\",\"request\":{\"feature\":\"', /sensor_0, ', ', /sensor_1, ', ', /sensor_2, ', ', /sensor_3, ', ', /sensor_4, ', ', /sensor_5, ', ', /sensor_6, ', ', /sensor_7, ', ', /sensor_8, ', ', /sensor_9, ', ', /sensor_10, ', ', /sensor_11, '\"}}')",
'request.method': "toString('post', 'UTF-8')",
},
})
canvas.create_connection(predict, fail_funnel, ['failure'])
canvas.create_connection(consume_kafka, predict, ['success'])
update_health = create_processor(
env.sensor_pg, 'Update health flag',
'org.apache.nifi.processors.standard.UpdateRecord', (700, 400), {
'properties': {
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
'replacement-value-strategy': 'record-path-value',
'/is_healthy': '/response/result',
},
})
canvas.create_connection(update_health, fail_funnel, ['failure'])
canvas.create_connection(predict, update_health, ['success'])
if get_kudu_version() >= (1, 14):
kudu_table_name = 'default.sensors'
else:
kudu_table_name = 'impala::default.sensors'
write_kudu = create_processor(
env.sensor_pg, 'Write to Kudu',
'org.apache.nifi.processors.kudu.PutKudu', (700, 600), {
'properties': {
'Kudu Masters':
_HOSTNAME + ':7051',
'Table Name':
kudu_table_name,
'record-reader':
env.json_reader_with_schema_svc.id,
'kerberos-credentials-service':
env.keytab_svc.id if _IS_TLS_ENABLED else None,
},
})
canvas.create_connection(write_kudu, fail_funnel, ['failure'])
canvas.create_connection(update_health, write_kudu, ['success'])
props = {
'topic': 'iot_enriched',
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.json_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'producer'))
pub_kafka_enriched = create_processor(
env.sensor_pg, 'Publish to Kafka topic: iot_enriched',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0',
(300, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched, ['success'])
props = {
'topic': 'iot_enriched_avro',
'record-reader': env.json_reader_with_schema_svc.id,
'record-writer': env.avro_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'producer'))
pub_kafka_enriched_avro = create_processor(
env.sensor_pg, 'Publish to Kafka topic: iot_enriched_avro',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0',
(-100, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched_avro,
['success'])
monitor_activity = create_processor(
env.sensor_pg, 'Monitor Activity',
'org.apache.nifi.processors.standard.MonitorActivity', (700, 800), {
'properties': {
'Threshold Duration': '45 secs',
'Continually Send Messages': 'true',
},
'autoTerminatedRelationships': ['activity.restored', 'success'],
})
canvas.create_connection(monitor_activity, fail_funnel, ['inactive'])
canvas.create_connection(write_kudu, monitor_activity, ['success'])
# Version flow
save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_id=env.sensor_flow.version_control_information.flow_id,
comment='Second version - ' + str(env.run_id))
# Prepare Impala/Kudu table
create_kudu_table()
# Set the variable with the CDSW access key
if env.cdsw_flag:
canvas.update_variable_registry(
env.sensor_pg, [('cdsw.access.key', get_cdsw_model_access_key())])
# Start everything
canvas.schedule_process_group(env.root_pg.id, True)
def lab4_rest_and_kudu(self):
# Prepare Impala/Kudu table
kudu.create_table()
kudu_table_name = kudu.get_kudu_table_name('default', 'sensors')
# Set required variables
if not self.context.skip_cdsw:
# Set the variable with the CDSW access key
canvas.update_variable_registry(
self.context.sensor_pg,
[('cdsw.access.key', cdsw.get_model_access_key())])
# Set the variable with the CDSW model API key
canvas.update_variable_registry(
self.context.sensor_pg,
[('cdsw.model.api.key', cdsw.create_model_api_key())])
# Create controllers
self.context.json_reader_with_schema_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'hwx-schema-ref-attributes',
'schema-registry': self.context.sr_svc.id
},
True,
name='JsonTreeReader - With schema identifier')
props = {
'rest-lookup-url': cdsw.get_model_endpoint_url(),
'rest-lookup-record-reader': self.context.json_reader_svc.id,
'rest-lookup-record-path': '/response',
'Authorization': 'Bearer ${cdsw.model.api.key}',
}
if is_tls_enabled():
props.update({
'rest-lookup-ssl-context-service':
self.context.ssl_svc.id,
})
rest_lookup_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.lookup.RestLookupService',
props, True)
# Build flow
fail_funnel = nf.create_funnel(self.context.sensor_pg.id, (1400, 340))
props = {
'topic': 'iot',
'topic_type': 'names',
'record-reader': self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.json_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'consumer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
consume_kafka = nf.create_processor(
self.context.sensor_pg, 'Consume Kafka iot messages', [
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_0'
], (700, 0), {'properties': props})
canvas.create_connection(consume_kafka, fail_funnel, ['parse.failure'])
predict = nf.create_processor(
self.context.sensor_pg, 'Predict machine health',
'org.apache.nifi.processors.standard.LookupRecord', (700, 200), {
'properties': {
'record-reader':
self.context.json_reader_with_schema_svc.id,
'record-writer':
self.context.json_writer_svc.id,
'lookup-service':
rest_lookup_svc.id,
'result-record-path':
'/response',
'routing-strategy':
'route-to-success',
'result-contents':
'insert-entire-record',
'mime.type':
"toString('application/json', 'UTF-8')",
'request.body':
"concat('{\"accessKey\":\"', '${cdsw.access.key}', "
"'\",\"request\":{\"feature\":\"', /sensor_0, ', ', "
"/sensor_1, ', ', /sensor_2, ', ', /sensor_3, ', ', "
"/sensor_4, ', ', /sensor_5, ', ', /sensor_6, ', ', "
"/sensor_7, ', ', /sensor_8, ', ', /sensor_9, ', ', "
"/sensor_10, ', ', /sensor_11, '\"}}')",
'request.method':
"toString('post', 'UTF-8')",
},
})
canvas.create_connection(predict, fail_funnel, ['failure'])
canvas.create_connection(consume_kafka, predict, ['success'])
update_health = nf.create_processor(
self.context.sensor_pg, 'Update health flag',
'org.apache.nifi.processors.standard.UpdateRecord', (700, 400), {
'properties': {
'record-reader':
self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.json_writer_svc.id,
'replacement-value-strategy': 'record-path-value',
'/is_healthy': '/response/result',
},
})
canvas.create_connection(update_health, fail_funnel, ['failure'])
canvas.create_connection(predict, update_health, ['success'])
write_kudu = nf.create_processor(
self.context.sensor_pg, 'Write to Kudu',
'org.apache.nifi.processors.kudu.PutKudu', (700, 600), {
'properties': {
'Kudu Masters':
get_hostname() + ':7051',
'Table Name':
kudu_table_name,
'record-reader':
self.context.json_reader_with_schema_svc.id,
'kerberos-credentials-service':
self.context.keytab_svc.id if is_tls_enabled() else None,
},
})
canvas.create_connection(write_kudu, fail_funnel, ['failure'])
canvas.create_connection(update_health, write_kudu, ['success'])
props = {
'topic': 'iot_enriched',
'record-reader': self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.json_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'producer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
pub_kafka_enriched = nf.create_processor(
self.context.sensor_pg, 'Publish to Kafka topic: iot_enriched', [
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0'
], (300, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched,
['success'])
props = {
'topic': 'iot_enriched_avro',
'record-reader': self.context.json_reader_with_schema_svc.id,
'record-writer': self.context.avro_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'producer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
pub_kafka_enriched_avro = nf.create_processor(
self.context.sensor_pg,
'Publish to Kafka topic: iot_enriched_avro', [
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0'
], (-100, 600), {
'properties': props,
'autoTerminatedRelationships': ['success', 'failure'],
})
canvas.create_connection(update_health, pub_kafka_enriched_avro,
['success'])
monitor_activity = nf.create_processor(
self.context.sensor_pg, 'Monitor Activity',
'org.apache.nifi.processors.standard.MonitorActivity', (700, 800),
{
'properties': {
'Threshold Duration': '45 secs',
'Continually Send Messages': 'true',
},
'autoTerminatedRelationships':
['activity.restored', 'success'],
})
canvas.create_connection(monitor_activity, fail_funnel, ['inactive'])
canvas.create_connection(write_kudu, monitor_activity, ['success'])
# Version flow
nifireg.save_flow_ver(self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_id=self.context.sensor_flow.
version_control_information.flow_id,
comment='Second version - {}'.format(
self.run_id))
# Start everything
canvas.schedule_process_group(self.context.root_pg.id, True)