def test_get_registry_bucket(fixture_registry_bucket):
r1 = versioning.get_registry_bucket(config.test_bucket_name)
assert r1.name == config.test_bucket_name
r2 = versioning.get_registry_bucket(r1.identifier, 'id')
assert r2.name == r1.name
with pytest.raises(ValueError):
_ = versioning.get_registry_bucket('Irrelevant', 'Invalid')
r3 = versioning.get_registry_bucket('NonExistantProbably')
assert r3 is None
def test_get_registry_bucket(regress_flow_reg, fix_bucket):
f_bucket = fix_bucket()
r1 = versioning.get_registry_bucket(f_bucket.name)
assert r1.name == conftest.test_bucket_name
r2 = versioning.get_registry_bucket(r1.identifier, 'id')
assert r2.name == r1.name
with pytest.raises(ValueError):
_ = versioning.get_registry_bucket('Irrelevant', 'Invalid')
r3 = versioning.get_registry_bucket('NonExistantProbably')
assert r3 is None
def step2_edge_flow(env):
# Create input port and funnel in NiFi
env.from_gw = canvas.create_port(env.root_pg.id, 'INPUT_PORT',
'from Gateway', 'STOPPED', (0, 200))
funnel_position = (96, 350)
env.temp_funnel = create_funnel(env.root_pg.id, (96, 350))
canvas.create_connection(env.from_gw, env.temp_funnel)
# Create flow in EFM
env.consume_mqtt = efm_create_processor(
env.flow_id, env.efm_pg_id, 'ConsumeMQTT',
'org.apache.nifi.processors.mqtt.ConsumeMQTT', (100, 100), {
'Broker URI': 'tcp://edge2ai-1.dim.local:1883',
'Client ID': 'minifi-iot',
'Topic Filter': 'iot/#',
'Max Queue Size': '60',
})
env.nifi_rpg = efm_create_remote_processor_group(env.flow_id,
env.efm_pg_id,
'Remote PG', _NIFI_URL,
'HTTP', (100, 400))
env.consume_conn = efm_create_connection(env.flow_id,
env.efm_pg_id,
env.consume_mqtt,
'PROCESSOR',
env.nifi_rpg,
'REMOTE_INPUT_PORT', ['Message'],
destination_port=env.from_gw.id)
# Create a bucket in NiFi Registry to save the edge flow versions
if not versioning.get_registry_bucket('IoT'):
versioning.create_registry_bucket('IoT')
# Publish/version the flow
efm_publish_flow(env.flow_id, 'First version - ' + env.run_id)
def nifireg_delete_flows(identifier, identifier_type='name'):
bucket = versioning.get_registry_bucket(identifier, identifier_type)
if bucket:
for flow in versioning.list_flows_in_bucket(bucket.identifier):
endpoint = '/buckets/{bucketId}/flows/{flowId}'.format(
bucketId=flow.bucket_identifier, flowId=flow.identifier)
resp = nifireg_api_delete(
endpoint, headers={'Content-Type': 'application/json'})
def lab4_nifi_flow(env):
LOG.info("Running step4_nifi_flow")
# Create a bucket in NiFi Registry to save the edge flow versions
env.sensor_bucket = versioning.get_registry_bucket('SensorFlows')
if not env.sensor_bucket:
env.sensor_bucket = versioning.create_registry_bucket('SensorFlows')
# Create NiFi Process Group
env.reg_client = versioning.create_registry_client('NiFi Registry', _NIFIREG_URL, 'The registry...')
env.sensor_pg = canvas.create_process_group(env.root_pg, PG_NAME, (330, 350))
#env.sensor_flow = versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_name='SensorProcessGroup', comment='Enabled version control - ' + env.run_id)
env.sensor_flow = save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_name='SensorProcessGroup', comment='Enabled version control - ' + str(env.run_id))
# Create controller services
env.sr_svc = create_controller(env.sensor_pg, 'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry', {'url': _SCHREG_API_URL}, True)
env.json_reader_svc = create_controller(env.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {'schema-access-strategy': 'schema-name', 'schema-registry': env.sr_svc.id}, True)
env.json_writer_svc = create_controller(env.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter', {'schema-access-strategy': 'schema-name', 'schema-registry': env.sr_svc.id, 'Schema Write Strategy': 'hwx-schema-ref-attributes'}, True)
# Create flow
sensor_port = canvas.create_port(env.sensor_pg.id, 'INPUT_PORT', 'Sensor Data', 'RUNNING', (0, 0))
upd_attr = create_processor(env.sensor_pg, 'Set Schema Name', 'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100),
{
'properties': {
'schema.name': 'SensorReading',
},
}
)
canvas.create_connection(sensor_port, upd_attr)
pub_kafka = create_processor(env.sensor_pg, 'Publish to Kafka topic: iot', 'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0', (0, 300),
{
'properties': {
'bootstrap.servers': 'edge2ai-1.dim.local:9092',
'topic': 'iot',
'record-reader': env.json_reader_svc.id,
'record-writer': env.json_writer_svc.id,
'use-transactions': 'false',
'attribute-name-regex': 'schema.*',
'client.id': PRODUCER_CLIENT_ID,
},
'autoTerminatedRelationships': ['success'],
}
)
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = create_funnel(env.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
#versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='First version - ' + env.run_id)
save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='First version - ' + str(env.run_id))
# Start flow
canvas.schedule_process_group(env.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
update_connection(env.from_gw, env.temp_funnel, sensor_port)
def test_nifi_registry_iot(run_id):
bucket = versioning.get_registry_bucket('IoT')
assert bucket
flows = versioning.list_flows_in_bucket(bucket.identifier)
assert len(flows) == 1
flow = flows[0]
versions = versioning.list_flow_versions(bucket.identifier, flow.identifier)
assert len(versions) >= 2
assert versions[0].comments == 'Second version - ' + run_id, 'Comments: ' + versions[0].comments
assert versions[1].comments == 'First version - ' + run_id, 'Comments: ' + versions[1].comments
def test_nifi_registry_sensorflows(run_id):
bucket = versioning.get_registry_bucket('SensorFlows')
assert bucket
flows = versioning.list_flows_in_bucket(bucket.identifier)
assert len(flows) == 1
flow = flows[0]
versions = versioning.list_flow_versions(bucket.identifier, flow.identifier)
assert len(versions) == 3
assert versions[0].comments == 'Second version - ' + run_id, 'Comments: ' + versions[0].comments
assert versions[1].comments == 'First version - ' + run_id, 'Comments: ' + versions[1].comments
assert versions[2].comments == 'Enabled version control - ' + run_id, 'Comments: ' + versions[2].comments
def test_save_flow_ver(regress_flow_reg, fix_bucket, fix_pg, fix_proc):
f_reg_client = conftest.ensure_registry_client(
config.registry_local_name
)
f_bucket = fix_bucket()
f_pg = fix_pg.generate()
test_bucket = versioning.get_registry_bucket(f_bucket.identifier, 'id')
assert test_bucket.name == conftest.test_bucket_name
r1 = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=test_bucket,
flow_name=conftest.test_versioned_flow_name,
comment='a test comment',
desc='a test description'
)
assert isinstance(r1, nifi.VersionControlInformationEntity)
# Next we validate you can't duplicate a flow name in a bucket
with pytest.raises(ValueError):
_ = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=f_bucket,
flow_name=conftest.test_versioned_flow_name,
comment='NiPyApi Test',
desc='NiPyApi Test'
)
# Add a processor, refresh status, and save a new version
fix_proc.generate(parent_pg=f_pg)
f_pg = canvas.get_process_group(f_pg.id, 'id')
r2 = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=f_bucket,
flow_id=r1.version_control_information.flow_id,
comment='a test comment'
)
assert isinstance(r2, nifi.VersionControlInformationEntity)
assert r2.version_control_information.version > \
r1.version_control_information.version
with pytest.raises(ValueError):
_ = versioning.save_flow_ver(
process_group=f_pg,
registry_client=f_reg_client,
bucket=f_bucket,
flow_name=conftest.test_versioned_flow_name,
comment='a test comment',
desc='a test description',
refresh=False
)
# shortcut to clean up the test objects when not using the fixture
conftest.cleanup_reg()
def lab2_edge_flow(self):
# Create input port and funnel in NiFi
self.context.from_gw = canvas.create_port(self.context.root_pg.id,
'INPUT_PORT', 'from Gateway',
'STOPPED', (0, 200))
self.context.temp_funnel = nf.create_funnel(self.context.root_pg.id,
(96, 350))
canvas.create_connection(self.context.from_gw,
self.context.temp_funnel)
canvas.schedule_components(self.context.root_pg.id, True,
[self.context.from_gw])
# Create flow in EFM
self.context.consume_mqtt = efm.create_processor(
self.context.flow_id, self.context.efm_pg_id, 'ConsumeMQTT',
'org.apache.nifi.processors.mqtt.ConsumeMQTT', (100, 100), {
'Broker URI':
'tcp://{hostname}:1883'.format(hostname=get_hostname()),
'Client ID':
'minifi-iot',
'Topic Filter':
'iot/#',
'Max Queue Size':
'60',
})
self.context.nifi_rpg = efm.create_remote_processor_group(
self.context.flow_id, self.context.efm_pg_id, 'Remote PG',
nf.get_url(), 'HTTP', (100, 400))
self.context.consume_conn = efm.create_connection(
self.context.flow_id,
self.context.efm_pg_id,
self.context.consume_mqtt,
'PROCESSOR',
self.context.nifi_rpg,
'REMOTE_INPUT_PORT', ['Message'],
destination_port=self.context.from_gw.id,
name='Sensor data',
flow_file_expiration='60 seconds')
# Create a bucket in NiFi Registry to save the edge flow versions
if not versioning.get_registry_bucket('IoT'):
versioning.create_registry_bucket('IoT')
# Publish/version the flow
efm.publish_flow(self.context.flow_id,
'First version - {}'.format(self.run_id))
def test_delete_registry_bucket(fixture_registry_bucket):
b1, c1 = fixture_registry_bucket
r = versioning.delete_registry_bucket(b1)
assert r.identifier == b1.identifier
with pytest.raises(ValueError):
_ = versioning.get_registry_bucket(b1.identifier, 'id')
def delete_flows(identifier, identifier_type='name'):
bucket = versioning.get_registry_bucket(identifier, identifier_type)
if bucket:
for flow in versioning.list_flows_in_bucket(bucket.identifier):
BucketFlowsApi().delete_flow(flow.bucket_identifier,
flow.identifier)
def lab4_nifi_flow(env):
LOG.info("Running step4_nifi_flow")
# Create a bucket in NiFi Registry to save the edge flow versions
env.sensor_bucket = versioning.get_registry_bucket('SensorFlows')
if not env.sensor_bucket:
env.sensor_bucket = versioning.create_registry_bucket('SensorFlows')
# Create NiFi Process Group
env.reg_client = versioning.create_registry_client('NiFi Registry',
_get_nifireg_url(),
'The registry...')
env.sensor_pg = canvas.create_process_group(env.root_pg, PG_NAME,
(330, 350))
#env.sensor_flow = versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_name='SensorProcessGroup', comment='Enabled version control - ' + env.run_id)
env.sensor_flow = save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_name='SensorProcessGroup',
comment='Enabled version control - ' +
str(env.run_id))
# Update default SSL context controller service
ssl_svc_name = 'Default NiFi SSL Context Service'
if _IS_TLS_ENABLED:
props = {
'SSL Protocol': 'TLS',
'Truststore Type': 'JKS',
'Truststore Filename': '/opt/cloudera/security/jks/truststore.jks',
'Truststore Password': _THE_PWD,
'Keystore Type': 'JKS',
'Keystore Filename': '/opt/cloudera/security/jks/keystore.jks',
'Keystore Password': _THE_PWD,
'key-password': _THE_PWD,
}
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
if env.ssl_svc:
canvas.schedule_controller(env.ssl_svc, False)
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
canvas.update_controller(
env.ssl_svc, nifi.ControllerServiceDTO(properties=props))
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
canvas.schedule_controller(env.ssl_svc, True)
else:
env.keytab_svc = create_controller(
env.root_pg,
'org.apache.nifi.ssl.StandardRestrictedSSLContextService',
props,
True,
name=ssl_svc_name)
# Create controller services
if _IS_TLS_ENABLED:
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
props = {
'Kerberos Keytab': '/keytabs/admin.keytab',
'Kerberos Principal': 'admin',
}
env.keytab_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.kerberos.KeytabCredentialsService',
props, True)
else:
env.ssl_svc = None
env.keytab_svc = None
props = {
'url': _get_schreg_api_url(),
}
if _IS_TLS_ENABLED:
props.update({
'kerberos-credentials-service': env.keytab_svc.id,
'ssl-context-service': env.ssl_svc.id,
})
env.sr_svc = create_controller(
env.sensor_pg,
'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry',
props, True)
env.json_reader_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id
}, True)
env.json_writer_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id,
'Schema Write Strategy': 'hwx-schema-ref-attributes'
}, True)
env.avro_writer_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.avro.AvroRecordSetWriter', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id,
'Schema Write Strategy': 'hwx-content-encoded-schema'
}, True)
# Create flow
sensor_port = canvas.create_port(env.sensor_pg.id, 'INPUT_PORT',
'Sensor Data', 'RUNNING', (0, 0))
upd_attr = create_processor(
env.sensor_pg, 'Set Schema Name',
'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100), {
'properties': {
'schema.name': 'SensorReading',
},
})
canvas.create_connection(sensor_port, upd_attr)
props = {
'topic': 'iot',
'record-reader': env.json_reader_svc.id,
'record-writer': env.json_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'producer'))
pub_kafka = create_processor(
env.sensor_pg, 'Publish to Kafka topic: iot',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0',
(0, 300), {
'properties': props,
'autoTerminatedRelationships': ['success'],
})
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = create_funnel(env.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
#versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='First version - ' + env.run_id)
save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_id=env.sensor_flow.version_control_information.flow_id,
comment='First version - ' + str(env.run_id))
# Start flow
canvas.schedule_process_group(env.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
update_connection(env.from_gw, env.temp_funnel, sensor_port)
def lab2_nifi_flow(self):
# Create a bucket in NiFi Registry to save the edge flow versions
self.context.sensor_bucket = versioning.get_registry_bucket(
'SensorFlows')
if not self.context.sensor_bucket:
self.context.sensor_bucket = versioning.create_registry_bucket(
'SensorFlows')
# Create NiFi Process Group
self.context.reg_client = versioning.create_registry_client(
'NiFi Registry', nifireg.get_url(), 'The registry...')
self.context.sensor_pg = canvas.create_process_group(
self.context.root_pg, PG_NAME, (330, 350))
self.context.sensor_flow = nifireg.save_flow_ver(
self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_name='SensorProcessGroup',
comment='Enabled version control - {}'.format(self.run_id))
# Update default SSL context controller service
ssl_svc_name = 'Default NiFi SSL Context Service'
if is_tls_enabled():
props = {
'SSL Protocol': 'TLS',
'Truststore Type': 'JKS',
'Truststore Filename':
'/opt/cloudera/security/jks/truststore.jks',
'Truststore Password': get_the_pwd(),
'Keystore Type': 'JKS',
'Keystore Filename': '/opt/cloudera/security/jks/keystore.jks',
'Keystore Password': get_the_pwd(),
'key-password': get_the_pwd(),
}
self.context.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
if self.context.ssl_svc:
canvas.schedule_controller(self.context.ssl_svc, False)
self.context.ssl_svc = canvas.get_controller(
ssl_svc_name, 'name')
canvas.update_controller(
self.context.ssl_svc,
nifi.ControllerServiceDTO(properties=props))
self.context.ssl_svc = canvas.get_controller(
ssl_svc_name, 'name')
canvas.schedule_controller(self.context.ssl_svc, True)
else:
self.context.keytab_svc = nf.create_controller(
self.context.root_pg,
'org.apache.nifi.ssl.StandardRestrictedSSLContextService',
props,
True,
name=ssl_svc_name)
# Create controller services
if is_tls_enabled():
self.context.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
props = {
'Kerberos Keytab': '/keytabs/admin.keytab',
'Kerberos Principal': 'admin',
}
self.context.keytab_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.kerberos.KeytabCredentialsService', props,
True)
else:
self.context.ssl_svc = None
self.context.keytab_svc = None
props = {
'url': schreg.get_api_url(),
}
if is_tls_enabled():
props.update({
'kerberos-credentials-service':
self.context.keytab_svc.id,
'ssl-context-service':
self.context.ssl_svc.id,
})
self.context.sr_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry',
props, True)
self.context.json_reader_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id
}, True)
self.context.json_writer_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter',
{
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id,
'Schema Write Strategy': 'hwx-schema-ref-attributes'
}, True)
self.context.avro_writer_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.avro.AvroRecordSetWriter',
{
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id,
'Schema Write Strategy': 'hwx-content-encoded-schema'
}, True)
# Create flow
sensor_port = canvas.create_port(self.context.sensor_pg.id,
'INPUT_PORT', 'Sensor Data',
'STOPPED', (0, 0))
upd_attr = nf.create_processor(
self.context.sensor_pg, 'Set Schema Name',
'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100),
{
'properties': {
'schema.name': 'SensorReading',
},
})
canvas.create_connection(sensor_port, upd_attr)
props = {
'topic': 'iot',
'record-reader': self.context.json_reader_svc.id,
'record-writer': self.context.json_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'producer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
pub_kafka = nf.create_processor(
self.context.sensor_pg, 'Publish to Kafka topic: iot', [
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0'
], (0, 300), {
'properties': props,
'autoTerminatedRelationships': ['success'],
})
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = nf.create_funnel(self.context.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
nifireg.save_flow_ver(self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_id=self.context.sensor_flow.
version_control_information.flow_id,
comment='First version - {}'.format(self.run_id))
# Start flow
canvas.schedule_process_group(self.context.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
nf.update_connection(self.context.from_gw, self.context.temp_funnel,
sensor_port)
canvas.schedule_components(self.context.root_pg.id, True,
[sensor_port])
dev_nifi_port = 8080
dev_reg_port = 18080
dev_nifi_url = 'http://edge2ai-1.dim.local:' + str(dev_nifi_port) + '/nifi'
dev_reg_url = 'http://edge2ai-1.dim.local:' + str(
dev_reg_port) + '/nifi-registry'
dev_nifi_api_url = dev_nifi_url + '-api'
dev_reg_api_url = dev_reg_url + '-api'
# edge2ai-1.dim.local
config.nifi_config.host = 'http://edge2ai-1.dim.local:8080/nifi-api'
config.registry_config.host = 'http://edge2ai-1.dim.local:18080/nifi-registry-api'
nipyapi.config.nifi_config.host = 'http://edge2ai-1.dim.local:8080/nifi-api'
config.nifi_config.host = 'http://edge2ai-1.dim.local:8080/nifi-api'
# Get Registry Bucket
sensor_bucket = versioning.get_registry_bucket('SensorFlows')
print("=====")
print("bucket %s", sensor_bucket)
print("=====")
if not sensor_bucket:
sensor_bucket = versioning.create_registry_bucket('SensorFlows')
# if you change it, back it up for migration
# Backup a Flow
#dlx = versioning.get_flow_version(sensor_bucket.identifier, 'e6d35be3-3073-405e-9c0d-46c8f980bf10', version=None, export=True)
# Export Flow Version
#xprt = versioning.export_flow_version(sensor_bucket.identifier, 'e6d35be3-3073-405e-9c0d-46c8f980bf10', version=None, file_path='/opt/demo/ApacheConAtHome2020/flows/ApacheConDemos.json', mode='json')
#print(dlx)
#print(xprt)
