def teardown(self):
root_pg, _, flow_id = nf.set_environment()
canvas.schedule_process_group(root_pg.id, False)
while True:
failed = False
for controller in canvas.list_all_controllers(root_pg.id):
try:
canvas.schedule_controller(controller, False)
LOG.debug('Controller %s stopped.',
controller.component.name)
except ApiException as exc:
if exc.status == 409 and 'is referenced by' in exc.body:
LOG.debug(
'Controller %s failed to stop. Will retry later.',
controller.component.name)
failed = True
if not failed:
break
nf.delete_all(root_pg)
efm.delete_all(flow_id)
schreg.delete_all_schemas()
reg_client = versioning.get_registry_client('NiFi Registry')
if reg_client:
versioning.delete_registry_client(reg_client)
nifireg.delete_flows('SensorFlows')
kudu.drop_table()
def create_controller(pg, controller_type, properties, start, name=None):
controller_type = get_controller_type(controller_type)
controller = canvas.create_controller(pg, controller_type, name)
controller = canvas.get_controller(controller.id, 'id')
canvas.update_controller(controller, nifi.ControllerServiceDTO(properties=properties))
controller = canvas.get_controller(controller.id, 'id')
canvas.schedule_controller(controller, start)
return canvas.get_controller(controller.id, 'id')
def test_schedule_controller(regress_nifi, fix_pg, fix_cont):
f_pg = fix_pg.generate()
f_c1 = fix_cont(parent_pg=f_pg)
f_c1 = canvas.update_controller(
f_c1, nifi.ControllerServiceDTO(properties={'Server Hostname': 'Bob'}))
with pytest.raises(AssertionError):
_ = canvas.schedule_controller('pie', False)
with pytest.raises(AssertionError):
_ = canvas.schedule_controller(f_c1, 'pie')
r1 = canvas.schedule_controller(f_c1, True)
assert r1.component.state == 'ENABLED'
r2 = canvas.schedule_controller(r1, False)
assert r2.component.state == 'DISABLED'
def test_delete_controller(regress_nifi, fix_pg, fix_cont):
f_pg = fix_pg.generate()
f_c1 = fix_cont(parent_pg=f_pg)
r1 = canvas.delete_controller(f_c1)
assert r1.revision is None
f_c2 = fix_cont(parent_pg=f_pg)
f_c2 = canvas.update_controller(
f_c2, nifi.ControllerServiceDTO(properties={'Server Hostname': 'Bob'}))
f_c2 = canvas.schedule_controller(f_c2, True)
with pytest.raises(AssertionError):
_ = canvas.delete_controller('pie')
with pytest.raises(AssertionError):
_ = canvas.delete_controller(f_c2, 'pie')
with pytest.raises(ValueError):
_ = canvas.delete_controller(f_c2)
assert f_c2.revision is not None
r2 = canvas.delete_controller(f_c2, True)
assert r2.revision is None
# Test for only delete within a PG
f_c_root = fix_cont()
f_c_pg = fix_cont(parent_pg=f_pg)
r3 = canvas.delete_process_group(f_pg)
assert r3.revision is None
r4 = canvas.get_controller(identifier=f_c_root.id, identifier_type='id')
assert r4.revision is not None
def global_teardown(run_id=None,
schema_text=None,
cdsw_flag=True,
target_lab=99):
(run_id, root_pg, efm_pg_id, flow_id) = set_environment(run_id)
canvas.schedule_process_group(root_pg.id, False)
while True:
failed = False
for controller in canvas.list_all_controllers(root_pg.id):
try:
canvas.schedule_controller(controller, False)
LOG.debug('Controller %s stopped.', controller.component.name)
except ApiException as exc:
if exc.status == 409 and 'is referenced by' in exc.body:
LOG.debug(
'Controller %s failed to stop. Will retry later.',
controller.component.name)
failed = True
if not failed:
break
nifi_delete_all(root_pg)
class _Env(object):
pass
env = _Env()
env.run_id, env.root_pg, env.efm_pg_id, env.flow_id = set_environment(
run_id)
env.schema_text = schema_text if schema_text is not None else read_in_schema(
)
LOG.info("Using Schema: %s", schema_text)
env.cdsw_flag = cdsw_flag
LOG.info("Build just enough")
lab2_edge_flow(env)
lab3_register_schema(env)
lab6_expand_edge_flow(env)
create_kudu_table()
LOG.info("Created edge, schema, kudu")
def test_delete_controller(regress_nifi, fix_pg, fix_cont):
f_pg = fix_pg.generate()
f_c1 = fix_cont(parent_pg=f_pg)
r1 = canvas.delete_controller(f_c1)
assert r1.revision is None
f_c2 = fix_cont(parent_pg=f_pg)
f_c2 = canvas.update_controller(
f_c2, nifi.ControllerServiceDTO(properties={'Server Hostname': 'Bob'}))
f_c2 = canvas.schedule_controller(f_c2, True)
with pytest.raises(AssertionError):
_ = canvas.delete_controller('pie')
with pytest.raises(AssertionError):
_ = canvas.delete_controller(f_c2, 'pie')
with pytest.raises(ValueError):
_ = canvas.delete_controller(f_c2)
assert f_c2.revision is not None
r2 = canvas.delete_controller(f_c2, True)
assert r2.revision is None
root_pg = canvas.get_process_group(root_pg_id, 'id')
location_x = 2000
location_y = 2000
location = (location_x, location_y)
template_entity = nipyapi.templates.upload_template(root_pg_id,
'jangowave_demo.xml')
flow = nipyapi.templates.deploy_template(root_pg_id, template_entity.id, 2000,
2000)
jd = canvas.get_process_group('jangowave_demo')
for cs in canvas.list_all_processors(jd.id):
if cs.status.run_status != "ENABLED" and cs.component.name == "RecordIngest":
config_update = nifi.models.processor_config_dto.ProcessorConfigDTO(
properties={
"Accumulo User": "******",
"Instance Name": instance,
"Accumulo Password": "******",
"ZooKeeper Quorum": zookeeper_list
})
canvas.update_processor(cs, config_update)
for cs in canvas.list_all_controllers(jd.id):
canvas.schedule_controller(cs, True)
for cs in canvas.list_all_processors(jd.id):
canvas.schedule_processor(cs, True)
canvas.schedule_process_group(jd.id, True)
def lab4_nifi_flow(env):
LOG.info("Running step4_nifi_flow")
# Create a bucket in NiFi Registry to save the edge flow versions
env.sensor_bucket = versioning.get_registry_bucket('SensorFlows')
if not env.sensor_bucket:
env.sensor_bucket = versioning.create_registry_bucket('SensorFlows')
# Create NiFi Process Group
env.reg_client = versioning.create_registry_client('NiFi Registry',
_get_nifireg_url(),
'The registry...')
env.sensor_pg = canvas.create_process_group(env.root_pg, PG_NAME,
(330, 350))
#env.sensor_flow = versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_name='SensorProcessGroup', comment='Enabled version control - ' + env.run_id)
env.sensor_flow = save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_name='SensorProcessGroup',
comment='Enabled version control - ' +
str(env.run_id))
# Update default SSL context controller service
ssl_svc_name = 'Default NiFi SSL Context Service'
if _IS_TLS_ENABLED:
props = {
'SSL Protocol': 'TLS',
'Truststore Type': 'JKS',
'Truststore Filename': '/opt/cloudera/security/jks/truststore.jks',
'Truststore Password': _THE_PWD,
'Keystore Type': 'JKS',
'Keystore Filename': '/opt/cloudera/security/jks/keystore.jks',
'Keystore Password': _THE_PWD,
'key-password': _THE_PWD,
}
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
if env.ssl_svc:
canvas.schedule_controller(env.ssl_svc, False)
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
canvas.update_controller(
env.ssl_svc, nifi.ControllerServiceDTO(properties=props))
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
canvas.schedule_controller(env.ssl_svc, True)
else:
env.keytab_svc = create_controller(
env.root_pg,
'org.apache.nifi.ssl.StandardRestrictedSSLContextService',
props,
True,
name=ssl_svc_name)
# Create controller services
if _IS_TLS_ENABLED:
env.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
props = {
'Kerberos Keytab': '/keytabs/admin.keytab',
'Kerberos Principal': 'admin',
}
env.keytab_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.kerberos.KeytabCredentialsService',
props, True)
else:
env.ssl_svc = None
env.keytab_svc = None
props = {
'url': _get_schreg_api_url(),
}
if _IS_TLS_ENABLED:
props.update({
'kerberos-credentials-service': env.keytab_svc.id,
'ssl-context-service': env.ssl_svc.id,
})
env.sr_svc = create_controller(
env.sensor_pg,
'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry',
props, True)
env.json_reader_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id
}, True)
env.json_writer_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id,
'Schema Write Strategy': 'hwx-schema-ref-attributes'
}, True)
env.avro_writer_svc = create_controller(
env.sensor_pg, 'org.apache.nifi.avro.AvroRecordSetWriter', {
'schema-access-strategy': 'schema-name',
'schema-registry': env.sr_svc.id,
'Schema Write Strategy': 'hwx-content-encoded-schema'
}, True)
# Create flow
sensor_port = canvas.create_port(env.sensor_pg.id, 'INPUT_PORT',
'Sensor Data', 'RUNNING', (0, 0))
upd_attr = create_processor(
env.sensor_pg, 'Set Schema Name',
'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100), {
'properties': {
'schema.name': 'SensorReading',
},
})
canvas.create_connection(sensor_port, upd_attr)
props = {
'topic': 'iot',
'record-reader': env.json_reader_svc.id,
'record-writer': env.json_writer_svc.id,
}
props.update(_get_common_kafka_client_properties(env, 'producer'))
pub_kafka = create_processor(
env.sensor_pg, 'Publish to Kafka topic: iot',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0',
(0, 300), {
'properties': props,
'autoTerminatedRelationships': ['success'],
})
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = create_funnel(env.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
#versioning.save_flow_ver(env.sensor_pg, env.reg_client, env.sensor_bucket, flow_id=env.sensor_flow.version_control_information.flow_id, comment='First version - ' + env.run_id)
save_flow_ver(env.sensor_pg,
env.reg_client,
env.sensor_bucket,
flow_id=env.sensor_flow.version_control_information.flow_id,
comment='First version - ' + str(env.run_id))
# Start flow
canvas.schedule_process_group(env.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
update_connection(env.from_gw, env.temp_funnel, sensor_port)
def lab2_nifi_flow(self):
# Create a bucket in NiFi Registry to save the edge flow versions
self.context.sensor_bucket = versioning.get_registry_bucket(
'SensorFlows')
if not self.context.sensor_bucket:
self.context.sensor_bucket = versioning.create_registry_bucket(
'SensorFlows')
# Create NiFi Process Group
self.context.reg_client = versioning.create_registry_client(
'NiFi Registry', nifireg.get_url(), 'The registry...')
self.context.sensor_pg = canvas.create_process_group(
self.context.root_pg, PG_NAME, (330, 350))
self.context.sensor_flow = nifireg.save_flow_ver(
self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_name='SensorProcessGroup',
comment='Enabled version control - {}'.format(self.run_id))
# Update default SSL context controller service
ssl_svc_name = 'Default NiFi SSL Context Service'
if is_tls_enabled():
props = {
'SSL Protocol': 'TLS',
'Truststore Type': 'JKS',
'Truststore Filename':
'/opt/cloudera/security/jks/truststore.jks',
'Truststore Password': get_the_pwd(),
'Keystore Type': 'JKS',
'Keystore Filename': '/opt/cloudera/security/jks/keystore.jks',
'Keystore Password': get_the_pwd(),
'key-password': get_the_pwd(),
}
self.context.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
if self.context.ssl_svc:
canvas.schedule_controller(self.context.ssl_svc, False)
self.context.ssl_svc = canvas.get_controller(
ssl_svc_name, 'name')
canvas.update_controller(
self.context.ssl_svc,
nifi.ControllerServiceDTO(properties=props))
self.context.ssl_svc = canvas.get_controller(
ssl_svc_name, 'name')
canvas.schedule_controller(self.context.ssl_svc, True)
else:
self.context.keytab_svc = nf.create_controller(
self.context.root_pg,
'org.apache.nifi.ssl.StandardRestrictedSSLContextService',
props,
True,
name=ssl_svc_name)
# Create controller services
if is_tls_enabled():
self.context.ssl_svc = canvas.get_controller(ssl_svc_name, 'name')
props = {
'Kerberos Keytab': '/keytabs/admin.keytab',
'Kerberos Principal': 'admin',
}
self.context.keytab_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.kerberos.KeytabCredentialsService', props,
True)
else:
self.context.ssl_svc = None
self.context.keytab_svc = None
props = {
'url': schreg.get_api_url(),
}
if is_tls_enabled():
props.update({
'kerberos-credentials-service':
self.context.keytab_svc.id,
'ssl-context-service':
self.context.ssl_svc.id,
})
self.context.sr_svc = nf.create_controller(
self.context.sensor_pg,
'org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry',
props, True)
self.context.json_reader_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.json.JsonTreeReader', {
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id
}, True)
self.context.json_writer_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.json.JsonRecordSetWriter',
{
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id,
'Schema Write Strategy': 'hwx-schema-ref-attributes'
}, True)
self.context.avro_writer_svc = nf.create_controller(
self.context.sensor_pg, 'org.apache.nifi.avro.AvroRecordSetWriter',
{
'schema-access-strategy': 'schema-name',
'schema-registry': self.context.sr_svc.id,
'Schema Write Strategy': 'hwx-content-encoded-schema'
}, True)
# Create flow
sensor_port = canvas.create_port(self.context.sensor_pg.id,
'INPUT_PORT', 'Sensor Data',
'STOPPED', (0, 0))
upd_attr = nf.create_processor(
self.context.sensor_pg, 'Set Schema Name',
'org.apache.nifi.processors.attributes.UpdateAttribute', (0, 100),
{
'properties': {
'schema.name': 'SensorReading',
},
})
canvas.create_connection(sensor_port, upd_attr)
props = {
'topic': 'iot',
'record-reader': self.context.json_reader_svc.id,
'record-writer': self.context.json_writer_svc.id,
}
props.update(
kafka.get_common_client_properties(self.context, 'producer',
CONSUMER_GROUP_ID,
PRODUCER_CLIENT_ID))
pub_kafka = nf.create_processor(
self.context.sensor_pg, 'Publish to Kafka topic: iot', [
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6',
'org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_0'
], (0, 300), {
'properties': props,
'autoTerminatedRelationships': ['success'],
})
canvas.create_connection(upd_attr, pub_kafka, ['success'])
fail_funnel = nf.create_funnel(self.context.sensor_pg.id, (600, 343))
canvas.create_connection(pub_kafka, fail_funnel, ['failure'])
# Commit changes
nifireg.save_flow_ver(self.context.sensor_pg,
self.context.reg_client,
self.context.sensor_bucket,
flow_id=self.context.sensor_flow.
version_control_information.flow_id,
comment='First version - {}'.format(self.run_id))
# Start flow
canvas.schedule_process_group(self.context.root_pg.id, True)
# Update "from Gateway" input port to connect to the process group
nf.update_connection(self.context.from_gw, self.context.temp_funnel,
sensor_port)
canvas.schedule_components(self.context.root_pg.id, True,
[sensor_port])
