def get_zookeeper_configuration(params):
zookeeper = params['zookeeper']
zookeeper_auth_scheme = params['zookeeper_auth_scheme']
zookeeper_auth_value = params['zookeeper_auth_value']
zookeeper_ssl_check_hostname = params['zookeeper_ssl_check_hostname']
zookeeper_ssl_cafile = params['zookeeper_ssl_cafile']
zookeeper_ssl_certfile = params['zookeeper_ssl_certfile']
zookeeper_ssl_keyfile = params['zookeeper_ssl_keyfile']
zookeeper_ssl_password = params['zookeeper_ssl_password']
zookeeper_ssl_files = generate_ssl_object(zookeeper_ssl_cafile,
zookeeper_ssl_certfile,
zookeeper_ssl_keyfile)
zookeeper_use_ssl = bool(
zookeeper_ssl_files['keyfile']['path'] is not None
and zookeeper_ssl_files['certfile']['path'] is not None)
zookeeper_auth = []
if zookeeper_auth_value != '':
auth = (zookeeper_auth_scheme, zookeeper_auth_value)
zookeeper_auth.append(auth)
return dict(hosts=zookeeper,
auth_data=zookeeper_auth,
keyfile=zookeeper_ssl_files['keyfile']['path'],
use_ssl=zookeeper_use_ssl,
keyfile_password=zookeeper_ssl_password,
certfile=zookeeper_ssl_files['certfile']['path'],
ca=zookeeper_ssl_files['cafile']['path'],
verify_certs=zookeeper_ssl_check_hostname)
def get_manager_from_params(params):
bootstrap_servers = params['bootstrap_servers']
security_protocol = params['security_protocol']
ssl_check_hostname = params['ssl_check_hostname']
ssl_cafile = params['ssl_cafile']
ssl_certfile = params['ssl_certfile']
ssl_keyfile = params['ssl_keyfile']
ssl_password = params['ssl_password']
ssl_crlfile = params['ssl_crlfile']
ssl_supported_protocols = params['ssl_supported_protocols']
ssl_ciphers = params['ssl_ciphers']
sasl_mechanism = params['sasl_mechanism']
sasl_plain_username = params['sasl_plain_username']
sasl_plain_password = params['sasl_plain_password']
sasl_kerberos_service_name = params['sasl_kerberos_service_name']
api_version = tuple(
int(p) for p in params['api_version'].strip(".").split("."))
kafka_ssl_files = generate_ssl_object(ssl_cafile, ssl_certfile,
ssl_keyfile, ssl_crlfile)
# Generate ssl context to support limit ssl protocols & ciphers
ssl_context = None
if security_protocol in ('SSL', 'SASL_SSL'):
ssl_context = generate_ssl_context(
ssl_check_hostname=ssl_check_hostname,
ssl_cafile=kafka_ssl_files['cafile']['path'],
ssl_certfile=kafka_ssl_files['certfile']['path'],
ssl_keyfile=kafka_ssl_files['keyfile']['path'],
ssl_password=ssl_password,
ssl_crlfile=kafka_ssl_files['crlfile']['path'],
ssl_supported_protocols=ssl_supported_protocols,
ssl_ciphers=ssl_ciphers)
manager = KafkaManager(
bootstrap_servers=bootstrap_servers,
security_protocol=security_protocol,
api_version=api_version,
ssl_context=ssl_context,
sasl_mechanism=sasl_mechanism,
sasl_plain_username=sasl_plain_username,
sasl_plain_password=sasl_plain_password,
sasl_kerberos_service_name=sasl_kerberos_service_name)
if parse_version(manager.get_api_version()) < parse_version('0.11.0'):
raise IncompatibleVersion(
'Current version of library is not compatible with '
'Kafka < 0.11.0.')
return manager
def maybe_clean_zk_ssl_files(params):
zookeeper_ssl_cafile = params['zookeeper_ssl_cafile']
zookeeper_ssl_certfile = params['zookeeper_ssl_certfile']
zookeeper_ssl_keyfile = params['zookeeper_ssl_keyfile']
zookeeper_ssl_files = generate_ssl_object(zookeeper_ssl_cafile,
zookeeper_ssl_certfile,
zookeeper_ssl_keyfile)
for _key, value in zookeeper_ssl_files.items():
if (value['path'] is not None and value['is_temp']
and os.path.exists(os.path.dirname(value['path']))):
os.remove(value['path'])
def maybe_clean_kafka_ssl_files(params):
ssl_cafile = params['ssl_cafile']
ssl_certfile = params['ssl_certfile']
ssl_keyfile = params['ssl_keyfile']
ssl_crlfile = params['ssl_crlfile']
kafka_ssl_files = generate_ssl_object(ssl_cafile, ssl_certfile,
ssl_keyfile, ssl_crlfile)
for _key, value in kafka_ssl_files.items():
if (value['path'] is not None and value['is_temp']
and os.path.exists(os.path.dirname(value['path']))):
os.remove(value['path'])
def main():
module = AnsibleModule(argument_spec=dict(
consummer_group=dict(type='str', required=True),
ignore_empty_partition=dict(type='bool', default=False),
bootstrap_servers=dict(type='str', required=True),
security_protocol=dict(
choices=['PLAINTEXT', 'SSL', 'SASL_SSL', 'SASL_PLAINTEXT'],
default='PLAINTEXT'),
api_version=dict(type='str', required=True, default=None),
ssl_check_hostname=dict(default=True, type='bool', required=False),
ssl_cafile=dict(required=False, default=None, type='path'),
ssl_certfile=dict(required=False, default=None, type='path'),
ssl_keyfile=dict(
required=False, default=None, no_log=True, type='path'),
ssl_password=dict(type='str', no_log=True, required=False),
ssl_crlfile=dict(required=False, default=None, type='path'),
ssl_supported_protocols=dict(required=False,
default=None,
type='list',
choices=['TLSv1', 'TLSv1.1', 'TLSv1.2']),
ssl_ciphers=dict(required=False, default=None, type='str'),
# only PLAIN is currently available
sasl_mechanism=dict(choices=['PLAIN', 'GSSAPI'], default='PLAIN'),
sasl_plain_username=dict(type='str', required=False),
sasl_plain_password=dict(type='str', no_log=True, required=False),
sasl_kerberos_service_name=dict(type='str', required=False),
))
params = module.params
consummer_group = params['consummer_group']
ignore_empty_partition = params['ignore_empty_partition']
bootstrap_servers = params['bootstrap_servers']
security_protocol = params['security_protocol']
ssl_check_hostname = params['ssl_check_hostname']
ssl_cafile = params['ssl_cafile']
ssl_certfile = params['ssl_certfile']
ssl_keyfile = params['ssl_keyfile']
ssl_password = params['ssl_password']
ssl_crlfile = params['ssl_crlfile']
ssl_supported_protocols = params['ssl_supported_protocols']
ssl_ciphers = params['ssl_ciphers']
sasl_mechanism = params['sasl_mechanism']
sasl_plain_username = params['sasl_plain_username']
sasl_plain_password = params['sasl_plain_password']
sasl_kerberos_service_name = params['sasl_kerberos_service_name']
api_version = tuple(
int(p) for p in params['api_version'].strip(".").split("."))
kafka_ssl_files = generate_ssl_object(module, ssl_cafile, ssl_certfile,
ssl_keyfile, ssl_crlfile)
try:
# Generate ssl context to support limit ssl protocols & ciphers
ssl_context = None
if security_protocol in ('SSL', 'SASL_SSL'):
ssl_context = generate_ssl_context(
ssl_check_hostname=ssl_check_hostname,
ssl_cafile=kafka_ssl_files['cafile']['path'],
ssl_certfile=kafka_ssl_files['certfile']['path'],
ssl_keyfile=kafka_ssl_files['keyfile']['path'],
ssl_password=ssl_password,
ssl_crlfile=kafka_ssl_files['crlfile']['path'],
ssl_supported_protocols=ssl_supported_protocols,
ssl_ciphers=ssl_ciphers)
manager = KafkaManager(
module=module,
bootstrap_servers=bootstrap_servers,
security_protocol=security_protocol,
api_version=api_version,
ssl_context=ssl_context,
sasl_mechanism=sasl_mechanism,
sasl_plain_username=sasl_plain_username,
sasl_plain_password=sasl_plain_password,
sasl_kerberos_service_name=sasl_kerberos_service_name)
klag = KafkaConsumerLag(manager.client)
if parse_version(manager.get_api_version()) < parse_version('0.11.0'):
module.fail_json(
msg='Current version of library is not compatible with '
'Kafka < 0.11.0.')
results = klag.get_lag_stats(consummer_group, ignore_empty_partition)
except Exception:
e = get_exception()
module.fail_json(msg='Error while initializing Kafka client : %s ' %
str(e))
finally:
manager.close()
for _key, value in kafka_ssl_files.items():
if (value['path'] is not None and value['is_temp']
and os.path.exists(os.path.dirname(value['path']))):
os.remove(value['path'])
module.exit_json(changed=True, msg=json.dumps(results))
def main():
"""
Module usage
"""
module = AnsibleModule(
argument_spec=dict(
# resource managed, more to come (acl,broker)
resource=dict(choices=['topic', 'acl'], default='topic'),
# resource name
name=dict(type='str', required=True),
partitions=dict(type='int', required=False, default=0),
replica_factor=dict(type='int', required=False, default=0),
acl_resource_type=dict(choices=[
'topic', 'broker', 'delegation_token', 'group',
'transactional_id'
],
default='topic'),
acl_principal=dict(type='str', required=False),
acl_operation=dict(choices=[
'all', 'alter', 'alter_configs', 'cluster_action', 'create',
'delete', 'describe', 'describe_configs', 'idempotent_write',
'read', 'write'
],
required=False),
acl_pattern_type=dict(
choice=['any', 'match', 'literal', 'prefixed'],
required=False,
default='literal'),
acl_permission=dict(choices=['allow', 'deny'], default='allow'),
acl_host=dict(type='str', required=False, default="*"),
state=dict(choices=['present', 'absent'], default='present'),
options=dict(required=False, type='dict', default=None),
zookeeper=dict(type='str', required=False),
zookeeper_auth_scheme=dict(choices=['digest', 'sasl'],
default='digest'),
zookeeper_auth_value=dict(type='str',
no_log=True,
required=False,
default=''),
zookeeper_ssl_check_hostname=dict(default=True,
type='bool',
required=False),
zookeeper_ssl_cafile=dict(required=False,
default=None,
type='path'),
zookeeper_ssl_certfile=dict(required=False,
default=None,
type='path'),
zookeeper_ssl_keyfile=dict(required=False,
default=None,
no_log=True,
type='path'),
zookeeper_ssl_password=dict(type='str',
no_log=True,
required=False),
zookeeper_sleep_time=dict(type='int', required=False, default=5),
zookeeper_max_retries=dict(type='int', required=False, default=5),
bootstrap_servers=dict(type='str', required=True),
security_protocol=dict(
choices=['PLAINTEXT', 'SSL', 'SASL_SSL', 'SASL_PLAINTEXT'],
default='PLAINTEXT'),
api_version=dict(type='str', required=True, default=None),
ssl_check_hostname=dict(default=True, type='bool', required=False),
ssl_cafile=dict(required=False, default=None, type='path'),
ssl_certfile=dict(required=False, default=None, type='path'),
ssl_keyfile=dict(required=False,
default=None,
no_log=True,
type='path'),
ssl_password=dict(type='str', no_log=True, required=False),
ssl_crlfile=dict(required=False, default=None, type='path'),
ssl_supported_protocols=dict(
required=False,
default=None,
type='list',
choices=['TLSv1', 'TLSv1.1', 'TLSv1.2']),
ssl_ciphers=dict(required=False, default=None, type='str'),
# only PLAIN is currently available
sasl_mechanism=dict(choices=['PLAIN', 'GSSAPI'], default='PLAIN'),
sasl_plain_username=dict(type='str', required=False),
sasl_plain_password=dict(type='str', no_log=True, required=False),
sasl_kerberos_service_name=dict(type='str', required=False),
),
supports_check_mode=True)
params = module.params
resource = params['resource']
name = params['name']
partitions = params['partitions']
replica_factor = params['replica_factor']
state = params['state']
zookeeper = params['zookeeper']
zookeeper_auth_scheme = params['zookeeper_auth_scheme']
zookeeper_auth_value = params['zookeeper_auth_value']
zookeeper_ssl_check_hostname = params['zookeeper_ssl_check_hostname']
zookeeper_ssl_cafile = params['zookeeper_ssl_cafile']
zookeeper_ssl_certfile = params['zookeeper_ssl_certfile']
zookeeper_ssl_keyfile = params['zookeeper_ssl_keyfile']
zookeeper_ssl_password = params['zookeeper_ssl_password']
zookeeper_sleep_time = params['zookeeper_sleep_time']
zookeeper_max_retries = params['zookeeper_max_retries']
bootstrap_servers = params['bootstrap_servers']
security_protocol = params['security_protocol']
ssl_check_hostname = params['ssl_check_hostname']
ssl_cafile = params['ssl_cafile']
ssl_certfile = params['ssl_certfile']
ssl_keyfile = params['ssl_keyfile']
ssl_password = params['ssl_password']
ssl_crlfile = params['ssl_crlfile']
ssl_supported_protocols = params['ssl_supported_protocols']
ssl_ciphers = params['ssl_ciphers']
sasl_mechanism = params['sasl_mechanism']
sasl_plain_username = params['sasl_plain_username']
sasl_plain_password = params['sasl_plain_password']
sasl_kerberos_service_name = params['sasl_kerberos_service_name']
acl_resource_type = params['acl_resource_type']
acl_principal = params['acl_principal']
acl_operation = params['acl_operation']
acl_permission = params['acl_permission']
acl_pattern_type = params['acl_pattern_type']
acl_host = params['acl_host']
api_version = tuple(
int(p) for p in params['api_version'].strip(".").split("."))
options = []
if params['options'] is not None:
options = params['options'].items()
kafka_ssl_files = generate_ssl_object(module, ssl_cafile, ssl_certfile,
ssl_keyfile, ssl_crlfile)
zookeeper_ssl_files = generate_ssl_object(module, zookeeper_ssl_cafile,
zookeeper_ssl_certfile,
zookeeper_ssl_keyfile)
zookeeper_use_ssl = bool(
zookeeper_ssl_files['keyfile']['path'] is not None
and zookeeper_ssl_files['certfile']['path'] is not None)
zookeeper_auth = []
if zookeeper_auth_value != '':
auth = (zookeeper_auth_scheme, zookeeper_auth_value)
zookeeper_auth.append(auth)
try:
# Generate ssl context to support limit ssl protocols & ciphers
ssl_context = None
if security_protocol in ('SSL', 'SASL_SSL'):
ssl_context = generate_ssl_context(
ssl_check_hostname=ssl_check_hostname,
ssl_cafile=kafka_ssl_files['cafile']['path'],
ssl_certfile=kafka_ssl_files['certfile']['path'],
ssl_keyfile=kafka_ssl_files['keyfile']['path'],
ssl_password=ssl_password,
ssl_crlfile=kafka_ssl_files['crlfile']['path'],
ssl_supported_protocols=ssl_supported_protocols,
ssl_ciphers=ssl_ciphers)
manager = KafkaManager(
module=module,
bootstrap_servers=bootstrap_servers,
security_protocol=security_protocol,
api_version=api_version,
ssl_context=ssl_context,
sasl_mechanism=sasl_mechanism,
sasl_plain_username=sasl_plain_username,
sasl_plain_password=sasl_plain_password,
sasl_kerberos_service_name=sasl_kerberos_service_name)
except Exception:
e = get_exception()
module.fail_json(msg='Error while initializing Kafka client : %s ' %
str(e))
changed = False
if parse_version(manager.get_api_version()) < parse_version('0.11.0'):
module.fail_json(
msg='Current version of library is not compatible with '
'Kafka < 0.11.0.')
msg = '%s \'%s\': ' % (resource, name)
if resource == 'topic':
if state == 'present':
if name in manager.get_topics():
# topic is already there
if zookeeper != '' and partitions > 0 and replica_factor > 0:
try:
manager.init_zk_client(
hosts=zookeeper,
auth_data=zookeeper_auth,
keyfile=zookeeper_ssl_files['keyfile']['path'],
use_ssl=zookeeper_use_ssl,
keyfile_password=zookeeper_ssl_password,
certfile=zookeeper_ssl_files['certfile']['path'],
ca=zookeeper_ssl_files['cafile']['path'],
verify_certs=zookeeper_ssl_check_hostname)
except Exception:
e = get_exception()
module.fail_json(
msg='Error while initializing Zookeeper client : '
'%s. Is your Zookeeper server available and '
'running on \'%s\'?' % (str(e), zookeeper))
if manager.is_topic_configuration_need_update(
name, options):
if not module.check_mode:
manager.update_topic_configuration(name, options)
changed = True
if manager.is_topic_replication_need_update(
name, replica_factor):
json_assignment = (
manager.get_assignment_for_replica_factor_update(
name, replica_factor))
if not module.check_mode:
manager.update_admin_assignment(
json_assignment, zookeeper_sleep_time,
zookeeper_max_retries)
changed = True
if manager.is_topic_partitions_need_update(
name, partitions):
cur_version = parse_version(manager.get_api_version())
if not module.check_mode:
if cur_version < parse_version('1.0.0'):
json_assignment = (
manager.
get_assignment_for_partition_update(
name, partitions))
zknode = '/brokers/topics/%s' % name
manager.update_topic_assignment(
json_assignment, zknode)
else:
manager.update_topic_partitions(
name, partitions)
changed = True
manager.close_zk_client()
if changed:
msg += 'successfully updated.'
else:
module.fail_json(
msg='\'zookeeper\', \'partitions\' and '
'\'replica_factor\' parameters are needed when '
'parameter \'state\' is \'present\'')
else:
# topic is absent
if not module.check_mode:
manager.create_topic(name=name,
partitions=partitions,
replica_factor=replica_factor,
config_entries=options)
changed = True
msg += 'successfully created.'
elif state == 'absent':
if name in manager.get_topics():
# delete topic
if not module.check_mode:
manager.delete_topic(name)
changed = True
msg += 'successfully deleted.'
elif resource == 'acl':
if not acl_operation:
module.fail_json(msg="acl_operation is required")
api_version = parse_version(manager.get_api_version())
acl_resource = ACLResource(
resource_type=ACLResourceType.from_name(acl_resource_type),
operation=ACLOperation.from_name(acl_operation),
permission_type=ACLPermissionType.from_name(acl_permission),
pattern_type=ACLPatternType.from_name(acl_pattern_type),
name=name,
principal=acl_principal,
host=acl_host)
acl_resource_found = manager.describe_acls(acl_resource, api_version)
if state == 'present':
if not acl_resource_found:
if not module.check_mode:
manager.create_acls([acl_resource], api_version)
changed = True
msg += 'successfully created.'
elif state == 'absent':
if acl_resource_found:
if not module.check_mode:
manager.delete_acls([acl_resource])
changed = True
msg += 'successfully deleted.'
manager.close()
for _key, value in merge_dicts(kafka_ssl_files,
zookeeper_ssl_files).items():
if (value['path'] is not None and value['is_temp']
and os.path.exists(os.path.dirname(value['path']))):
os.remove(value['path'])
if not changed:
msg += 'nothing to do.'
module.exit_json(changed=changed, msg=msg)
def get_manager_from_params(params):
bootstrap_servers = params['bootstrap_servers']
security_protocol = params['security_protocol']
ssl_check_hostname = params['ssl_check_hostname']
ssl_cafile = params['ssl_cafile']
ssl_certfile = params['ssl_certfile']
ssl_keyfile = params['ssl_keyfile']
ssl_password = params['ssl_password']
ssl_crlfile = params['ssl_crlfile']
ssl_supported_protocols = params['ssl_supported_protocols']
ssl_ciphers = params['ssl_ciphers']
sasl_mechanism = params['sasl_mechanism']
sasl_plain_username = params['sasl_plain_username']
sasl_plain_password = params['sasl_plain_password']
sasl_kerberos_service_name = params['sasl_kerberos_service_name']
request_timeout_ms = params['request_timeout_ms']
connections_max_idle_ms = params['connections_max_idle_ms']
api_version = tuple(
int(p) for p in params['api_version'].strip(".").split(".")
) if params.get('api_version') else None
kafka_ssl_files = generate_ssl_object(
ssl_cafile, ssl_certfile, ssl_keyfile, ssl_crlfile
)
# Generate ssl context to support limit ssl protocols & ciphers
ssl_context = None
if security_protocol in ('SSL', 'SASL_SSL'):
ssl_context = generate_ssl_context(
ssl_check_hostname=ssl_check_hostname,
ssl_cafile=kafka_ssl_files['cafile']['path'],
ssl_certfile=kafka_ssl_files['certfile']['path'],
ssl_keyfile=kafka_ssl_files['keyfile']['path'],
ssl_password=ssl_password,
ssl_crlfile=kafka_ssl_files['crlfile']['path'],
ssl_supported_protocols=ssl_supported_protocols,
ssl_ciphers=ssl_ciphers
)
manager = KafkaManager(
bootstrap_servers=bootstrap_servers,
security_protocol=security_protocol, api_version=api_version,
ssl_context=ssl_context,
sasl_mechanism=sasl_mechanism,
sasl_plain_username=sasl_plain_username,
sasl_plain_password=sasl_plain_password,
sasl_kerberos_service_name=sasl_kerberos_service_name,
request_timeout_ms=request_timeout_ms,
connections_max_idle_ms=connections_max_idle_ms
)
if parse_version(manager.get_api_version()) < parse_version('0.11.0'):
raise IncompatibleVersion(
'Current version of library is not compatible with '
'Kafka < 0.11.0.'
)
if 'kafka_sleep_time' in params:
manager.kafka_sleep_time = params['kafka_sleep_time']
if 'kafka_max_retries' in params:
manager.kafka_max_retries = params['kafka_max_retries']
if 'zookeeper' in params:
manager.zk_configuration = get_zookeeper_configuration(params)
if 'zookeeper_sleep_time' in params:
manager.zookeeper_sleep_time = params['zookeeper_sleep_time']
if 'zookeeper_max_retries' in params:
manager.zookeeper_max_retries = params['zookeeper_max_retries']
return manager
