def main():
module = AnsibleModule(argument_spec=dict(
consummer_group=dict(type='str', required=True),
ignore_empty_partition=dict(type='bool', default=False),
**module_commons))
params = module.params
consummer_group = params['consummer_group']
ignore_empty_partition = params['ignore_empty_partition']
manager = None
try:
manager = get_manager_from_params(params)
klag = KafkaConsumerLag(manager.client)
results = klag.get_lag_stats(consummer_group, ignore_empty_partition)
except KafkaError:
e = get_exception()
module.fail_json(msg='Error while getting lag from Kafka: %s' % e)
except Exception:
e = get_exception()
module.fail_json(msg='Seomthing went wrong: %s ' % e)
finally:
if manager:
manager.close()
maybe_clean_kafka_ssl_files(params)
# XXX: do we really need a JSON serialized value?
module.exit_json(changed=True, msg=json.dumps(results))
def process_module_topic(module):
params = module.params
name = params['name']
partitions = params['partitions']
replica_factor = params['replica_factor']
state = params['state']
options = []
if params['options'] is not None:
options = params['options'].items()
changed = False
msg = 'topic \'%s\': ' % (name)
warn = None
try:
manager = get_manager_from_params(params)
if state == 'present':
if name in manager.get_topics():
if not module.check_mode:
changed, warn = manager.ensure_topic(
name, options, partitions, replica_factor)
if changed:
msg += 'successfully updated.'
else:
if not module.check_mode:
manager.create_topic(name=name,
partitions=partitions,
replica_factor=replica_factor,
config_entries=options)
changed = True
msg += 'successfully created.'
elif state == 'absent':
if name in manager.get_topics():
if not module.check_mode:
manager.delete_topic(name)
changed = True
msg += 'successfully deleted.'
except KafkaError:
e = get_exception()
module.fail_json(msg='Unable to initialize Kafka manager: %s' % e)
except Exception:
e = get_exception()
module.fail_json(msg='Something went wrong: %s' % e)
finally:
manager.close()
maybe_clean_kafka_ssl_files(params)
maybe_clean_zk_ssl_files(params)
if not changed:
msg += 'nothing to do.'
if warn is not None:
module.warn(warn)
module.exit_json(changed=changed, msg=msg)
def main():
module = AnsibleModule(argument_spec=dict(
consumer_group=dict(type='str', required=True),
topics=dict(type='list',
elements='dict',
required=True,
options=dict(name=dict(type='str', required=True),
partitions=dict(type='list',
elements='int',
required=False))),
action=dict(type='str', required=True, required_one_of=[('delete')]),
**module_commons),
supports_check_mode=True)
params = module.params
consumer_group = params['consumer_group']
topics = params['topics']
action = params['action']
manager = None
try:
manager = get_manager_from_params(params)
api_version = parse_version(manager.get_api_version())
if (api_version < parse_version('2.4.0')):
module.fail_json(
msg='Delete offset API provided on kafka 2.4.0 (KIP-496)' +
' current version %s' % str(manager.get_api_version()))
if (action == 'delete'):
changed = manager.delete_group_offset(consumer_group, topics,
module.check_mode)
except KafkaError:
e = get_exception()
module.fail_json(
msg='Error while deleting kafka consumer group offset: %s' % e)
except Exception:
e = get_exception()
module.fail_json(msg='Seomthing went wrong: %s ' % e)
finally:
if manager:
manager.close()
maybe_clean_kafka_ssl_files(params)
if (changed):
msg = 'topics and partitions (%s) successfully deleted ' \
'for consumer group (%s)' % (topics, consumer_group)
else:
msg = 'nothing to do for consumer group %s and topics ' \
'partitions (%s)' % (consumer_group, topics)
module.exit_json(changed=changed, msg=msg)
def main():
module = AnsibleModule(
argument_spec=dict(
resource=dict(
choices=[
'topic',
'broker',
'consumer_group',
'acl',
'topic-config'
],
required=True
),
**module_commons
)
)
params = module.params
resource = params['resource']
try:
manager = get_manager_from_params(params)
results = manager.get_resource(resource)
except KafkaError:
e = get_exception()
module.fail_json(
msg='Error while getting %s from Kafka: %s ' % (resource, e)
)
except Exception:
e = get_exception()
module.fail_json(
msg='Seomthing went wrong: %s ' % e
)
finally:
manager.close()
maybe_clean_kafka_ssl_files(params)
module.exit_json(changed=True, results=results)
def process_module_topics(module, params=None):
params = params if params is not None else module.params
topics = params['topics']
mark_others_as_absent = params.get('mark_others_as_absent', False)
# Check for duplicated topics
duplicated_topics = [
topic for topic, count in collections.Counter(
[topic['name'] for topic in topics]).items() if count > 1
]
if len(duplicated_topics) > 0:
module.fail_json(msg='Got duplicated topics in \'topics\': %s' %
duplicated_topics)
return
changed = False
msg = ''
warn = None
changes = {}
manager = None
try:
manager = get_manager_from_params(params)
current_topics = manager.get_topics()
topics_to_create = [
topic for topic in topics
if (topic['state'] == 'present'
and topic['name'] not in current_topics)
]
if len(topics_to_create) > 0:
if not module.check_mode:
manager.create_topics(topics_to_create)
changed = True
msg += ''.join([
'topic %s successfully created. ' % topic['name']
for topic in topics_to_create
])
changes.update({'topic_created': topics_to_create})
topics_to_maybe_update = [
topic for topic in topics
if (topic['state'] == 'present' and topic['name'] in current_topics
)
]
if len(topics_to_maybe_update) > 0:
if not module.check_mode:
topics_changed, warn = manager.ensure_topics(
topics_to_maybe_update)
changed = len(topics_changed) > 0
if changed:
msg += ''.join([
'topic %s successfully updated. ' % topic
for topic in topics_changed
])
changes.update({'topic_updated': topics_changed})
topics_to_delete = [
topic for topic in topics
if (topic['state'] == 'absent' and topic['name'] in current_topics)
]
# Cleanup existing if necessary
if mark_others_as_absent:
defined_topics = [topic['name'] for topic in topics]
for existing_topic in set(current_topics) - set(defined_topics):
topics_to_delete.append({
'name': existing_topic,
'state': 'absent'
})
if len(topics_to_delete) > 0:
if not module.check_mode:
manager.delete_topics(topics_to_delete)
changed = True
msg += ''.join([
'topic %s successfully deleted. ' % topic['name']
for topic in topics_to_delete
])
changes.update({'topic_deleted': topics_to_delete})
except KafkaError:
e = get_exception()
module.fail_json(msg='Unable to initialize Kafka manager: %s' % e)
except Exception:
e = get_exception()
module.fail_json(msg='Something went wrong: (%s) %s' %
(e, traceback.format_exc(e)),
changes=changes)
finally:
if manager:
manager.close()
maybe_clean_kafka_ssl_files(params)
maybe_clean_zk_ssl_files(params)
if not changed:
msg += 'nothing to do.'
if warn is not None and len(warn) > 0:
module.warn(warn)
module.exit_json(changed=changed, msg=msg, changes=changes)
def process_module_acls(module, params=None):
params = params if params is not None else module.params
acls = params['acls']
mark_others_as_absent = params.get('mark_others_as_absent', False)
changed = False
msg = ''
warn = None
changes = {}
manager = None
try:
manager = get_manager_from_params(params)
api_version = parse_version(manager.get_api_version())
for acl in acls:
if not acl['acl_operation']:
module.fail_json(msg="acl_operation is required")
if acl['acl_resource_type'].lower() == 'broker':
module.deprecate(
'Usage of "broker" is deprecated, please use "cluster" '
'instead'
)
if len(acls) > 1:
acl_resource = ACLResource(
resource_type=ACLResourceType.ANY,
operation=ACLOperation.ANY,
permission_type=ACLPermissionType.ANY,
pattern_type=ACLPatternType.ANY,
name=None,
principal=None,
host=None
)
else:
acl = acls[0]
acl_name = acl['name']
acl_resource_type = acl['acl_resource_type']
acl_principal = acl['acl_principal']
acl_operation = acl['acl_operation']
acl_permission = acl['acl_permission']
acl_pattern_type = acl['acl_pattern_type']
acl_host = acl['acl_host']
acl_resource = ACLResource(
resource_type=ACLResourceType.from_name(acl_resource_type),
operation=ACLOperation.from_name(acl_operation),
permission_type=ACLPermissionType.from_name(
acl_permission
),
pattern_type=ACLPatternType.from_name(acl_pattern_type),
name=acl_name,
principal=acl_principal,
host=acl_host
)
acl_resource_found = manager.describe_acls(
acl_resource, api_version
)
acls_marked_present = [ACLResource(
resource_type=ACLResourceType.from_name(acl['acl_resource_type']),
operation=ACLOperation.from_name(acl['acl_operation']),
permission_type=ACLPermissionType.from_name(
acl['acl_permission']
),
pattern_type=ACLPatternType.from_name(acl['acl_pattern_type']),
name=acl['name'],
principal=acl['acl_principal'],
host=acl['acl_host']
) for acl in acls if acl['state'] == 'present']
acls_marked_absent = [ACLResource(
resource_type=ACLResourceType.from_name(acl['acl_resource_type']),
operation=ACLOperation.from_name(acl['acl_operation']),
permission_type=ACLPermissionType.from_name(
acl['acl_permission']
),
pattern_type=ACLPatternType.from_name(acl['acl_pattern_type']),
name=acl['name'],
principal=acl['acl_principal'],
host=acl['acl_host']
) for acl in acls if acl['state'] == 'absent']
# Check for duplicated acls
duplicated_acls = [acl for acl, count in collections.Counter(
acls_marked_absent + acls_marked_present
).items() if count > 1]
if len(duplicated_acls) > 0:
module.fail_json(
msg='Got duplicated acls in \'acls\': %s' % duplicated_acls
)
return
acls_to_add = [acl for acl in acls_marked_present
if acl not in acl_resource_found]
acls_to_delete = [acl for acl in acls_marked_absent
if acl in acl_resource_found]
# Cleanup others acls
if mark_others_as_absent:
acls_to_delete.extend(
[acl for acl in acl_resource_found
if acl not in acls_marked_present + acls_marked_absent]
)
if len(acls_to_add) > 0:
if not module.check_mode:
manager.create_acls(acls_to_add, api_version)
changed = True
msg += ''.join(['acl %s successfully created. ' %
acl for acl in acls_to_add])
changes.update({
'acls_added': [acl.to_dict() for acl in acls_to_add]
})
if len(acls_to_delete) > 0:
if not module.check_mode:
manager.delete_acls(acls_to_delete, api_version)
changed = True
msg += ''.join(['acl %s successfully deleted. ' %
acl for acl in acls_to_delete])
changes.update({
'acls_deleted': [acl.to_dict() for acl in acls_to_delete]
})
except KafkaError:
e = get_exception()
module.fail_json(
msg='Unable to initialize Kafka manager: %s' % e
)
except Exception:
e = get_exception()
module.fail_json(
msg='Something went wrong: (%s) %s' % (e, traceback.format_exc(e)),
changes=changes
)
finally:
if manager:
manager.close()
maybe_clean_kafka_ssl_files(params)
maybe_clean_zk_ssl_files(params)
if not changed:
msg += 'nothing to do.'
if warn is not None:
module.warn(warn)
module.exit_json(changed=changed, msg=msg, changes=changes)
def process_module_quotas(module, params=None):
params = params if params is not None else module.params
entries = _map_entries(params['entries'])
changed = False
msg = ''
changes = dict()
manager = None
try:
manager = get_manager_from_params(params)
current_entries = manager.describe_quotas()
alter_entries = []
for entry in entries:
found = False
entry_quotas = {
key: value
for key, value in entry['quotas'].items() if value is not None
}
for current_entry in current_entries:
current_entry_entity_sorted = \
sorted(current_entry['entity'],
key=lambda e: e['entity_type'])
entry_entity_sorted = sorted(entry['entity'],
key=lambda e: e['entity_type'])
if current_entry_entity_sorted == entry_entity_sorted:
found = True
if current_entry['quotas'] != entry_quotas:
keys_to_add = {
key: value
for key, value in entry_quotas.items()
if key not in current_entry['quotas']
}
keys_to_delete = {
key: value
for key, value in current_entry['quotas'].items()
if key not in entry_quotas
}
keys_to_alter = {
key: value
for key, value in entry_quotas.items()
if key in current_entry['quotas']
and current_entry['quotas'][key] != value
}
alter_entries.append({
'entity': entry['entity'],
'quotas_to_add': keys_to_add,
'quotas_to_alter': keys_to_alter,
'quotas_to_delete': keys_to_delete
})
if not found and len(entry_quotas) > 0:
alter_entries.append({
'entity': entry['entity'],
'quotas_to_add': entry_quotas,
'quotas_to_delete': dict(),
'quotas_to_alter': dict()
})
if len(alter_entries) > 0:
if not module.check_mode:
manager.alter_quotas(alter_entries)
changed = True
msg = 'entries altered'
changes = alter_entries
except KafkaError:
e = get_exception()
module.fail_json(msg='Something went wrong: (%s) %s' %
(e, traceback.format_exc(e)),
changes=changes)
except Exception:
e = get_exception()
module.fail_json(msg='Something went wrong: %s' % e)
finally:
if manager:
manager.close()
maybe_clean_kafka_ssl_files(params)
maybe_clean_zk_ssl_files(params)
if not changed:
msg += 'nothing to do.'
module.exit_json(changed=changed, msg=msg, changes=changes)
def main():
"""
Module usage
"""
module = AnsibleModule(
argument_spec=dict(
# resource managed, more to come (broker)
resource=dict(choices=['topic', 'acl'], default='topic'),
# resource name
name=dict(type='str', required=True),
partitions=dict(type='int', required=False, default=0),
replica_factor=dict(type='int', required=False, default=0),
acl_resource_type=dict(choices=[
'topic', 'broker', 'cluster', 'delegation_token', 'group',
'transactional_id'
],
default='topic'),
acl_principal=dict(type='str', required=False),
acl_operation=dict(choices=[
'all', 'alter', 'alter_configs', 'cluster_action', 'create',
'delete', 'describe', 'describe_configs', 'idempotent_write',
'read', 'write'
],
required=False),
acl_pattern_type=dict(
choice=['any', 'match', 'literal', 'prefixed'],
required=False,
default='literal'),
acl_permission=dict(choices=['allow', 'deny'], default='allow'),
acl_host=dict(type='str', required=False, default="*"),
state=dict(choices=['present', 'absent'], default='present'),
options=dict(required=False, type='dict', default=None),
zookeeper=dict(type='str', required=False),
zookeeper_auth_scheme=dict(choices=['digest', 'sasl'],
default='digest'),
zookeeper_auth_value=dict(type='str',
no_log=True,
required=False,
default=''),
zookeeper_ssl_check_hostname=dict(default=True,
type='bool',
required=False),
zookeeper_ssl_cafile=dict(required=False,
default=None,
type='path'),
zookeeper_ssl_certfile=dict(required=False,
default=None,
type='path'),
zookeeper_ssl_keyfile=dict(required=False,
default=None,
no_log=True,
type='path'),
zookeeper_ssl_password=dict(type='str',
no_log=True,
required=False),
zookeeper_sleep_time=dict(type='int', required=False, default=5),
zookeeper_max_retries=dict(type='int', required=False, default=5),
**module_commons),
supports_check_mode=True)
params = module.params
resource = params['resource']
name = params['name']
partitions = params['partitions']
replica_factor = params['replica_factor']
state = params['state']
zookeeper_sleep_time = params['zookeeper_sleep_time']
zookeeper_max_retries = params['zookeeper_max_retries']
acl_resource_type = params['acl_resource_type']
acl_principal = params['acl_principal']
acl_operation = params['acl_operation']
acl_permission = params['acl_permission']
acl_pattern_type = params['acl_pattern_type']
acl_host = params['acl_host']
options = []
if params['options'] is not None:
options = params['options'].items()
zk_configuration = get_zookeeper_configuration(params)
changed = False
msg = '%s \'%s\': ' % (resource, name)
warn = None
try:
manager = get_manager_from_params(params)
if resource == 'topic':
if state == 'present':
if name in manager.get_topics():
if not module.check_mode:
changed, warn = manager.ensure_topic(
name, zk_configuration, options, partitions,
replica_factor, zookeeper_sleep_time,
zookeeper_max_retries)
if changed:
msg += 'successfully updated.'
else:
if not module.check_mode:
manager.create_topic(name=name,
partitions=partitions,
replica_factor=replica_factor,
config_entries=options)
changed = True
msg += 'successfully created.'
elif state == 'absent':
if name in manager.get_topics():
if not module.check_mode:
manager.delete_topic(name)
changed = True
msg += 'successfully deleted.'
elif resource == 'acl':
if not acl_operation:
module.fail_json(msg="acl_operation is required")
api_version = parse_version(manager.get_api_version())
if acl_resource_type.lower() == 'broker':
module.deprecate(
'Usage of "broker" is deprecated, please use "cluster" '
'instead')
acl_resource = ACLResource(
resource_type=ACLResourceType.from_name(acl_resource_type),
operation=ACLOperation.from_name(acl_operation),
permission_type=ACLPermissionType.from_name(acl_permission),
pattern_type=ACLPatternType.from_name(acl_pattern_type),
name=name,
principal=acl_principal,
host=acl_host)
acl_resource_found = manager.describe_acls(acl_resource,
api_version)
if state == 'present':
if not acl_resource_found:
if not module.check_mode:
manager.create_acls([acl_resource], api_version)
changed = True
msg += 'successfully created.'
elif state == 'absent':
if acl_resource_found:
if not module.check_mode:
manager.delete_acls([acl_resource], api_version)
changed = True
msg += 'successfully deleted.'
except KafkaError:
e = get_exception()
module.fail_json(msg='Unable to initialize Kafka manager: %s' % e)
except Exception:
e = get_exception()
module.fail_json(msg='Something went wrong: %s' % e)
finally:
manager.close()
maybe_clean_kafka_ssl_files(params)
maybe_clean_zk_ssl_files(params)
if not changed:
msg += 'nothing to do.'
if warn is not None:
module.warn(warn)
module.exit_json(changed=changed, msg=msg)
def process_module_acl(module):
params = module.params
name = params['name']
state = params['state']
acl_resource_type = params['acl_resource_type']
acl_principal = params['acl_principal']
acl_operation = params['acl_operation']
acl_permission = params['acl_permission']
acl_pattern_type = params['acl_pattern_type']
acl_host = params['acl_host']
changed = False
msg = 'acl \'%s\': ' % (name)
warn = None
try:
manager = get_manager_from_params(params)
if not acl_operation:
module.fail_json(msg="acl_operation is required")
api_version = parse_version(manager.get_api_version())
if acl_resource_type.lower() == 'broker':
module.deprecate(
'Usage of "broker" is deprecated, please use "cluster" '
'instead')
acl_resource = ACLResource(
resource_type=ACLResourceType.from_name(acl_resource_type),
operation=ACLOperation.from_name(acl_operation),
permission_type=ACLPermissionType.from_name(acl_permission),
pattern_type=ACLPatternType.from_name(acl_pattern_type),
name=name,
principal=acl_principal,
host=acl_host)
acl_resource_found = manager.describe_acls(acl_resource, api_version)
if state == 'present':
if not acl_resource_found:
if not module.check_mode:
manager.create_acls([acl_resource], api_version)
changed = True
msg += 'successfully created.'
elif state == 'absent':
if acl_resource_found:
if not module.check_mode:
manager.delete_acls([acl_resource], api_version)
changed = True
msg += 'successfully deleted.'
except KafkaError:
e = get_exception()
module.fail_json(msg='Unable to initialize Kafka manager: %s' % e)
except Exception:
e = get_exception()
module.fail_json(msg='Something went wrong: %s' % e)
finally:
manager.close()
maybe_clean_kafka_ssl_files(params)
maybe_clean_zk_ssl_files(params)
if not changed:
msg += 'nothing to do.'
if warn is not None:
module.warn(warn)
module.exit_json(changed=changed, msg=msg)