def _update(cluster: Cluster, project: Project, current_project_json, parameter_contexts: list):
logger = logging.getLogger(__name__)
project_cluster = project.get_project_cluster(cluster)
if project_cluster is None:
logger.warning(f'No cluster configuration found for project: {project.name}')
return
project_cluster.project_process_group_id = current_project_json['id']
if project.description.lower() != current_project_json['component']['comments'].lower():
put_url = '/' + url_helper.construct_path_parts(['process-groups', project_cluster.project_process_group_id])
current_project_json['component']['comments'] = project.description
try:
response = requests.put(**cluster._get_connection_details(put_url), json=current_project_json)
if response.status_code != 200:
logger.warning(response.text)
return
logger.info(f'Updated project: {project.name}, in cluster: {cluster.name}, with new description: {project.description}.')
except requests.exceptions.RequestException as exception:
logger.warning(f'Unable to update project: {project.name}, in cluster: {cluster.name}.')
logger.warning(exception)
return
else:
logger.info(f'Project: {project.name}, in cluster: {cluster.name}, is up-to-date.')
_get_available_versions(cluster, project)
environment_service.sync(cluster, project, project_cluster, parameter_contexts)
def _delete(cluster: Cluster, access_policy_descriptor: AccessPolicyDescriptor,
delete_access_policy_json, component_id: str):
logger = logging.getLogger(__name__)
resource = access_policy_descriptor.resource
if component_id:
resource = resource.replace('{id}', component_id)
delete_url = '/' + url_helper.construct_path_parts(
['policies', delete_access_policy_json['id']])
try:
response = requests.delete(
**cluster._get_connection_details(delete_url),
params={
'version':
str(delete_access_policy_json['revision']['version'])
})
if response.status_code != 200:
logger.warning(
f'Unable to delete access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, from cluster: {cluster.name}.'
)
logger.warning(response.text)
return
logger.info(
f'Deleted access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, from cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to delete access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, from cluster: {cluster.name}.'
)
logger.warning(exception)
def _create(cluster: Cluster, user: User):
logger = logging.getLogger(__name__)
create_json = {
'revision': {
'version': 0
},
'component': {
'identity': user.identity
}
}
try:
response = requests.post(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['tenants', 'users'])),
json=create_json)
if response.status_code != 201:
logger.warning(
f'Unable to create user: {user.identity}, in cluster: {cluster.name}.'
)
logger.warning(response.text)
return
response_json = response.json()
user.component_id = response_json['id']
user.revision_version = response_json['revision']['version']
logger.info(
f'Created user: {user.identity}, in cluster: {cluster.name}.')
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to create user: {user.identity}, cluster: {cluster.name}.'
)
logger.warning(exception)
def _delete(cluster: Cluster, delete_parameter_context_json):
logger = logging.getLogger(__name__)
parameter_context_name = delete_parameter_context_json['component']['name']
delete_url = '/' + url_helper.construct_path_parts([
'parameter-contexts', delete_parameter_context_json['component']['id']
])
try:
response = requests.delete(
**cluster._get_connection_details(delete_url),
params={
'version':
str(delete_parameter_context_json['revision']['version'])
})
if response.status_code != 200:
logger.warning(
f'Unable to delete parameter context: {parameter_context_name}, from cluster: {cluster.name}.'
)
logger.warning(response.text)
return
logger.info(
f'Deleted parameter context: {parameter_context_name}, from cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to delete parameter context: {parameter_context_name}, from cluster: {cluster.name}.'
)
logger.warning(exception)
def _update_version(desired_version, cluster: Cluster, project: Project,
environment: ProjectEnvironment, environment_json):
logger = logging.getLogger(__name__)
registry_json = cluster.registeries_json_dict[project.registry_name]
version_update_json = {
'processGroupRevision': environment_json['revision'],
'versionControlInformation': {
'groupId': environment_json['id'],
'registryId': registry_json['id'],
'bucketId': project.bucket_id,
'flowId': project.flow_id,
'version': desired_version
}
}
# Todo: since version control is an async process inside nifi, we need an async logic or while loop to check
# the request status to issue delete request when the versionrequest is processed by the nifi server
post_url = '/' + url_helper.construct_path_parts([
'versions', 'update-requests', 'process-groups', environment_json['id']
])
try:
response = requests.post(**cluster._get_connection_details(post_url),
json=version_update_json)
if response.status_code != 200:
logger.warning(response.text)
return
logger.info(
f'Updated project: {project.name}, environment: {environment.name}, version: {desired_version}, in cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to update project: {project.name}, environment: {environment.name}, version: {desired_version}, in cluster: {cluster.name}.'
)
logger.warning(exception)
def _create(cluster: Cluster, project: Project, parameter_contexts: list):
logger = logging.getLogger(__name__)
project_cluster = project.get_project_cluster(cluster)
if project_cluster is None:
logger.info(f'No cluster configuration found for project: {project.name}')
return
post_url = '/' + url_helper.construct_path_parts(['process-groups', cluster.root_process_group_id, 'process-groups'])
create_json = {
'revision': {
'version': 0
},
'component': {
'name': project.name,
'comments': project.description
}
}
try:
response = requests.post(**cluster._get_connection_details(post_url), json=create_json)
if response.status_code != 201:
logger.warning(response.text)
return
project_cluster.project_process_group_id = response.json()['id']
logger.info(f'Created project: {project.name}, in cluster: {cluster.name}.')
except requests.exceptions.RequestException as exception:
logger.warning(f'Unable to create project: {project.name}, in cluster: {cluster.name}.')
logger.warning(exception)
return
_get_available_versions(cluster, project)
environment_service.sync(cluster, project, project_cluster, parameter_contexts)
def _get_current_user_json(cluster: Cluster):
current_user_identity = requests.get(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['flow', 'current-user']))).json(
)['identity']
users_json = requests.get(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['tenants', 'users']))).json()
current_user_jsons = [
u for u in users_json['users']
if u['component']['identity'].lower() == current_user_identity.lower()
]
if len(current_user_jsons) > 0:
return current_user_jsons[0]
return None
def sync(cluster: Cluster, configured_users: list):
"""Set the cluster users to desired configuration."""
logger = logging.getLogger(__name__)
logger.info(f'Getting coordinator user for cluster: {cluster.name}')
response = requests.get(**cluster._get_connection_details(
'/' +
url_helper.construct_path_parts(['flow', 'current-user']))).json()
current_user_identity = response['identity']
desired_users = configured_users
if len([
u for u in desired_users
if u.identity.lower() == current_user_identity.lower()
]) == 0:
desired_users.append(User(current_user_identity))
logger.info(
f'Collecting currently configured users for cluster: {cluster.name}')
response = requests.get(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['tenants', 'users']))).json()
current_users_json_dict = {
user['component']['identity']: user
for user in response['users']
}
for delete_user_name in list(
filter(
lambda k: current_user_identity.lower() != k.lower() and len([
u for u in desired_users
if u.identity.lower() == k.lower()
]) == 0, current_users_json_dict.keys())):
_delete(cluster, current_users_json_dict[delete_user_name])
for user in desired_users:
if user.identity in current_users_json_dict:
_update(cluster, user, current_users_json_dict[user.identity])
else:
_create(cluster, user)
def sync(cluster: Cluster, configured_registries: list):
"""Set the cluster nifi-registry entries to their desired configuration."""
logger = logging.getLogger(__name__)
logger.info(
f'Collecting currently configure registries for cluster: {cluster.name}'
)
cluster_registries_json = requests.get(**cluster._get_connection_details(
'/controller/registry-clients')).json()
# Build dictionaries for the desired registry configuration & the current registry configuration.
# The dictionaries are indexed by name to support efficient lookups & comparisons.
desired_registries_dict = {
registry.name: registry
for registry in configured_registries
}
current_registries_json_dict = {
registry['component']['name']: registry
for registry in cluster_registries_json['registries']
}
# loop through registries & set appropriate values.
for name, desired_registry, current_registry_json in _registry_munger(
desired_registries_dict, current_registries_json_dict):
if current_registry_json is None:
_create(cluster, desired_registry)
elif desired_registry is None:
# TODO: Delete the registry.
logger.warn(f'Deleting {current_registry_json}')
_delete(cluster, current_registry_json)
else:
_update(cluster, desired_registry, current_registry_json)
cluster_registries_json = requests.get(**cluster._get_connection_details(
'/controller/registry-clients')).json()
cluster.registeries_json_dict = {
registry['component']['name']: registry
for registry in cluster_registries_json['registries']
}
def _delete(cluster: Cluster, registry: Registry):
logger = logging.getLogger(__name__)
logger.info(f'Deleting registry with id {registry["id"]}')
try:
response = requests.delete(**cluster._get_connection_details(
f'/controller/registry-clients/{registry["id"]}'),
params={
'version':
registry['revision']['version']
})
logger.debug(response.text)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to reach {cluster.name}, will try again later.')
logger.warning(exception)
def sync(cluster: Cluster, project: Project, project_cluster: ProjectCluster,
parameter_contexts: list):
logger = logging.getLogger(__name__)
desired_environments = list(
filter(lambda e: e.is_coordinated, project_cluster.environments))
logger.info(
f'Collecting currently configured environments for project: {project.name}'
)
response = requests.get(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts([
'process-groups', project_cluster.project_process_group_id,
'process-groups'
]))).json()
current_environments_json_dict = {
project['component']['name']: project
for project in response['processGroups']
}
for delete_env_name in list(
filter(
lambda k: len([
p for p in project_cluster.environments
if p.name.lower() == k.lower()
]) == 0, current_environments_json_dict.keys())):
_delete(cluster, project,
current_environments_json_dict[delete_env_name])
for environment in desired_environments:
if environment.name in current_environments_json_dict:
_update(cluster, project, project_cluster, environment,
current_environments_json_dict[environment.name],
parameter_contexts)
else:
_create(cluster, project, project_cluster, environment,
parameter_contexts)
uncoordinated_environments = list(
filter(lambda e: not e.is_coordinated, project_cluster.environments))
for environment in uncoordinated_environments:
if environment.name in current_environments_json_dict:
environment.process_group_id = current_environments_json_dict[
environment.name]['id']
else:
environment.process_group_id = ''
def sync(cluster: Cluster, configured_parameter_contexts: list):
"""Set the cluster parameter contexts to desired configuration."""
logger = logging.getLogger(__name__)
desired_parameter_contexts = list(
filter(lambda pc: pc.is_coordinated, configured_parameter_contexts))
logger.info(
f'Collecting currently configure parameter contexts for cluster: {cluster.name}'
)
response = requests.get(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['flow', 'parameter-contexts']))
).json()
current_parameter_contexts_json_dict = {
context['component']['name']: context
for context in response['parameterContexts']
}
for delete_parameter_context_name in list(
filter(
lambda k: len([
pc for pc in configured_parameter_contexts
if pc.name.lower() == k.lower()
]) == 0, current_parameter_contexts_json_dict.keys())):
_delete(
cluster,
current_parameter_contexts_json_dict[delete_parameter_context_name]
)
for parameter_context in desired_parameter_contexts:
if parameter_context.name in current_parameter_contexts_json_dict:
_update(
cluster, parameter_context,
current_parameter_contexts_json_dict[parameter_context.name])
else:
_create(cluster, parameter_context)
uncoordinated_parameter_contexts = list(
filter(lambda pc: not pc.is_coordinated,
configured_parameter_contexts))
for parameter_context in uncoordinated_parameter_contexts:
if parameter_context.name in current_parameter_contexts_json_dict:
parameter_context.id = current_parameter_contexts_json_dict[
parameter_context.name]['id']
else:
parameter_context.id = None
def _delete(cluster: Cluster, delete_user_group_json):
logger = logging.getLogger(__name__)
user_group_identity = delete_user_group_json['component']['identity']
delete_url = '/' + url_helper.construct_path_parts(['tenants', 'user-groups', delete_user_group_json['id']])
try:
response = requests.delete(
**cluster._get_connection_details(delete_url),
params={'version': str(delete_user_group_json['revision']['version'])})
if response.status_code != 200:
logger.warning(f'Unable to delete user group: {user_group_identity}, from cluster: {cluster.name}.')
logger.warning(response.text)
return
logger.info(f'Deleted user group: {user_group_identity}, from cluster: {cluster.name}.')
except requests.exceptions.RequestException as exception:
logger.warning(f'Unable to delete user group: {user_group_identity}, from cluster: {cluster.name}.')
logger.warning(exception)
def _update(cluster: Cluster, parameter_context: ParameterContext,
current_parameter_context_json):
logger = logging.getLogger(__name__)
parameter_context.id = current_parameter_context_json['id']
if (parameter_context.description.lower() !=
current_parameter_context_json['component']['description'].lower()
or _did_parameters_change(parameter_context,
current_parameter_context_json)):
url = '/' + url_helper.construct_path_parts(
['parameter-contexts', parameter_context.id, 'update-requests'])
current_parameter_context_json['component'][
'description'] = parameter_context.description
current_parameter_context_json['component']['parameters'] = []
for parameter in parameter_context.parameters:
current_parameter_context_json['component']['parameters'].append({
'parameter': {
'name': parameter.name,
'description': parameter.description,
'sensitive': parameter.is_sensitive,
'value': parameter.value
}
})
try:
response = requests.post(**cluster._get_connection_details(url),
json=current_parameter_context_json)
if response.status_code != 200:
logger.warning(response.text)
return
logger.info(
f'Updated parameter context: {parameter_context.name}, in cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to update parameter context: {parameter_context.name}, in cluster: {cluster.name}.'
)
logger.warning(exception)
else:
logger.info(
f'Parameter context: {parameter_context.name}, in cluster: {cluster.name}, is up-to-date.'
)
def _create(cluster: Cluster, user_group: UserGroup, configured_users: list):
logger = logging.getLogger(__name__)
create_json = {
'revision': {
'version': 0
},
'component': {
'identity': user_group.identity,
'users': []
}
}
for user_identity in user_group.members:
users = [u for u in configured_users if u.identity.lower() == user_identity.lower()]
if len(users) > 0:
create_json['component']['users'].append({
'revision': {
'version': 0
},
'id': users[0].component_id,
'component': {
'identity': user_identity,
'id': users[0].component_id
}
})
else:
logger.warning(f'User: {user_identity} not found in configured users.')
try:
response = requests.post(
**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['tenants', 'user-groups'])), json=create_json)
if response.status_code != 201:
logger.warning(f'Unable to create user group: {user_group.identity}, in cluster: {cluster.name}.')
logger.warning(response.text)
return
response_json = response.json()
user_group.component_id = response_json['id']
user_group.revision_version = response_json['revision']['version']
logger.info(f'Created user group: {user_group.identity}, in cluster: {cluster.name}.')
except requests.exceptions.RequestException as exception:
logger.warning(f'Unable to create user group: {user_group.identity}, cluster: {cluster.name}.')
logger.warning(exception)
def _create(cluster: Cluster, access_policy_descriptor: AccessPolicyDescriptor,
policy_users: list, policy_user_groups: list,
configured_users: list, configured_user_groups: list,
component_id: str):
logger = logging.getLogger(__name__)
resource = access_policy_descriptor.resource
if component_id:
resource = resource.replace('{id}', component_id)
create_json = {
'revision': {
'version': 0
},
'component': {
'resource':
'/' + resource,
'action':
access_policy_descriptor.action,
'users':
_get_access_policy_users_json(policy_users, configured_users),
'userGroups':
_get_access_policy_user_groups_json(policy_user_groups,
configured_user_groups)
}
}
try:
response = requests.post(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['policies'])),
json=create_json)
if response.status_code != 201:
logger.warning(
f'Unable to create access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, in cluster: {cluster.name}.'
)
logger.warning(response.text)
return
logger.info(
f'Created access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, in cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to create access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, cluster: {cluster.name}.'
)
logger.warning(exception)
def _create(cluster: Cluster, parameter_context: ParameterContext):
logger = logging.getLogger(__name__)
create_json = {
'revision': {
'version': 0
},
'component': {
'name': parameter_context.name,
'description': parameter_context.description,
'parameters': []
}
}
for parameter in parameter_context.parameters:
create_json['component']['parameters'].append({
'parameter': {
'name': parameter.name,
'description': parameter.description,
'sensitive': parameter.is_sensitive,
'value': parameter.value
}
})
try:
response = requests.post(**cluster._get_connection_details(
'/parameter-contexts'),
json=create_json)
if response.status_code != 201:
logger.warning(
f'Unable to create parameter context: {parameter_context.name}, in cluster: {cluster.name}.'
)
logger.warning(response.text)
return
parameter_context.id = response.json()['id']
logger.info(
f'Created parameter context: {parameter_context.name}, in cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to create parameter context: {parameter_context.name}, cluster: {cluster.name}.'
)
logger.warning(exception)
def _get_available_versions(cluster: Cluster, project: Project):
logger = logging.getLogger(__name__)
if not(project.registry_name in cluster.registeries_json_dict):
logger.warning(f'Unable to find registry: {project.registry_name}, in cluster: {cluster.name}.')
return
registry_json = cluster.registeries_json_dict[project.registry_name]
url = '/' + url_helper.construct_path_parts(['flow', 'registries', registry_json['id'], 'buckets', project.bucket_id, 'flows', project.flow_id, 'versions'])
try:
response = requests.get(**cluster._get_connection_details(url))
if response.status_code != 200:
logger.info(f'Unable to get flow versions for project: {project.name}, Response: {response.text}')
return
project.available_versions_dict = {version['versionedFlowSnapshotMetadata']['version']: version for version in response.json()['versionedFlowSnapshotMetadataSet']}
except requests.exceptions.RequestException as exception:
logger.warning(f'Unable to get flow versions for project: {project.name}.')
logger.warning(exception)
def _update(cluster: Cluster, registry: Registry, current_registry_json):
"""Update the existing registry in the cluster if it has a different URI."""
logger = logging.getLogger(__name__)
if (registry.uri.lower() !=
current_registry_json['component']['uri'].lower()
or registry.description.lower() !=
current_registry_json['component']['description'].lower()):
logger.warning(
f'Registry details mismatch for {registry.name} and {cluster.name}, updating.'
)
current_registry_json['component']['uri'] = registry.uri
try:
response = requests.put(**cluster._get_connection_details(
f'/controller/registry-clients/{current_registry_json["id"]}'),
json=current_registry_json)
logger.debug(response.text)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to reach {cluster.name}, will try again later.')
logger.warning(exception)
def sync(cluster: Cluster, configured_projects: list, parameter_contexts: list):
"""Set the cluster projects to their desired configuration."""
logger = logging.getLogger(__name__)
desired_projects = list(filter(lambda p: len([c for c in p.clusters if c.name.lower() == cluster.name.lower()]) > 0, configured_projects))
logger.info(f'Collecting currently configure projects for cluster: {cluster.name}')
response = requests.get(
**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['process-groups', cluster.root_process_group_id, 'process-groups']))).json()
current_projects_json_dict = {project['component']['name']: project for project in response['processGroups']}
for delete_project_name in list(filter(lambda k: len([p for p in desired_projects if p.name.lower() == k.lower()]) == 0, current_projects_json_dict.keys())):
_delete(cluster, current_projects_json_dict[delete_project_name])
for desired_project in desired_projects:
if desired_project.name in current_projects_json_dict:
_update(cluster, desired_project, current_projects_json_dict[desired_project.name], parameter_contexts)
else:
_create(cluster, desired_project, parameter_contexts)
def _update(cluster: Cluster, user_group: UserGroup, current_user_group_json, configured_users: list):
logger = logging.getLogger(__name__)
user_group.component_id = current_user_group_json['id']
user_group.revision_version = current_user_group_json['revision']['version']
if _did_members_change(user_group, current_user_group_json):
url = '/' + url_helper.construct_path_parts(['tenants', 'user-groups', current_user_group_json['id']])
current_user_group_json['component']['users'] = []
for user_identity in user_group.members:
users = [u for u in configured_users if u.identity.lower() == user_identity.lower()]
if len(users) > 0:
current_user_group_json['component']['users'].append({
'revision': {
'version': users[0].revision_version
},
'id': users[0].component_id,
'component': {
'identity': user_identity,
'id': users[0].component_id
}
})
else:
logger.warning(f'User: {user_identity} not found in configured users.')
try:
response = requests.put(**cluster._get_connection_details(url), json=current_user_group_json)
if response.status_code != 200:
logger.warning(f'Unable to update members for user group: {user_group.identity}, in cluster: {cluster.name}.')
logger.warning(response.text)
return
user_group.revision_version = response.json()['revision']['version']
logger.info(f'Updated members for user group: {user_group.identity}, in cluster: {cluster.name}.')
except requests.exceptions.RequestException as exception:
logger.warning(f'Unable to update members for user group: {user_group.identity}, in cluster: {cluster.name}.')
logger.warning(exception)
else:
logger.info(f'Members for user group: {user_group.identity}, in cluster: {cluster.name}, are up-to-date.')
def _delete(cluster: Cluster, project: Project, delete_environment_json):
logger = logging.getLogger(__name__)
environment_name = delete_environment_json['component']['name']
delete_url = '/' + url_helper.construct_path_parts(
['process-groups', delete_environment_json['component']['id']])
try:
response = requests.delete(**cluster._get_connection_details(
delete_url),
params=delete_environment_json['revision'])
if response.status_code != 200:
logger.warning(response.text)
return
logger.info(
f'Deleted project: {project.name}, environment: {environment_name}, from cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to delete project: {project.name}, environment: {environment_name}, from cluster: {cluster.name}.'
)
logger.warning(exception)
def _get_access_policy_json(cluster: Cluster,
access_policy_descriptor: AccessPolicyDescriptor,
component_id: str):
logger = logging.getLogger(__name__)
resource = access_policy_descriptor.resource
if component_id:
resource = resource.replace('{id}', component_id)
try:
response = requests.get(**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(
['policies', access_policy_descriptor.action, resource])))
if response.status_code != 200:
return None
return response.json()
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to get access policy by action: {access_policy_descriptor.action}, resource: {resource}, in cluster: {cluster.name}.'
)
logger.warning(exception)
return None
def sync(cluster: Cluster, configured_user_groups: list, configured_users: list):
"""Set the cluster user groups to desired configuration."""
logger = logging.getLogger(__name__)
desired_users_groups = configured_user_groups
logger.info(f'Collecting currently configured user groups for cluster: {cluster.name}')
response = requests.get(
**cluster._get_connection_details(
'/' + url_helper.construct_path_parts(['tenants', 'user-groups']))).json()
current_user_groups_json_dict = {ug['component']['identity']: ug for ug in response['userGroups']}
for delete_user_name in list(
filter(
lambda k: len([
u for u in desired_users_groups if u.identity.lower() == k.lower()
]) == 0, current_user_groups_json_dict.keys())):
_delete(cluster, current_user_groups_json_dict[delete_user_name])
for user_group in desired_users_groups:
if user_group.identity in current_user_groups_json_dict:
_update(cluster, user_group, current_user_groups_json_dict[user_group.identity], configured_users)
else:
_create(cluster, user_group, configured_users)
def _create(cluster: Cluster, registry: Registry):
"""Create the missing registry in the cluster."""
logger = logging.getLogger(__name__)
logger.info(f'Adding {registry.name} to cluster: {cluster.name}')
data = {
'revision': revision_0,
'component': vars(
registry
) # HACK: vars(foo) creates a dictonary of values, need to find a better way to do this
}
try:
response = requests.post(**cluster._get_connection_details(
'/controller/registry-clients'),
json=data)
logger.debug(response.text)
if response.status_code != 201:
# NiFi clusters which are not configured with keystore/truststores cannot be configured
# to talk to https nifi registries. This will catch
logger.warning(response.text)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to reach {cluster.name}, will try again later.')
logger.warning(exception)
def _update(cluster: Cluster, access_policy_descriptor: AccessPolicyDescriptor,
policy_users: list, policy_user_groups: list,
configured_users: list, configured_user_groups: list,
current_access_policy_json, current_user_json, component_id: str):
logger = logging.getLogger(__name__)
resource = access_policy_descriptor.resource
if component_id:
resource = resource.replace('{id}', component_id)
desired_policy_users = policy_users
append_current_user = (_does_current_user_has_policy(
access_policy_descriptor.action, resource, current_user_json) and len([
u for u in desired_policy_users
if u.lower() == current_user_json['component']['identity'].lower()
]) == 0)
if append_current_user:
desired_policy_users.append(current_user_json['component']['identity'])
if _did_users_or_groups_change(desired_policy_users, policy_user_groups,
current_access_policy_json):
url = '/' + url_helper.construct_path_parts(
['policies', current_access_policy_json['id']])
current_access_policy_json['component'][
'users'] = _get_access_policy_users_json(policy_users,
configured_users)
current_access_policy_json['component'][
'userGroups'] = _get_access_policy_user_groups_json(
policy_user_groups, configured_user_groups)
if append_current_user:
current_access_policy_json['component']['users'].append({
'revision':
current_user_json['revision'],
'id':
current_user_json['id'],
'component': {
'id': current_user_json['id'],
'identity': current_user_json['component']['identity']
}
})
try:
response = requests.put(**cluster._get_connection_details(url),
json=current_access_policy_json)
if response.status_code != 200:
logger.warning(
f'Unable to update access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, in cluster: {cluster.name}.'
)
logger.warning(response.text)
return
logger.info(
f'Updated access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, in cluster: {cluster.name}.'
)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to update access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, in cluster: {cluster.name}.'
)
logger.warning(exception)
else:
logger.info(
f'Access policy ({access_policy_descriptor.name}): {access_policy_descriptor.action}/{resource}, in cluster: {cluster.name}, is up-to-date.'
)
def _create(cluster: Cluster, project: Project,
project_cluster: ProjectCluster, environment: ProjectEnvironment,
parameter_contexts: list):
logger = logging.getLogger(__name__)
desired_version = _get_desired_version(project, environment)
if desired_version is None:
logger.warning(
f'Unable to find project version: {environment.version}, for project: {project.name}, environment: {environment.name}.'
)
return
project_cluster = project.get_project_cluster(cluster)
if project_cluster is None:
logger.info(
f'No cluster configuration found for project: {project.name}')
return
post_url = '/' + url_helper.construct_path_parts([
'process-groups', project_cluster.project_process_group_id,
'process-groups'
])
create_json = {
'revision': {
'version': 0
},
'component': {
'name': environment.name,
'comments': environment.description,
}
}
if environment.parameter_context_name:
parameter_context = _get_parameter_context_by_name(
environment.parameter_context_name, parameter_contexts)
if parameter_context is None or (parameter_context.is_coordinated
and parameter_context.id is None):
logger.warning(
f'Unable to find parameter context: {environment.parameter_context_name}, for project: {project.name}, environment: {environment.name}.'
)
return
if not (parameter_context.id is None):
create_json['component']['parameterContext'] = {
'id': parameter_context.id,
'component': {
'id': parameter_context.id,
'name': environment.parameter_context_name
}
}
try:
response = requests.post(**cluster._get_connection_details(post_url),
json=create_json)
if response.status_code != 201:
logger.warning(response.text)
return
response_json = response.json()
environment.process_group_id = response_json['id']
logger.info(
f'Created project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}.'
)
_update(cluster, project, project_cluster, environment, response_json,
parameter_contexts)
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to create project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}.'
)
logger.warning(exception)
def _update(cluster: Cluster, project: Project,
project_cluster: ProjectCluster, environment: ProjectEnvironment,
environment_json, parameter_contexts: list):
logger = logging.getLogger(__name__)
environment.process_group_id = environment_json['id']
desired_version = _get_desired_version(project, environment)
if desired_version is None:
logger.warning(
f'Unable to find project version: {environment.version}, for project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}.'
)
return
update_version = True
if ('versionControlInformation' in environment_json['component']
and desired_version == environment_json['component']
['versionControlInformation']['version']):
update_version = False
if (environment.name.lower() !=
environment_json['component']['name'].lower()
# Temporarily commented: comments get overridden by versioned flow, nifi may have an issue here
# or environment.description.lower() != environment_json['component']['comments'].lower()
or
(environment.parameter_context_name
and not ('parameterContext' in environment_json['component'])) or
(not environment.parameter_context_name
and 'parameterContext' in environment_json['component']) or
(environment.parameter_context_name
and 'parameterContext' in environment_json['component']
and environment.parameter_context_name.lower() != environment_json[
'component']['parameterContext']['component']['name'].lower())):
update_json = {
'revision': environment_json['revision'],
'component': {
'id': environment_json['id'],
'name': environment.name,
'comments': environment.description,
}
}
if environment.parameter_context_name:
parameter_context = _get_parameter_context_by_name(
environment.parameter_context_name, parameter_contexts)
if parameter_context is None or (parameter_context.is_coordinated
and parameter_context.id is None):
logger.warning(
f'Unable to find parameter context: {environment.parameter_context_name}, for project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}.'
)
return
if not (parameter_context.id is None):
update_json['component']['parameterContext'] = {
'id': parameter_context.id,
'component': {
'id': parameter_context.id,
'name': environment.parameter_context_name
}
}
# remove version control to update
if 'versionControlInformation' in environment_json['component']:
try:
delete_version_url = '/' + url_helper.construct_path_parts(
['versions', 'process-groups', environment_json['id']])
response = requests.delete(**cluster._get_connection_details(
delete_version_url),
params=environment_json['revision'])
if response.status_code != 200:
logger.warning(response.text)
return
update_version = True
# update json with new revision
delete_version_response_json = response.json()
if 'processGroupRevision' in delete_version_response_json:
update_json['revision'] = delete_version_response_json[
'processGroupRevision']
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to temporarily delete version control from project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}.'
)
logger.warning(exception)
return
try:
put_url = '/' + url_helper.construct_path_parts(
['process-groups', environment_json['id']])
response = requests.put(**cluster._get_connection_details(put_url),
json=update_json)
if response.status_code != 200:
logger.warning(response.text)
return
logger.info(
f'Updated project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}.'
)
environment_json = response.json()
except requests.exceptions.RequestException as exception:
logger.warning(
f'Unable to update project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}.'
)
logger.warning(exception)
elif not update_version:
logger.info(
f'Project: {project.name}, environment: {environment.name}, in cluster: {cluster.name}, is up-to-date.'
)
if update_version:
_update_version(desired_version, cluster, project, environment,
environment_json)
