def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall,
ignorefirstpromoted):
"""Clean Content Views"""
# Set the task name to be displayed in the task monitoring stage
task_name = "Cleanup content views"
# Now we have all the info needed, we can actually trigger the cleanup.
task_list = []
ref_list = {}
# Catch scenario that no CV versions are found matching cleanup criteria
if not ver_list:
msg = "No content view versions found matching cleanup criteria"
helpers.log_msg(msg, 'ERROR')
sys.exit(1)
for cvid in ver_list.keys():
msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'"
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
# Check if there is a publish/promote already running on this content view
locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid])
if locked:
continue
# For the given content view we need to find the orphaned versions
cvinfo = get_content_view_info(cvid)
# Find the oldest published version
version_list = []
orphan_versions = []
orphan_dict = {}
all_versions = []
ccv_versions = []
for version in cvinfo['versions']:
# Check if the version is part of a published view.
# This is not returned in cvinfo, and we need to see if we are part of a CCV
version_in_use, version_in_ccv = check_version_views(version['id'])
# Build a list of ALL version numbers
all_versions.append(float(version['version']))
# Add any version numbers that are part of a CCV to a list
if version_in_ccv:
ccv_versions.append(float(version['version']))
if not version['environment_ids']:
# These are the versions that don't belong to an environment (i.e. orphans)
# We also cross-check for versions that may be in a CCV here.
# We add the version name and id into a dictionary so we can delete by id.
if not version_in_use:
orphan_versions.append(float(version['version']))
orphan_dict[version['version']] = version['id']
continue
else:
msg = "Found version " + str(version['version'])
helpers.log_msg(msg, 'DEBUG')
# Add the version id to a list
version_list.append(float(version['version']))
# Find the oldest 'in use' version id
if not version_list:
msg = "No oldest in-use version found"
else:
lastver = min(version_list)
msg = "Oldest in-use version is " + str(lastver)
helpers.log_msg(msg, 'DEBUG')
# Find the oldest 'NOT in use' version id
if not orphan_versions:
msg = "No oldest NOT-in-use version found"
else:
msg = "Oldest NOT-in-use version is " + str(min(orphan_versions))
helpers.log_msg(msg, 'DEBUG')
# Find the element position in the all_versions list of the oldest in-use version
# e.g. vers 102.0 is oldest in-use and is element [5] in the all_versions list
list_position = [i for i, x in enumerate(all_versions) if x == lastver]
# Remove the number of views to keep from the element position of the oldest in-use
# e.g. keep=2 results in an adjusted list element position [3]
num_to_delete = list_position[0] - int(ver_keep[cvid])
# Delete from position [0] to the first 'keep' position
# e.g. first keep element is [3] so list of elements [0, 1, 2] is created
list_pos_to_delete = [i for i in range(num_to_delete)]
# Find versions to delete (based on keep parameter)
# Make sure the version list is in order
orphan_versions.sort()
if cleanall:
# Remove all orphaned versions
todelete = orphan_versions
elif ignorefirstpromoted:
# Remove the last 'keep' elements from the orphans list (from PR #26)
todelete = orphan_versions[:(len(orphan_versions) -
int(ver_keep[cvid]))]
else:
todelete = []
# Remove the element numbers for deletion from the list all versions
for i in sorted(list_pos_to_delete, reverse=True):
todelete.append(orphan_versions[i])
msg = "Versions to remove: " + str(todelete)
helpers.log_msg(msg, 'DEBUG')
for version in all_versions:
if not locked:
if version in todelete:
msg = "Orphan view version " + str(version) + " found in '" +\
str(ver_descr[cvid]) + "'"
helpers.log_msg(msg, 'DEBUG')
# Lookup the version_id from our orphan_dict
delete_id = orphan_dict.get(str(version))
msg = "Removing version " + str(version)
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
else:
if version in ccv_versions:
msg = "Skipping delete of version " + str(
version) + " (member of a CCV)"
elif version in orphan_versions:
msg = "Skipping delete of version " + str(
version) + " (due to keep value)"
else:
msg = "Skipping delete of version " + str(
version) + " (in use)"
helpers.log_msg(msg, 'INFO')
print msg
continue
else:
msg = "Version " + str(version) + " is locked"
helpers.log_msg(msg, 'WARNING')
continue
# Delete the view version from the content view
if not dry_run and not locked:
try:
task_id = helpers.put_json(
helpers.KATELLO_API + "content_views/" + str(cvid) +
"/remove/",
json.dumps({
"id": cvid,
"content_view_version_ids": delete_id
}))['id']
# Wait for the task to complete
helpers.wait_for_task(task_id, 'clean')
# Check if the deletion completed successfully
tinfo = helpers.get_task_status(task_id)
if tinfo['state'] != 'running' and tinfo[
'result'] == 'success':
msg = "Removal of content view version OK"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + "OK" + helpers.ENDC
else:
msg = "Failed"
helpers.log_msg(msg, 'ERROR')
except Warning:
msg = "Failed to initiate removal"
helpers.log_msg(msg, 'WARNING')
except KeyError:
msg = "Failed to initiate removal (KeyError)"
helpers.log_msg(msg, 'WARNING')
# Exit in the case of a dry-run
if dry_run:
msg = "Dry run - not actually performing removal"
helpers.log_msg(msg, 'WARNING')
sys.exit(2)
def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefirstpromoted):
"""Clean Content Views"""
# Set the task name to be displayed in the task monitoring stage
task_name = "Cleanup content views"
# Now we have all the info needed, we can actually trigger the cleanup.
task_list = []
ref_list = {}
# Catch scenario that no CV versions are found matching cleanup criteria
if not ver_list:
msg = "No content view versions found matching cleanup criteria"
helpers.log_msg(msg, 'ERROR')
sys.exit(1)
for cvid in sorted(ver_list.keys(),reverse=True):
# Check if there is a publish/promote already running on this content view
locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid])
msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'"
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
# For the given content view we need to find the orphaned versions
cvinfo = get_content_view_info(cvid)
# Find the oldest published version
version_list = []
version_list_all = []
for version in cvinfo['versions']:
if not version['environment_ids']:
version_list_all.append(float(version['version']))
continue
else:
msg = "Found version " + str(version['version'])
helpers.log_msg(msg, 'DEBUG')
# Add the version id to a list
version_list.append(float(version['version']))
# Find the oldest 'in use' version id
if not version_list:
msg = "No oldest in-use version found"
else:
lastver = min(version_list)
msg = "Oldest in-use version is " + str(lastver)
helpers.log_msg(msg, 'DEBUG')
# Find the oldest 'NOT in use' version id
if not version_list_all:
msg = "No oldest NOT-in-use version found"
else:
msg = "Oldest NOT-in-use version is " + str(min(version_list_all))
helpers.log_msg(msg, 'DEBUG')
# Find version to delete (based on keep parameter) if --ignorefirstpromoted
version_list_all.sort()
todelete = version_list_all[:(len(version_list_all) - int(ver_keep[cvid]))]
msg = "Versions to remove if --ignorefirstpromoted: " + str(todelete)
helpers.log_msg(msg, 'DEBUG')
for version in cvinfo['versions']:
# Get composite content views for version
cvv = get_content_view_version(version['id'])
# Find versions that are not in any environment and not in any composite content view
if not version['environment_ids'] and not cvv['composite_content_view_ids']:
if not locked:
msg = "Orphan view version " + str(version['version']) + " found in '" +\
str(ver_descr[cvid]) + "'"
helpers.log_msg(msg, 'DEBUG')
if ignorefirstpromoted:
if cleanall:
msg = "Removing version " + str(version['version'])
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
else:
if float(version['version']) in todelete:
# If ignorefirstpromoted delete CV
msg = "Removing version " + str(version['version'])
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
else:
msg = "Skipping delete of version " + str(version['version']) + " due to --keep value"
helpers.log_msg(msg, 'INFO')
print msg
continue
else:
if float(version['version']) > float(lastver):
# If we have chosen to remove all orphans
if cleanall:
msg = "Removing version " + str(version['version'])
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
else:
msg = "Skipping delete of version " + str(version['version'])
helpers.log_msg(msg, 'INFO')
print msg
continue
else:
if float(version['version']) < (lastver - float(ver_keep[cvid])):
msg = "Removing version " + str(version['version'])
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
else:
msg = "Skipping delete of version " + str(version['version']) + " due to --keep value"
helpers.log_msg(msg, 'INFO')
print msg
continue
# Delete the view version from the content view
if not dry_run and not locked:
try:
task = helpers.put_json(
helpers.KATELLO_API + "content_views/" + str(cvid) + "/remove/",
json.dumps(
{
"id": cvid,
"content_view_version_ids": version['id']
}
)
)
if id in task:
task_id = task['id']
# Wait for the task to complete
helpers.wait_for_task(task_id,'clean')
# Check if the deletion completed successfully
tinfo = helpers.get_task_status(task_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Removal of content view version OK"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + "OK" + helpers.ENDC
else:
msg = "Failed"
helpers.log_msg(msg, 'ERROR')
else:
msg = "Can't remove content view " + str(cvid)
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
except Warning:
msg = "Failed to initiate removal"
helpers.log_msg(msg, 'WARNING')
# Exit in the case of a dry-run
if dry_run:
msg = "Dry run - not actually performing removal"
helpers.log_msg(msg, 'WARNING')
sys.exit(2)
def sync_content(org_id, imported_repos):
"""
Synchronize the repositories
Triggers a sync of all repositories belonging to the configured sync plan
"""
repos_to_sync = []
delete_override = False
# Get a listing of repositories in this Satellite
enabled_repos = helpers.get_p_json(
helpers.KATELLO_API + "/repositories/", \
json.dumps(
{
"organization_id": org_id,
"per_page": '1000',
}
))
# Loop through each repo to be imported/synced
for repo in imported_repos:
do_import = False
for repo_result in enabled_repos['results']:
if repo in repo_result['label']:
# Ensure we have an exact match on the repo label
if repo == repo_result['label']:
do_import = True
repos_to_sync.append(repo_result['id'])
# Ensure Mirror-on-sync flag is set to FALSE to make sure incremental
# import does not (cannot) delete existing packages.
msg = "Setting mirror-on-sync=false for repo id " + str(
repo_result['id'])
helpers.log_msg(msg, 'DEBUG')
helpers.put_json(
helpers.KATELLO_API + "/repositories/" + str(repo_result['id']), \
json.dumps(
{
"mirror_on_sync": False
}
))
if do_import:
msg = "Repo " + repo + " found in Satellite"
helpers.log_msg(msg, 'DEBUG')
else:
msg = "Repo " + repo + " is not enabled in Satellite"
# If the repo is not enabled, don't delete the input files.
# This gives the admin a chance to manually enable the repo and re-import
delete_override = True
helpers.log_msg(msg, 'WARNING')
# TODO: We could go on here and try to enable the Red Hat repo .....
# If we get to here and nothing was added to repos_to_sync we will abort the import.
# This will probably occur on the initial import - nothing will be enabled in Satellite.
# Also if there are no updates during incremental sync.
if not repos_to_sync:
msg = "No updates in imported content - skipping sync"
helpers.log_msg(msg, 'WARNING')
return
else:
msg = "Repo ids to sync: " + str(repos_to_sync)
helpers.log_msg(msg, 'DEBUG')
msg = "Syncing repositories"
helpers.log_msg(msg, 'INFO')
print msg
# Break repos_to_sync into groups of n
repochunks = [
repos_to_sync[i:i + helpers.SYNCBATCH]
for i in range(0, len(repos_to_sync), helpers.SYNCBATCH)
]
# Loop through the smaller batches of repos and sync them
for chunk in repochunks:
chunksize = len(chunk)
msg = "Syncing repo batch " + str(chunk)
helpers.log_msg(msg, 'DEBUG')
task_id = helpers.post_json(
helpers.KATELLO_API + "repositories/bulk/sync", \
json.dumps(
{
"ids": chunk,
}
))["id"]
msg = "Repo sync task id = " + task_id
helpers.log_msg(msg, 'DEBUG')
# Now we need to wait for the sync to complete
helpers.wait_for_task(task_id, 'sync')
tinfo = helpers.get_task_status(task_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Batch of " + str(chunksize) + " repos complete"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
else:
msg = "Batch sync has errors"
helpers.log_msg(msg, 'WARNING')
return delete_override
def cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefirstpromoted):
"""Clean Content Views"""
# Set the task name to be displayed in the task monitoring stage
task_name = "Cleanup content views"
# Now we have all the info needed, we can actually trigger the cleanup.
task_list = []
ref_list = {}
# Catch scenario that no CV versions are found matching cleanup criteria
if not ver_list:
msg = "No content view versions found matching cleanup criteria"
helpers.log_msg(msg, 'ERROR')
sys.exit(1)
for cvid in ver_list.keys():
msg = "Cleaning content view '" + str(ver_descr[cvid]) + "'"
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
# Check if there is a publish/promote already running on this content view
locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid])
if locked:
continue
# For the given content view we need to find the orphaned versions
cvinfo = get_content_view_info(cvid)
# Find the oldest published version
version_list = []
orphan_versions = []
orphan_dict = {}
all_versions = []
ccv_versions = []
for version in cvinfo['versions']:
# Check if the version is part of a published view.
# This is not returned in cvinfo, and we need to see if we are part of a CCV
version_in_use, version_in_ccv = check_version_views(version['id'])
# Build a list of ALL version numbers
all_versions.append(float(version['version']))
# Add any version numbers that are part of a CCV to a list
if version_in_ccv:
ccv_versions.append(float(version['version']))
if not version['environment_ids']:
# These are the versions that don't belong to an environment (i.e. orphans)
# We also cross-check for versions that may be in a CCV here.
# We add the version name and id into a dictionary so we can delete by id.
if not version_in_use:
orphan_versions.append(float(version['version']))
orphan_dict[version['version']] = version['id']
continue
else:
msg = "Found version " + str(version['version'])
helpers.log_msg(msg, 'DEBUG')
# Add the version id to a list
version_list.append(float(version['version']))
# Find the oldest 'in use' version id
if not version_list:
msg = "No oldest in-use version found"
else:
lastver = min(version_list)
msg = "Oldest in-use version is " + str(lastver)
helpers.log_msg(msg, 'DEBUG')
# Find the oldest 'NOT in use' version id
if not orphan_versions:
msg = "No oldest NOT-in-use version found"
else:
msg = "Oldest NOT-in-use version is " + str(min(orphan_versions))
helpers.log_msg(msg, 'DEBUG')
# Find the element position in the all_versions list of the oldest in-use version
# e.g. vers 102.0 is oldest in-use and is element [5] in the all_versions list
list_position = [i for i,x in enumerate(all_versions) if x == lastver]
# Remove the number of views to keep from the element position of the oldest in-use
# e.g. keep=2 results in an adjusted list element position [3]
num_to_delete = list_position[0] - int(ver_keep[cvid])
# Delete from position [0] to the first 'keep' position
# e.g. first keep element is [3] so list of elements [0, 1, 2] is created
list_pos_to_delete = [i for i in range(num_to_delete)]
# Find versions to delete (based on keep parameter)
# Make sure the version list is in order
orphan_versions.sort()
if cleanall:
# Remove all orphaned versions
todelete = orphan_versions
elif ignorefirstpromoted:
# Remove the last 'keep' elements from the orphans list (from PR #26)
todelete = orphan_versions[:(len(orphan_versions) - int(ver_keep[cvid]))]
else:
todelete = []
# Remove the element numbers for deletion from the list all versions
for i in sorted(list_pos_to_delete, reverse=True):
todelete.append(orphan_versions[i])
msg = "Versions to remove: " + str(todelete)
helpers.log_msg(msg, 'DEBUG')
for version in all_versions:
if not locked:
if version in todelete:
msg = "Orphan view version " + str(version) + " found in '" +\
str(ver_descr[cvid]) + "'"
helpers.log_msg(msg, 'DEBUG')
# Lookup the version_id from our orphan_dict
delete_id = orphan_dict.get(str(version))
msg = "Removing version " + str(version)
helpers.log_msg(msg, 'INFO')
print helpers.HEADER + msg + helpers.ENDC
else:
if version in ccv_versions:
msg = "Skipping delete of version " + str(version) + " (member of a CCV)"
elif version in orphan_versions:
msg = "Skipping delete of version " + str(version) + " (due to keep value)"
else:
msg = "Skipping delete of version " + str(version) + " (in use)"
helpers.log_msg(msg, 'INFO')
print msg
continue
else:
msg = "Version " + str(version) + " is locked"
helpers.log_msg(msg, 'WARNING')
continue
# Delete the view version from the content view
if not dry_run and not locked:
try:
task_id = helpers.put_json(
helpers.KATELLO_API + "content_views/" + str(cvid) + "/remove/",
json.dumps(
{
"id": cvid,
"content_view_version_ids": delete_id
}
))['id']
# Wait for the task to complete
helpers.wait_for_task(task_id,'clean')
# Check if the deletion completed successfully
tinfo = helpers.get_task_status(task_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Removal of content view version OK"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + "OK" + helpers.ENDC
else:
msg = "Failed"
helpers.log_msg(msg, 'ERROR')
except Warning:
msg = "Failed to initiate removal"
helpers.log_msg(msg, 'WARNING')
except KeyError:
msg = "Failed to initiate removal (KeyError)"
helpers.log_msg(msg, 'WARNING')
# Exit in the case of a dry-run
if dry_run:
msg = "Dry run - not actually performing removal"
helpers.log_msg(msg, 'WARNING')
sys.exit(2)
def sync_content(org_id, imported_repos):
"""
Synchronize the repositories
Triggers a sync of all repositories belonging to the configured sync plan
"""
repos_to_sync = []
delete_override = False
# Get a listing of repositories in this Satellite
enabled_repos = helpers.get_p_json(
helpers.KATELLO_API + "/repositories/", \
json.dumps(
{
"organization_id": org_id,
"per_page": '1000',
}
))
# Loop through each repo to be imported/synced
for repo in imported_repos:
do_import = False
for repo_result in enabled_repos['results']:
if repo in repo_result['label']:
do_import = True
repos_to_sync.append(repo_result['id'])
# Ensure Mirror-on-sync flag is set to FALSE to make sure incremental
# import does not (cannot) delete existing packages.
msg = "Setting mirror-on-sync=false for repo id " + str(repo_result['id'])
helpers.log_msg(msg, 'DEBUG')
helpers.put_json(
helpers.KATELLO_API + "/repositories/" + str(repo_result['id']), \
json.dumps(
{
"mirror_on_sync": False
}
))
if do_import:
msg = "Repo " + repo + " found in Satellite"
helpers.log_msg(msg, 'DEBUG')
else:
msg = "Repo " + repo + " is not enabled in Satellite"
# If the repo is not enabled, don't delete the input files.
# This gives the admin a chance to manually enable the repo and re-import
delete_override = True
helpers.log_msg(msg, 'WARNING')
# TODO: We could go on here and try to enable the Red Hat repo .....
# If we get to here and nothing was added to repos_to_sync we will abort the import.
# This will probably occur on the initial import - nothing will be enabled in Satellite.
# Also if there are no updates during incremental sync.
if not repos_to_sync:
msg = "No updates in imported content - skipping sync"
helpers.log_msg(msg, 'WARNING')
return
else:
msg = "Repo ids to sync: " + str(repos_to_sync)
helpers.log_msg(msg, 'DEBUG')
msg = "Syncing repositories"
helpers.log_msg(msg, 'INFO')
print msg
# Break repos_to_sync into groups of n
repochunks = [ repos_to_sync[i:i+helpers.SYNCBATCH] for i in range(0, len(repos_to_sync), helpers.SYNCBATCH) ]
# Loop through the smaller batches of repos and sync them
for chunk in repochunks:
chunksize = len(chunk)
msg = "Syncing repo batch " + str(chunk)
helpers.log_msg(msg, 'DEBUG')
task_id = helpers.post_json(
helpers.KATELLO_API + "repositories/bulk/sync", \
json.dumps(
{
"ids": chunk,
}
))["id"]
msg = "Repo sync task id = " + task_id
helpers.log_msg(msg, 'DEBUG')
# Now we need to wait for the sync to complete
helpers.wait_for_task(task_id, 'sync')
tinfo = helpers.get_task_status(task_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Batch of " + str(chunksize) + " repos complete"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
else:
msg = "Batch sync has errors"
helpers.log_msg(msg, 'WARNING')
return delete_override