def main(args):
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if helpers.DISCONNECTED:
msg = "Export cannot be run on the disconnected Satellite host"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Log the fact we are starting
msg = "------------- Content export started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(
description='Performs Export of Default Content View.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o',
'--org',
help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-e',
'--env',
help='Environment config file',
required=False)
group.add_argument('-a',
'--all',
help='Export ALL content',
required=False,
action="store_true")
group.add_argument('-i',
'--incr',
help='Incremental Export of content since last run',
required=False,
action="store_true")
group.add_argument('-s',
'--since',
help='Export content since YYYY-MM-DD HH:MM:SS',
required=False,
type=helpers.valid_date)
parser.add_argument('-l',
'--last',
help='Display time of last export',
required=False,
action="store_true")
parser.add_argument('-n',
'--nogpg',
help='Skip GPG checking',
required=False,
action="store_true")
parser.add_argument('-r',
'--repodata',
help='Include repodata for repos with no new packages',
required=False,
action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
since = args.since
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
exported_repos = []
# If a specific environment is requested, find and read that config file
repocfg = os.path.join(dir, 'config/' + args.env + '.yml')
if args.env:
if not os.path.exists(repocfg):
print "ERROR: Config file " + repocfg + " not found."
sys.exit(-1)
cfg = yaml.safe_load(open(repocfg, 'r'))
ename = args.env
erepos = cfg["env"]["repos"]
msg = "Specific environment export called for " + ename + ". Configured repos:"
helpers.log_msg(msg, 'DEBUG')
for repo in erepos:
msg = " - " + repo
helpers.log_msg(msg, 'DEBUG')
else:
ename = 'DoV'
label = 'DoV'
msg = "DoV export called"
helpers.log_msg(msg, 'DEBUG')
# Get the current time - this will be the 'last export' time if the export is OK
start_time = datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S')
print "START: " + start_time + " (" + ename + " export)"
# Read the last export date pickle for our selected repo group.
export_times = read_pickle(ename)
export_type = 'incr'
if args.all:
print "Performing full content export for " + ename
export_type = 'full'
since = False
else:
if not since:
since = False
if args.last:
if export_times:
print "Last successful export for " + ename + ":"
for time in export_times:
repo = "{:<70}".format(time)
print repo[:70] + '\t' + str(export_times[time])
else:
print "Export has never been performed for " + ename
sys.exit(-1)
if not export_times:
print "No prior export recorded for " + ename + ", performing full content export"
export_type = 'full'
else:
# Re-populate export_times dictionary so each repo has 'since' date
since_export = str(since)
# We have our timestamp so we can kick of an incremental export
print "Incremental export of content for " + ename + " synchronised after " \
+ str(since)
# Check the available space in /var/lib/pulp
check_disk_space(export_type)
# Remove any previous exported content left behind by prior unclean exit
if os.path.exists(helpers.EXPORTDIR + '/export'):
msg = "Removing existing export directory"
helpers.log_msg(msg, 'DEBUG')
shutil.rmtree(helpers.EXPORTDIR + '/export')
# Collect a list of enabled repositories. This is needed for:
# 1. Matching specific repo exports, and
# 2. Running import sync per repo on the disconnected side
repolist = helpers.get_p_json(
helpers.KATELLO_API + "/repositories/", \
json.dumps(
{
"organization_id": org_id,
"per_page": '1000',
}
))
# If we are running a full DoV export we run a different set of API calls...
if ename == 'DoV':
cola = "Exporting DoV"
if export_type == 'incr' and 'DoV' in export_times:
last_export = export_times['DoV']
if since:
last_export = since_export
colb = "(INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
colb = "(FULL)"
msg = cola + " " + colb
helpers.log_msg(msg, 'INFO')
output = "{:<70}".format(cola)
print output[:70] + ' ' + colb
# Check if there are any currently running tasks that will conflict with an export
check_running_tasks(label, ename)
# Get the version of the CV (Default Org View) to export
dov_ver = get_cv(org_id)
# Now we have a CV ID and a starting date, and no conflicting tasks, we can export
export_id = export_cv(dov_ver, last_export, export_type)
# Now we need to wait for the export to complete
helpers.wait_for_task(export_id, 'export')
# Check if the export completed OK. If not we exit the script.
tinfo = helpers.get_task_status(export_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Content View Export OK"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
# Update the export timestamp for this repo
export_times['DoV'] = start_time
# Generate a list of repositories that were exported
for repo_result in repolist['results']:
if repo_result['content_type'] == 'yum':
# Add the repo to the successfully exported list
exported_repos.append(repo_result['label'])
else:
msg = "Content View Export FAILED"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
else:
# Verify that defined repos exist in Satellite
for repo in erepos:
repo_in_sat = False
for repo_x in repolist['results']:
if re.findall("\\b" + repo + "\\b$", repo_x['label']):
repo_in_sat = True
break
if repo_in_sat == False:
msg = "'" + repo + "' not found in Satellite"
helpers.log_msg(msg, 'WARNING')
# Process each repo
for repo_result in repolist['results']:
if repo_result['content_type'] == 'yum':
# If we have a match, do the export
if repo_result['label'] in erepos:
# Extract the last export time for this repo
orig_export_type = export_type
cola = "Export " + repo_result['label']
if export_type == 'incr' and repo_result[
'label'] in export_times:
last_export = export_times[repo_result['label']]
if since:
last_export = since_export
colb = "(INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
colb = "(FULL)"
msg = cola + " " + colb
helpers.log_msg(msg, 'INFO')
output = "{:<70}".format(cola)
print output[:70] + ' ' + colb
# Check if there are any currently running tasks that will conflict
ok_to_export = check_running_tasks(repo_result['label'],
ename)
if ok_to_export:
# Trigger export on the repo
export_id = export_repo(repo_result['id'], last_export,
export_type)
# Now we need to wait for the export to complete
helpers.wait_for_task(export_id, 'export')
# Check if the export completed OK. If not we exit the script.
tinfo = helpers.get_task_status(export_id)
if tinfo['state'] != 'running' and tinfo[
'result'] == 'success':
# Count the number of exported packages
# First resolve the product label - this forms part of the export path
product = get_product(
org_id, repo_result['product']['cp_id'])
# Now we can build the export path itself
basepath = helpers.EXPORTDIR + "/" + org_name + "-" + product + "-" + repo_result[
'label']
if export_type == 'incr':
basepath = basepath + "-incremental"
exportpath = basepath + "/" + repo_result[
'relative_path']
msg = "\nExport path = " + exportpath
helpers.log_msg(msg, 'DEBUG')
os.chdir(exportpath)
numrpms = len([
f for f in os.walk(".").next()[2]
if f[-4:] == ".rpm"
])
msg = "Repository Export OK (" + str(
numrpms) + " new packages)"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
# Update the export timestamp for this repo
export_times[repo_result['label']] = start_time
# Add the repo to the successfully exported list
if numrpms != 0 or args.repodata:
msg = "Adding " + repo_result[
'label'] + " to export list"
helpers.log_msg(msg, 'DEBUG')
exported_repos.append(repo_result['label'])
else:
msg = "Not including repodata for empty repo " + repo_result[
'label']
helpers.log_msg(msg, 'DEBUG')
else:
msg = "Export FAILED"
helpers.log_msg(msg, 'ERROR')
# Reset the export type to the user specified, in case we overrode it.
export_type = orig_export_type
else:
msg = "Skipping " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
# Handle FILE type exports (ISO repos)
elif repo_result['content_type'] == 'file':
# If we have a match, do the export
if repo_result['label'] in erepos:
# Extract the last export time for this repo
orig_export_type = export_type
cola = "Export " + repo_result['label']
if export_type == 'incr' and repo_result[
'label'] in export_times:
last_export = export_times[repo_result['label']]
if since:
last_export = since_export
colb = "(INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
colb = "(FULL)"
msg = cola + " " + colb
helpers.log_msg(msg, 'INFO')
output = "{:<70}".format(cola)
print output[:70] + ' ' + colb
# Check if there are any currently running tasks that will conflict
ok_to_export = check_running_tasks(repo_result['label'],
ename)
if ok_to_export:
# Trigger export on the repo
numfiles = export_iso(repo_result['id'],
repo_result['label'],
repo_result['relative_path'],
last_export, export_type)
# Reset the export type to the user specified, in case we overrode it.
export_type = orig_export_type
# Update the export timestamp for this repo
export_times[repo_result['label']] = start_time
# Add the repo to the successfully exported list
if numfiles != 0 or args.repodata:
msg = "Adding " + repo_result[
'label'] + " to export list"
helpers.log_msg(msg, 'DEBUG')
exported_repos.append(repo_result['label'])
else:
msg = "Not including repodata for empty repo " + repo_result[
'label']
helpers.log_msg(msg, 'DEBUG')
else:
msg = "Skipping " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
# Combine resulting directory structures into a single repo format (top level = /content)
prep_export_tree(org_name)
# Now we need to process the on-disk export data.
# Define the location of our exported data.
export_dir = helpers.EXPORTDIR + "/export"
# Write out the list of exported repos. This will be transferred to the disconnected system
# and used to perform the repo sync tasks during the import.
pickle.dump(exported_repos, open(export_dir + '/exported_repos.pkl', 'wb'))
# Run GPG Checks on the exported RPMs
if not args.nogpg:
do_gpg_check(export_dir)
# Add our exported data to a tarfile
create_tar(export_dir, ename)
# We're done. Write the start timestamp to file for next time
os.chdir(script_dir)
pickle.dump(export_times, open(vardir + '/exports_' + ename + '.pkl',
"wb"))
# And we're done!
print helpers.GREEN + "Export complete.\n" + helpers.ENDC
print 'Please transfer the contents of ' + helpers.EXPORTDIR + \
' to your disconnected Satellite system content import location.\n' \
'Once transferred, please run ' + helpers.BOLD + ' sat_import' \
+ helpers.ENDC + ' to extract it.'
def main(args):
"""
Main routine
"""
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
confdir = os.path.join(dir, 'config')
# Check for sane input
parser = argparse.ArgumentParser(
description='Publishes content views for specified organization.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
group.add_argument('-a', '--all', help='Publish ALL content views', required=False,
action="store_true")
parser.add_argument('-d', '--dryrun', help='Dry Run - Only show what will be published',
required=False, action="store_true")
parser.add_argument('-l', '--last', help='Display last promotions', required=False,
action="store_true")
args = parser.parse_args()
# Log the fact we are starting
if not args.last:
msg = "-------- Content view publish started by " + runuser + " -----------"
helpers.log_msg(msg, 'INFO')
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dry_run = args.dryrun
# Load the promotion history
if not os.path.exists(vardir + '/promotions.pkl'):
if not os.path.exists(vardir):
os.makedirs(vardir)
phistory = {}
else:
phistory = pickle.load(open(vardir + '/promotions.pkl', 'rb'))
# Read the promotion history if --last requested
if args.last:
if phistory:
print 'Last promotions:'
for lenv, time in phistory.iteritems():
print lenv, time
else:
print 'No promotions recorded'
sys.exit(0)
publish_list = []
if not args.all:
publish_list = helpers.CONFIG['publish']['content_views']
if not publish_list:
msg = "Cannot find publish configuration"
helpers.log_msg(msg, 'ERROR')
sys.exit(1)
msg = "Config found for CV's " + str(publish_list)
helpers.log_msg(msg, 'DEBUG')
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the list of Content Views along with the latest view version in each environment
(ver_list, ver_descr, ver_version) = get_cv(org_id, publish_list)
# Publish the content views. Returns a list of task IDs.
(task_list, ref_list, task_name) = publish(ver_list, ver_descr, ver_version, dry_run, runuser)
# Add/Update the promotion history dictionary so we can check when we last promoted
phistory['Library'] = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
pickle.dump(phistory, open(vardir + '/promotions.pkl', 'wb'))
# Monitor the status of the publish tasks
helpers.watch_tasks(task_list, ref_list, task_name)
# Exit cleanly
sys.exit(0)
def main(args):
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
confdir = os.path.join(dir, 'config')
# Check for sane input
parser = argparse.ArgumentParser(
description='Exports puppet modules in puppet-forge-server format.')
# pylint: disable=bad-continuation
parser.add_argument('-o',
'--org',
help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-r',
'--repo',
help='Puppetforge repo label',
required=False)
parser.add_argument(
'-t',
'--type',
help='Puppetforge server type (puppet-forge-server|artifiactory)',
required=False)
parser.add_argument('-s',
'--server',
help='puppet-forge-server hostname',
required=False)
parser.add_argument('-m',
'--modulepath',
help='path to puppet-forge-server modules',
required=False)
parser.add_argument(
'-u',
'--user',
help=
'Username to push modules to server as (default is user running script)',
required=False)
parser.add_argument(
'-p',
'--password',
help='Password (token) for username to push modules to Artifactory',
required=False)
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
# Define the type of puppet-forge server
if args.type:
pftype = args.type
else:
if not helpers.PFMETHOD:
print "Puppet forge server type not specified"
sys.exit(1)
else:
pftype = helpers.PFMETHOD
# Define the puppet-forge-server hostname
if args.server:
pfserver = args.server
else:
if not helpers.PFSERVER:
print "Puppet forge server not defined"
sys.exit(1)
else:
pfserver = helpers.PFSERVER
# Set the remote (puppet-forge-server) modules directory
if args.modulepath:
modpath = args.modulepath
else:
if not helpers.PFMODPATH:
print "Puppet forge module path not defined"
sys.exit(1)
else:
modpath = helpers.PFMODPATH
# Set the username to use to push modules
if args.user:
pfuser = args.user
else:
pfuser = helpers.PFUSER
# Read in the token for Artifiactory
if args.password:
pftoken = args.password
else:
pftoken = helpers.PFTOKEN
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Read the repo label given by the user
if args.repo:
pfrepo = args.repo
else:
print "Puppetforge repo not defined"
sys.exit(1)
# Remove any previous exported content left behind by prior unclean exit
if os.path.exists(helpers.EXPORTDIR + '/export'):
msg = "Removing existing export directory"
helpers.log_msg(msg, 'DEBUG')
shutil.rmtree(helpers.EXPORTDIR + '/export')
# Collect a list of enabled repositories. This is needed for:
# 1. Matching specific repo exports, and
# 2. Running import sync per repo on the disconnected side
repolist = helpers.get_p_json(
helpers.KATELLO_API + "/repositories/", \
json.dumps(
{
"organization_id": org_id,
"per_page": '1000',
}
))
# Process each repo
for repo_result in repolist['results']:
if repo_result['content_type'] == 'puppet':
# If we have a match, do the export
if repo_result['label'] == pfrepo:
# Trigger export on the repo
numfiles = export_puppet(repo_result['id'],
repo_result['label'],
repo_result['relative_path'], 'full')
else:
msg = "Skipping " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
# Now we need to process the on-disk export data.
# Define the location of our exported data.
export_dir = helpers.EXPORTDIR + "/puppetforge"
if (pftype == 'puppet-forge-server'):
# Method for posting to puppet-forge-server
os.chdir(script_dir)
copy_to_pfserver(export_dir, pfserver, modpath, pfuser)
elif (pftype == 'artifactory'):
# Method for posting to Artifactory repository
for module in os.listdir(export_dir):
print("Posing: " + module)
postModule(module, export_dir, pfserver, modpath, pfuser, pftoken)
else:
print("Unknown puppet-forge server type defined")
sys.exit(1)
# And we're done!
print helpers.GREEN + "Puppet Forge export complete.\n" + helpers.ENDC
sys.exit(0)
def main(args):
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if not helpers.DISCONNECTED:
msg = "Import cannot be run on the connected Satellite (Sync) host"
helpers.log_msg(msg, 'ERROR')
sys.exit(1)
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Log the fact we are starting
msg = "------------- Content import started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(
description='Performs Import of Default Content View.')
# pylint: disable=bad-continuation
parser.add_argument('-o',
'--org',
help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-d', '--dataset', \
help='Date/name of Import dataset to process (YYYY-MM-DD_NAME)', required=False)
parser.add_argument('-n',
'--nosync',
help='Do not trigger a sync after extracting content',
required=False,
action="store_true")
parser.add_argument('-r',
'--remove',
help='Remove input files after import has completed',
required=False,
action="store_true")
parser.add_argument('-l',
'--last',
help='Display the last successful import performed',
required=False,
action="store_true")
parser.add_argument('-c',
'--count',
help='Display all package counts after import',
required=False,
action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dataset = args.dataset
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Display the last successful import
if args.last:
if os.path.exists(vardir + '/imports.pkl'):
last_import = pickle.load(open(vardir + '/imports.pkl', 'rb'))
msg = "Last successful import was " + last_import
helpers.log_msg(msg, 'INFO')
print msg
else:
msg = "Import has never been performed"
helpers.log_msg(msg, 'INFO')
print msg
sys.exit(0)
# If we got this far without -d being specified, error out cleanly
if args.dataset is None:
parser.error("--dataset is required")
# Figure out if we have the specified input fileset
basename = get_inputfiles(dataset)
# Cleanup from any previous imports
os.system("rm -rf " + helpers.IMPORTDIR +
"/{content,custom,listing,*.pkl}")
# Extract the input files
extract_content(basename)
# Trigger a sync of the content into the Library
if args.nosync:
#print helpers.GREEN + "Import complete.\n" + helpers.ENDC
msg = "Repository sync was requested to be skipped"
helpers.log_msg(msg, 'WARNING')
print 'Please synchronise all repositories to make new content available for publishing.'
delete_override = True
else:
# We need to figure out which repos to sync. This comes to us via a pickle containing
# a list of repositories that were exported
imported_repos = pickle.load(open('exported_repos.pkl', 'rb'))
package_count = pickle.load(open('package_count.pkl', 'rb'))
# Run a repo sync on each imported repo
(delete_override) = sync_content(org_id, imported_repos)
print helpers.GREEN + "Import complete.\n" + helpers.ENDC
print 'Please publish content views to make new content available.'
# Verify the repository package/erratum counts match the sync host
check_counts(org_id, package_count, args.count)
if os.path.exists(helpers.IMPORTDIR + '/puppetforge'):
print 'Offline puppet-forge-server bundle is available to import seperately in '\
+ helpers.IMPORTDIR + '/puppetforge\n'
if args.remove and not delete_override:
msg = "Removing input files from " + helpers.IMPORTDIR
helpers.log_msg(msg, 'INFO')
print msg
os.system("rm -f " + helpers.IMPORTDIR + "/sat6_export_" + dataset +
"*")
os.system("rm -rf " + helpers.IMPORTDIR +
"/{content,custom,listing,*.pkl}")
excode = 0
elif delete_override:
msg = "* Not removing input files due to incomplete sync *"
helpers.log_msg(msg, 'INFO')
print msg
excode = 2
else:
msg = " (Removal of input files was not requested)"
helpers.log_msg(msg, 'INFO')
print msg
excode = 0
msg = "Import Complete"
helpers.log_msg(msg, 'INFO')
# Save the last completed import data
os.chdir(script_dir)
if not os.path.exists(vardir):
os.makedirs(vardir)
pickle.dump(dataset, open(vardir + '/imports.pkl', "wb"))
# And exit.
sys.exit(excode)
def main():
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if helpers.DISCONNECTED:
msg = "Export cannot be run on the disconnected Satellite host"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
# Who is running this script?
runuser = helpers.who_is_running()
# Log the fact we are starting
msg = "------------- Puppet export started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(
description='Performs Export of Puppet modules.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization', required=True)
group.add_argument('-a',
'--all',
help='Export ALL puppet modules',
required=False,
action="store_true")
group.add_argument(
'-i',
'--incr',
help='Incremental Export of puppet modules since last run',
required=False,
action="store_true")
group.add_argument('-s',
'--since',
help='Export puppet modules since YYYY-MM-DD HH:MM:SS',
required=False,
type=helpers.valid_date)
parser.add_argument('-l',
'--last',
help='Display time of last export',
required=False,
action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
org_name = args.org
since = args.since
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the current time - this will be the 'last export' time if the export is OK
start_time = datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S')
print "START: " + start_time
# Get the last export date. If we're exporting all, this isn't relevant
# If we are given a start date, use that, otherwise we need to get the last date from file
# If there is no last export, we'll set an arbitrary start date to grab everything (2000-01-01)
last_export = read_timestamp()
export_type = 'incr'
if args.all:
print "Performing full puppet module export"
export_type = 'full'
else:
if not since:
if args.last:
if last_export:
print "Last successful export was started at " + last_export
else:
print "Export has never been performed"
sys.exit(-1)
if not last_export:
print "No previous export recorded, performing full puppet module export"
export_type = 'full'
else:
last_export = str(since)
# We have our timestamp so we can kick of an incremental export
print "Incremental export of puppet modules synchronised after " + last_export
# TODO: Remove any previous exported content
# os.chdir(helpers.EXPORTDIR)
# shutil.rmtree()
# Check if there are any currently running tasks that will conflict with an export
check_running_tasks()
# Now we have a CV ID and a starting date, and no conflicting tasks, we can export
export_puppet(last_export, export_type)
# Now we need to process the on-disk export data
# Find the name of our export dir. This ASSUMES that the export dir is the ONLY dir.
sat_export_dir = os.walk(helpers.EXPORTDIR).next()[1]
export_path = sat_export_dir[0]
# This portion finds the full directory tree of the Puppet repo, starting at the level
# containing the Org_Name (/var/lib/pulp/published/puppet/http/repos/<org_name>/...)
# pylint: disable=unused-variable
for dirpath, subdirs, files in os.walk(helpers.EXPORTDIR):
for tdir in subdirs:
if org_name in tdir:
export_dir = os.path.join(dirpath, tdir)
# Add our exported data to a tarfile
create_tar(export_dir, export_path)
# We're done. Write the start timestamp to file for next time
os.chdir(script_dir)
write_timestamp(start_time)
# And we're done!
print helpers.GREEN + "Puppet module export complete.\n" + helpers.ENDC
print 'Please transfer the contents of ' + helpers.EXPORTDIR + \
' to your disconnected puppet-forge server content location.\n'
def main(args):
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if not helpers.DISCONNECTED:
msg = "Import cannot be run on the connected Satellite (Sync) host"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Log the fact we are starting
msg = "------------- Content import started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(description='Performs Import of Default Content View.')
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-d', '--date', \
help='Date/name of Import fileset to process (YYYY-MM-DD_NAME)', required=False)
parser.add_argument('-n', '--nosync', help='Do not trigger a sync after extracting content',
required=False, action="store_true")
parser.add_argument('-r', '--remove', help='Remove input files after import has completed',
required=False, action="store_true")
parser.add_argument('-l', '--last', help='Display the last successful import performed',
required=False, action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
expdate = args.date
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Display the last successful import
if args.last:
if os.path.exists(vardir + '/imports.pkl'):
last_import = pickle.load(open(vardir + '/imports.pkl', 'rb'))
msg = "Last successful import was " + last_import
helpers.log_msg(msg, 'INFO')
print msg
else:
msg = "Import has never been performed"
helpers.log_msg(msg, 'INFO')
print msg
sys.exit(-1)
# If we got this far without -d being specified, error out cleanly
if args.date is None:
parser.error("--date is required")
# Figure out if we have the specified input fileset
basename = get_inputfiles(expdate)
# Cleanup from any previous imports
os.system("rm -rf " + helpers.IMPORTDIR + "/{content,custom,listing,*.pkl}")
# Extract the input files
extract_content(basename)
# Trigger a sync of the content into the Library
if args.nosync:
#print helpers.GREEN + "Import complete.\n" + helpers.ENDC
msg = "Repository sync was requested to be skipped"
helpers.log_msg(msg, 'WARNING')
print 'Please synchronise all repositories to make new content available for publishing.'
delete_override = True
else:
# We need to figure out which repos to sync. This comes to us via a pickle containing
# a list of repositories that were exported
imported_repos = pickle.load(open('exported_repos.pkl', 'rb'))
# Run a repo sync on each imported repo
(delete_override) = sync_content(org_id, imported_repos)
print helpers.GREEN + "Import complete.\n" + helpers.ENDC
print 'Please publish content views to make new content available.'
if args.remove and not delete_override:
msg = "Removing input files from " + helpers.IMPORTDIR
helpers.log_msg(msg, 'INFO')
print msg
os.system("rm -f " + helpers.IMPORTDIR + "/sat6_export_" + expdate + "*")
os.system("rm -rf " + helpers.IMPORTDIR + "/{content,custom,listing,*.pkl}")
elif delete_override:
msg = "* Not removing input files due to incomplete sync *"
helpers.log_msg(msg, 'INFO')
print msg
else:
msg = " (Removal of input files was not requested)"
helpers.log_msg(msg, 'INFO')
print msg
msg = "Import Complete"
helpers.log_msg(msg, 'INFO')
# Save the last completed import data
os.chdir(script_dir)
if not os.path.exists(vardir):
os.makedirs(vardir)
pickle.dump(expdate, open(vardir + '/imports.pkl', "wb"))
def main():
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if helpers.DISCONNECTED:
msg = "Export cannot be run on the disconnected Satellite host"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
# Who is running this script?
runuser = helpers.who_is_running()
# Log the fact we are starting
msg = "------------- Content export started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(description='Performs Export of Default Content View.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization', required=True)
parser.add_argument('-e', '--env', help='Environment config file', required=False)
group.add_argument('-a', '--all', help='Export ALL content', required=False,
action="store_true")
group.add_argument('-i', '--incr', help='Incremental Export of content since last run',
required=False, action="store_true")
group.add_argument('-s', '--since', help='Export content since YYYY-MM-DD HH:MM:SS',
required=False, type=helpers.valid_date)
parser.add_argument('-l', '--last', help='Display time of last export', required=False,
action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
org_name = args.org
since = args.since
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
exported_repos = []
# If a specific environment is requested, find and read that config file
if args.env:
if not os.path.exists('config/' + args.env + '.yml'):
print "ERROR: Config file 'config/" + args.env + ".yml' not found."
sys.exit(-1)
cfg = yaml.safe_load(open("config/" + args.env + ".yml", 'r'))
ename = args.env
erepos = cfg["env"]["repos"]
msg = "Specific environment export called for " + ename + ". Configured repos:"
helpers.log_msg(msg, 'DEBUG')
for repo in erepos:
msg = " - " + repo
helpers.log_msg(msg, 'DEBUG')
else:
ename = 'DoV'
label = 'DoV'
msg = "DoV export called"
helpers.log_msg(msg, 'DEBUG')
# Get the current time - this will be the 'last export' time if the export is OK
start_time = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')
print "START: " + start_time + " (" + ename + " export)"
# Read the last export date pickle for our selected repo group.
export_times = read_pickle(ename)
export_type = 'incr'
if args.all:
print "Performing full content export for " + ename
export_type = 'full'
since = False
else:
if not since:
since = False
if args.last:
if export_times:
print "Last successful export for " + ename + ":"
for time in export_times:
print str(time) + '\t' + str(export_times[time])
else:
print "Export has never been performed for " + ename
sys.exit(-1)
if not export_times:
print "No prior export recorded for " + ename + ", performing full content export"
export_type = 'full'
else:
# TODO: Re-populate export_times dictionary so each repo has 'since' date
since = True
since_export = str(since)
# We have our timestamp so we can kick of an incremental export
print "Incremental export of content for " + ename + " synchronised after " \
+ str(since)
# Check the available space in /var/lib/pulp
check_disk_space(export_type)
# TODO: Remove any previous exported content
# os.chdir(helpers.EXPORTDIR)
# shutil.rmtree()
# Collect a list of enabled repositories. This is needed for:
# 1. Matching specific repo exports, and
# 2. Running import sync per repo on the disconnected side
repolist = helpers.get_p_json(
helpers.KATELLO_API + "/repositories/", \
json.dumps(
{
"organization_id": org_id,
"per_page": '1000',
}
))
# If we are running a full DoV export we run a different set of API calls...
if ename == 'DoV':
if export_type == 'incr' and 'DoV' in export_times:
last_export = export_times['DoV']
if since:
last_export = since_export
msg = "Exporting DoV (INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
msg = "Exporting DoV (FULL)"
helpers.log_msg(msg, 'INFO')
print msg
# Check if there are any currently running tasks that will conflict with an export
check_running_tasks(label, ename)
# Get the version of the CV (Default Org View) to export
dov_ver = get_cv(org_id)
# Now we have a CV ID and a starting date, and no conflicting tasks, we can export
export_id = export_cv(dov_ver, last_export, export_type)
# Now we need to wait for the export to complete
helpers.wait_for_task(export_id, 'export')
# Check if the export completed OK. If not we exit the script.
tinfo = helpers.get_task_status(export_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Content View Export OK"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
# Update the export timestamp for this repo
export_times['DoV'] = start_time
# Generate a list of repositories that were exported
for repo_result in repolist['results']:
if repo_result['content_type'] == 'yum':
# Add the repo to the successfully exported list
exported_repos.append(repo_result['label'])
else:
msg = "Content View Export FAILED"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
else:
# Verify that defined repos exist in our DoV
for repo_result in repolist['results']:
if repo_result['content_type'] == 'yum':
# If we have a match, do the export
if repo_result['label'] in erepos:
# Extract the last export time for this repo
if export_type == 'incr' and repo_result['label'] in export_times:
last_export = export_times[repo_result['label']]
if since:
last_export = since_export
msg = "Exporting " + repo_result['label'] \
+ " (INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
msg = "Exporting " + repo_result['label'] + "(FULL)"
helpers.log_msg(msg, 'INFO')
print msg
# Check if there are any currently running tasks that will conflict
ok_to_export = check_running_tasks(repo_result['label'], ename)
if ok_to_export:
# Trigger export on the repo
export_id = export_repo(repo_result['id'], last_export, export_type)
# Now we need to wait for the export to complete
helpers.wait_for_task(export_id, 'export')
# Check if the export completed OK. If not we exit the script.
tinfo = helpers.get_task_status(export_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Repository Export OK"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
# Update the export timestamp for this repo
export_times[repo_result['label']] = start_time
# Add the repo to the successfully exported list
exported_repos.append(repo_result['label'])
else:
msg = "Export FAILED"
helpers.log_msg(msg, 'ERROR')
else:
msg = "Skipping " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
# Combine resulting directory structures into a single repo format (top level = /content)
prep_export_tree(org_name)
# Now we need to process the on-disk export data.
# Define the location of our exported data.
export_dir = helpers.EXPORTDIR + "/export"
# Write out the list of exported repos. This will be transferred to the disconnected system
# and used to perform the repo sync tasks during the import.
pickle.dump(exported_repos, open(export_dir + '/exported_repos.pkl', 'wb'))
# Run GPG Checks on the exported RPMs
do_gpg_check(export_dir)
# Add our exported data to a tarfile
create_tar(export_dir, ename)
# We're done. Write the start timestamp to file for next time
os.chdir(script_dir)
pickle.dump(export_times, open('var/exports_' + ename + '.pkl', "wb"))
# And we're done!
print helpers.GREEN + "Export complete.\n" + helpers.ENDC
print 'Please transfer the contents of ' + helpers.EXPORTDIR + \
'to your disconnected Satellite system content import location.\n' \
'Once transferred, please run ' + helpers.BOLD + ' sat_import' \
+ helpers.ENDC + ' to extract it.'
def main():
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if helpers.DISCONNECTED:
msg = "Export cannot be run on the disconnected Satellite host"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
# Who is running this script?
runuser = helpers.who_is_running()
# Log the fact we are starting
msg = "------------- Puppet export started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(description='Performs Export of Puppet modules.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization', required=True)
group.add_argument('-a', '--all', help='Export ALL puppet modules', required=False,
action="store_true")
group.add_argument('-i', '--incr', help='Incremental Export of puppet modules since last run',
required=False, action="store_true")
group.add_argument('-s', '--since', help='Export puppet modules since YYYY-MM-DD HH:MM:SS',
required=False, type=helpers.valid_date)
parser.add_argument('-l', '--last', help='Display time of last export', required=False,
action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
org_name = args.org
since = args.since
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the current time - this will be the 'last export' time if the export is OK
start_time = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')
print "START: " + start_time
# Get the last export date. If we're exporting all, this isn't relevant
# If we are given a start date, use that, otherwise we need to get the last date from file
# If there is no last export, we'll set an arbitrary start date to grab everything (2000-01-01)
last_export = read_timestamp()
export_type = 'incr'
if args.all:
print "Performing full puppet module export"
export_type = 'full'
else:
if not since:
if args.last:
if last_export:
print "Last successful export was started at " + last_export
else:
print "Export has never been performed"
sys.exit(-1)
if not last_export:
print "No previous export recorded, performing full puppet module export"
export_type = 'full'
else:
last_export = str(since)
# We have our timestamp so we can kick of an incremental export
print "Incremental export of puppet modules synchronised after " + last_export
# TODO: Remove any previous exported content
# os.chdir(helpers.EXPORTDIR)
# shutil.rmtree()
# Check if there are any currently running tasks that will conflict with an export
check_running_tasks()
# Now we have a CV ID and a starting date, and no conflicting tasks, we can export
export_puppet(last_export, export_type)
# Now we need to process the on-disk export data
# Find the name of our export dir. This ASSUMES that the export dir is the ONLY dir.
sat_export_dir = os.walk(helpers.EXPORTDIR).next()[1]
export_path = sat_export_dir[0]
# This portion finds the full directory tree of the Puppet repo, starting at the level
# containing the Org_Name (/var/lib/pulp/published/puppet/http/repos/<org_name>/...)
# pylint: disable=unused-variable
for dirpath, subdirs, files in os.walk(helpers.EXPORTDIR):
for tdir in subdirs:
if org_name in tdir:
export_dir = os.path.join(dirpath, tdir)
# Add our exported data to a tarfile
create_tar(export_dir, export_path)
# We're done. Write the start timestamp to file for next time
os.chdir(script_dir)
write_timestamp(start_time)
# And we're done!
print helpers.GREEN + "Puppet module export complete.\n" + helpers.ENDC
print 'Please transfer the contents of ' + helpers.EXPORTDIR + \
' to your disconnected puppet-forge server content location.\n'
def main(args):
"""
Main routine
"""
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Check for sane input
parser = argparse.ArgumentParser(
description='Promotes content views for specified organization to the target environment.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-e', '--env', help='Target Environment (e.g. Development, Quality, Production)',
required=False)
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
group.add_argument('-a', '--all', help='Promote ALL content views', required=False,
action="store_true")
parser.add_argument('-d', '--dryrun', help='Dry Run - Only show what will be promoted',
required=False, action="store_true")
parser.add_argument('-l', '--last', help='Display last promotions', required=False,
action="store_true")
parser.add_argument('-q', '--quiet', help="Suppress progress output updates", required=False,
action="store_true")
parser.add_argument('-m', '--forcemeta', help="Force metadata regeneration", required=False,
action="store_true")
args = parser.parse_args()
# Log the fact we are starting
msg = "-------- Content view promotion started by " + runuser + " -----------"
helpers.log_msg(msg, 'INFO')
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
target_env = args.env
dry_run = args.dryrun
# Load the promotion history
if not os.path.exists(vardir + '/promotions.pkl'):
if not os.path.exists(vardir):
os.makedirs(vardir)
phistory = {}
else:
phistory = pickle.load(open(vardir + '/promotions.pkl', 'rb'))
# Read the promotion history if --last requested
if args.last:
if phistory:
print 'Last promotions:'
for lenv, time in phistory.iteritems():
print lenv, time
else:
print 'No promotions recorded'
sys.exit(0)
# Error if no environment to promote to is given
if args.env is None:
parser.error('--env is required')
promote_list = []
if not args.all:
for x in helpers.CONFIG['promotion']:
if x == 'batch':
continue
if helpers.CONFIG['promotion'][x]['name'] == target_env:
promote_list = helpers.CONFIG['promotion'][x]['content_views']
if not promote_list:
msg = "Cannot find promotion configuration for '" + target_env + "'"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FP, output)
sys.exit(1)
msg = "Config found for CV's " + str(promote_list)
helpers.log_msg(msg, 'DEBUG')
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Now, let's fetch all available lifecycle environments for this org...
(env_list, prior_list) = get_envs(org_id)
# Get the list of Content Views along with the latest view version in each environment
(ver_list, ver_descr, ver_version) = get_cv(org_id, target_env, env_list, prior_list,
promote_list)
# Promote to the given environment. Returns a list of task IDs.
promote(target_env, ver_list, ver_descr, ver_version, env_list, prior_list, dry_run,
args.quiet, args.forcemeta)
# Add/Update the promotion history dictionary so we can check when we last promoted
phistory[target_env] = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
pickle.dump(phistory, open(vardir + '/promotions.pkl', 'wb'))
# Run the mailout
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
message = "Promotion completed successfully\n\n" + output
subject = "Satellite 6 promotion completed"
helpers.mailout(subject, message)
# Exit cleanly
sys.exit(0)
def main(args):
"""
Main routine
"""
# Who is running this script?
runuser = helpers.who_is_running()
# Check for sane input
parser = argparse.ArgumentParser(
description='export GPG keys to defined directory')
# pylint: disable=bad-continuation
parser.add_argument('-o',
'--org',
help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-d',
'--dryrun',
help='Dry Run - Only show GPG keys',
required=False,
action="store_true")
parser.add_argument('-t',
'--target',
help='Define target director for keys',
required=False)
parser.add_argument('-p',
'--plain',
help='No faked directory structure for satellite',
required=False,
action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
if args.plain:
plain = args.plain
else:
plain = False
dry_run = args.dryrun
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the list of Content Views along with the latest view version in each environment
gpg_result = get_gpg(org_id)
# store GPG keys to given export dir
if not dry_run:
if args.target:
targetdir = args.target
store_gpg(gpg_result, targetdir, plain)
else:
parser.print_help()
else:
print json.dumps(gpg_result, indent=4, sort_keys=False)
# Exit cleanly
sys.exit(0)
def main(args):
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if helpers.DISCONNECTED:
msg = "Export cannot be run on the disconnected Satellite host"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Log the fact we are starting
msg = "------------- Content export started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(description='Performs Export of Default Content View.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-e', '--env', help='Environment config file', required=False)
group.add_argument('-a', '--all', help='Export ALL content', required=False,
action="store_true")
group.add_argument('-i', '--incr', help='Incremental Export of content since last run',
required=False, action="store_true")
group.add_argument('-s', '--since', help='Export content since YYYY-MM-DD HH:MM:SS',
required=False, type=helpers.valid_date)
parser.add_argument('-l', '--last', help='Display time of last export', required=False,
action="store_true")
parser.add_argument('-n', '--nogpg', help='Skip GPG checking', required=False,
action="store_true")
parser.add_argument('-r', '--repodata', help='Include repodata for repos with no new packages',
required=False, action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
since = args.since
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
exported_repos = []
# If a specific environment is requested, find and read that config file
repocfg = os.path.join(dir, 'config/' + args.env + '.yml')
if args.env:
if not os.path.exists(repocfg):
print "ERROR: Config file " + repocfg + " not found."
sys.exit(-1)
cfg = yaml.safe_load(open(repocfg, 'r'))
ename = args.env
erepos = cfg["env"]["repos"]
msg = "Specific environment export called for " + ename + ". Configured repos:"
helpers.log_msg(msg, 'DEBUG')
for repo in erepos:
msg = " - " + repo
helpers.log_msg(msg, 'DEBUG')
else:
ename = 'DoV'
label = 'DoV'
msg = "DoV export called"
helpers.log_msg(msg, 'DEBUG')
# Get the current time - this will be the 'last export' time if the export is OK
start_time = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')
print "START: " + start_time + " (" + ename + " export)"
# Read the last export date pickle for our selected repo group.
export_times = read_pickle(ename)
export_type = 'incr'
if args.all:
print "Performing full content export for " + ename
export_type = 'full'
since = False
else:
if not since:
since = False
if args.last:
if export_times:
print "Last successful export for " + ename + ":"
for time in export_times:
repo = "{:<70}".format(time)
print repo[:70] + '\t' + str(export_times[time])
else:
print "Export has never been performed for " + ename
sys.exit(-1)
if not export_times:
print "No prior export recorded for " + ename + ", performing full content export"
export_type = 'full'
else:
# Re-populate export_times dictionary so each repo has 'since' date
since_export = str(since)
# We have our timestamp so we can kick of an incremental export
print "Incremental export of content for " + ename + " synchronised after " \
+ str(since)
# Check the available space in /var/lib/pulp
check_disk_space(export_type)
# Remove any previous exported content left behind by prior unclean exit
if os.path.exists(helpers.EXPORTDIR + '/export'):
msg = "Removing existing export directory"
helpers.log_msg(msg, 'DEBUG')
shutil.rmtree(helpers.EXPORTDIR + '/export')
# Collect a list of enabled repositories. This is needed for:
# 1. Matching specific repo exports, and
# 2. Running import sync per repo on the disconnected side
repolist = helpers.get_p_json(
helpers.KATELLO_API + "/repositories/", \
json.dumps(
{
"organization_id": org_id,
"per_page": '1000',
}
))
# If we are running a full DoV export we run a different set of API calls...
if ename == 'DoV':
cola = "Exporting DoV"
if export_type == 'incr' and 'DoV' in export_times:
last_export = export_times['DoV']
if since:
last_export = since_export
colb = "(INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
colb = "(FULL)"
msg = cola + " " + colb
helpers.log_msg(msg, 'INFO')
output = "{:<70}".format(cola)
print output[:70] + ' ' + colb
# Check if there are any currently running tasks that will conflict with an export
check_running_tasks(label, ename)
# Get the version of the CV (Default Org View) to export
dov_ver = get_cv(org_id)
# Now we have a CV ID and a starting date, and no conflicting tasks, we can export
export_id = export_cv(dov_ver, last_export, export_type)
# Now we need to wait for the export to complete
helpers.wait_for_task(export_id, 'export')
# Check if the export completed OK. If not we exit the script.
tinfo = helpers.get_task_status(export_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
msg = "Content View Export OK"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
# Update the export timestamp for this repo
export_times['DoV'] = start_time
# Generate a list of repositories that were exported
for repo_result in repolist['results']:
if repo_result['content_type'] == 'yum':
# Add the repo to the successfully exported list
exported_repos.append(repo_result['label'])
else:
msg = "Content View Export FAILED"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
else:
# Verify that defined repos exist in Satellite
for repo in erepos:
repo_in_sat = False
for repo_x in repolist['results']:
if re.findall("\\b" + repo + "\\b$", repo_x['label']):
repo_in_sat = True
break
if repo_in_sat == False:
msg = "'" + repo + "' not found in Satellite"
helpers.log_msg(msg, 'WARNING')
# Process each repo
for repo_result in repolist['results']:
if repo_result['content_type'] == 'yum':
# If we have a match, do the export
if repo_result['label'] in erepos:
# Extract the last export time for this repo
orig_export_type = export_type
cola = "Export " + repo_result['label']
if export_type == 'incr' and repo_result['label'] in export_times:
last_export = export_times[repo_result['label']]
if since:
last_export = since_export
colb = "(INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
colb = "(FULL)"
msg = cola + " " + colb
helpers.log_msg(msg, 'INFO')
output = "{:<70}".format(cola)
print output[:70] + ' ' + colb
# Check if there are any currently running tasks that will conflict
ok_to_export = check_running_tasks(repo_result['label'], ename)
if ok_to_export:
# Trigger export on the repo
export_id = export_repo(repo_result['id'], last_export, export_type)
# Now we need to wait for the export to complete
helpers.wait_for_task(export_id, 'export')
# Check if the export completed OK. If not we exit the script.
tinfo = helpers.get_task_status(export_id)
if tinfo['state'] != 'running' and tinfo['result'] == 'success':
# Count the number of exported packages
# First resolve the product label - this forms part of the export path
product = get_product(org_id, repo_result['product']['cp_id'])
# Now we can build the export path itself
basepath = helpers.EXPORTDIR + "/" + org_name + "-" + product + "-" + repo_result['label']
if export_type == 'incr':
basepath = basepath + "-incremental"
exportpath = basepath + "/" + repo_result['relative_path']
msg = "\nExport path = " + exportpath
helpers.log_msg(msg, 'DEBUG')
os.chdir(exportpath)
numrpms = len([f for f in os.walk(".").next()[2] if f[ -4: ] == ".rpm"])
msg = "Repository Export OK (" + str(numrpms) + " new packages)"
helpers.log_msg(msg, 'INFO')
print helpers.GREEN + msg + helpers.ENDC
# Update the export timestamp for this repo
export_times[repo_result['label']] = start_time
# Add the repo to the successfully exported list
if numrpms != 0 or args.repodata:
msg = "Adding " + repo_result['label'] + " to export list"
helpers.log_msg(msg, 'DEBUG')
exported_repos.append(repo_result['label'])
else:
msg = "Not including repodata for empty repo " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
else:
msg = "Export FAILED"
helpers.log_msg(msg, 'ERROR')
# Reset the export type to the user specified, in case we overrode it.
export_type = orig_export_type
else:
msg = "Skipping " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
# Handle FILE type exports (ISO repos)
elif repo_result['content_type'] == 'file':
# If we have a match, do the export
if repo_result['label'] in erepos:
# Extract the last export time for this repo
orig_export_type = export_type
cola = "Export " + repo_result['label']
if export_type == 'incr' and repo_result['label'] in export_times:
last_export = export_times[repo_result['label']]
if since:
last_export = since_export
colb = "(INCR since " + last_export + ")"
else:
export_type = 'full'
last_export = '2000-01-01 12:00:00' # This is a dummy value, never used.
colb = "(FULL)"
msg = cola + " " + colb
helpers.log_msg(msg, 'INFO')
output = "{:<70}".format(cola)
print output[:70] + ' ' + colb
# Check if there are any currently running tasks that will conflict
ok_to_export = check_running_tasks(repo_result['label'], ename)
if ok_to_export:
# Trigger export on the repo
numfiles = export_iso(repo_result['id'], repo_result['label'], repo_result['relative_path'], last_export, export_type)
# Reset the export type to the user specified, in case we overrode it.
export_type = orig_export_type
# Update the export timestamp for this repo
export_times[repo_result['label']] = start_time
# Add the repo to the successfully exported list
if numfiles != 0 or args.repodata:
msg = "Adding " + repo_result['label'] + " to export list"
helpers.log_msg(msg, 'DEBUG')
exported_repos.append(repo_result['label'])
else:
msg = "Not including repodata for empty repo " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
else:
msg = "Skipping " + repo_result['label']
helpers.log_msg(msg, 'DEBUG')
# Combine resulting directory structures into a single repo format (top level = /content)
prep_export_tree(org_name)
# Now we need to process the on-disk export data.
# Define the location of our exported data.
export_dir = helpers.EXPORTDIR + "/export"
# Write out the list of exported repos. This will be transferred to the disconnected system
# and used to perform the repo sync tasks during the import.
pickle.dump(exported_repos, open(export_dir + '/exported_repos.pkl', 'wb'))
# Run GPG Checks on the exported RPMs
if not args.nogpg:
do_gpg_check(export_dir)
# Add our exported data to a tarfile
create_tar(export_dir, ename)
# We're done. Write the start timestamp to file for next time
os.chdir(script_dir)
pickle.dump(export_times, open(vardir + '/exports_' + ename + '.pkl', "wb"))
# And we're done!
print helpers.GREEN + "Export complete.\n" + helpers.ENDC
print 'Please transfer the contents of ' + helpers.EXPORTDIR + \
' to your disconnected Satellite system content import location.\n' \
'Once transferred, please run ' + helpers.BOLD + ' sat_import' \
+ helpers.ENDC + ' to extract it.'
def main():
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if not helpers.DISCONNECTED:
msg = "Import cannot be run on the connected Satellite (Sync) host"
helpers.log_msg(msg, 'ERROR')
sys.exit(-1)
# Who is running this script?
runuser = helpers.who_is_running()
# Log the fact we are starting
msg = "------------- Content import started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(description='Performs Import of Default Content View.')
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization', required=True)
parser.add_argument('-d', '--date', \
help='Date/name of Import fileset to process (YYYY-MM-DD_NAME)', required=True)
parser.add_argument('-n', '--nosync', help='Do not trigger a sync after extracting content',
required=False, action="store_true")
parser.add_argument('-r', '--remove', help='Remove input files after import has completed',
required=False, action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
org_name = args.org
expdate = args.date
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Figure out if we have the specified input fileset
basename = get_inputfiles(expdate)
# Cleanup from any previous imports
os.system("rm -rf " + helpers.IMPORTDIR + "/{content,custom,listing,*.pkl}")
# Extract the input files
extract_content(basename)
# Trigger a sync of the content into the Library
if args.nosync:
print helpers.GREEN + "Import complete.\n" + helpers.ENDC
msg = "Repository sync was requested to be skipped"
helpers.log_msg(msg, 'WARNING')
print 'Please synchronise all repositories to make new content available for publishing.'
delete_override = False
else:
# We need to figure out which repos to sync. This comes to us via a pickle containing
# a list of repositories that were exported
imported_repos = pickle.load(open('exported_repos.pkl', 'rb'))
# Run a repo sync on each imported repo
(task_id, delete_override) = sync_content(org_id, imported_repos)
# Now we need to wait for the sync to complete
helpers.wait_for_task(task_id, 'sync')
print helpers.GREEN + "Import complete.\n" + helpers.ENDC
print 'Please publish content views to make new content available.'
if args.remove and not delete_override:
msg = "Removing " + helpers.IMPORTDIR + "/sat6_export_" + expdate + "* input files"
helpers.log_msg(msg, 'DEBUG')
# os.system("rm -f " + helpers.IMPORTDIR + "/sat6_export_" + expdate) + "*"
msg = "Import Complete"
helpers.log_msg(msg, 'INFO')
def main(args):
"""Promote Content Views from the previous lifecycle environment."""
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
confdir = os.path.join(dir, 'config')
# Check for sane input
parser = argparse.ArgumentParser(
description='Publishes content views for specified organization.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
group.add_argument('-a', '--all', help='Publish ALL content views', required=False,
action="store_true")
parser.add_argument('-d', '--dryrun', help='Dry Run - Only show what will be published',
required=False, action="store_true")
parser.add_argument('-l', '--last', help='Display last promotions', required=False,
action="store_true")
parser.add_argument('-c', '--comment', help="Add a custom description", required=False)
parser.add_argument('-q', '--quiet', help="Suppress progress output updates", required=False,
action="store_true")
parser.add_argument('-m', '--forcemeta', help="Force metadata regeneration", required=False,
action="store_true")
args = parser.parse_args()
# Log the fact we are starting
if not args.last:
msg = "-------- Content view publish started by " + runuser + " -----------"
helpers.log_msg(msg, 'INFO')
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dry_run = args.dryrun
# Set up the description that will be added to the published version
if args.comment:
description = args.comment
else:
description = "Published by " + runuser + "\n via API script"
# Load the promotion history
if not os.path.exists(vardir + '/promotions.pkl'):
if not os.path.exists(vardir):
os.makedirs(vardir)
phistory = {}
else:
phistory = pickle.load(open(vardir + '/promotions.pkl', 'rb'))
# Read the promotion history if --last requested
if args.last:
if phistory:
print 'Last promotions:'
for lenv, time in phistory.iteritems():
print lenv, time
else:
print 'No promotions recorded'
sys.exit(0)
publish_list = []
if not args.all:
publish_list = helpers.CONFIG['publish']['content_views']
if not publish_list:
msg = "Cannot find publish configuration"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FP, output)
sys.exit(1)
msg = "Config found for CV's " + str(publish_list)
helpers.log_msg(msg, 'DEBUG')
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the list of Content Views along with the latest view version in each environment
(ver_list, ver_descr, ver_version) = get_cv(org_id, publish_list)
# Publish the content views. Returns a list of task IDs.
publish(ver_list, ver_descr, ver_version, dry_run, runuser, description, args.quiet,
args.forcemeta)
# Add/Update the promotion history dictionary so we can check when we last promoted
phistory['Library'] = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
pickle.dump(phistory, open(vardir + '/promotions.pkl', 'wb'))
# Run the mailout
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
message = "Publish completed successfully\n\n" + output
subject = "Satellite 6 publish completed"
helpers.mailout(subject, message)
# Exit cleanly
sys.exit(0)
def main(args):
"""
Main routine
"""
# Who is running this script?
runuser = helpers.who_is_running()
# Check for sane input
parser = argparse.ArgumentParser(
description='Cleans content views for specified organization.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-k', '--keep', help='How many old versions to keep (only used with -a)',
required=False)
group.add_argument('-a', '--all', help='Clean ALL content views', required=False,
action="store_true")
parser.add_argument('-c', '--cleanall', help='Remove orphan versions between in-use views',
required=False, action="store_true")
parser.add_argument('-i', '--ignorefirstpromoted', help='Version to keep count starts from first CV, not first promoted CV',
required=False, action="store_true")
parser.add_argument('-d', '--dryrun', help='Dry Run - Only show what will be cleaned',
required=False, action="store_true")
args = parser.parse_args()
# Log the fact we are starting
msg = "-------- Content view cleanup started by " + runuser + " -----------"
helpers.log_msg(msg, 'INFO')
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dry_run = args.dryrun
cleanall = args.cleanall
ignorefirstpromoted = args.ignorefirstpromoted
if args.keep:
keep = args.keep
else:
keep = "0"
cleanup_list = []
if not args.all:
cleanup_list = helpers.CONFIG['cleanup']['content_views']
if not cleanup_list:
msg = "Cannot find cleanup configuration"
helpers.log_msg(msg, 'ERROR')
sys.exit(1)
msg = "Config found for CV's " + str(cleanup_list)
helpers.log_msg(msg, 'DEBUG')
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the list of Content Views along with the latest view version in each environment
(ver_list, ver_descr, ver_keep) = get_cv(org_id, cleanup_list, keep)
# Clean the content views. Returns a list of task IDs.
cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall, ignorefirstpromoted)
# Exit cleanly
sys.exit(0)
def main(args):
"""Perform import of Default Content View."""
#pylint: disable-msg=R0912,R0914,R0915
if not helpers.DISCONNECTED:
msg = "Import cannot be run on the connected Satellite (Sync) host"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FI, output)
sys.exit(1)
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Log the fact we are starting
msg = "------------- Content import started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(
description='Performs Import of Default Content View.')
# pylint: disable=bad-continuation
parser.add_argument('-o',
'--org',
help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-d', '--dataset', \
help='Date/name of Import dataset to process (YYYY-MM-DD_NAME)', required=False)
parser.add_argument('-n',
'--nosync',
help='Do not trigger a sync after extracting content',
required=False,
action="store_true")
parser.add_argument('-r',
'--remove',
help='Remove input files after import has completed',
required=False,
action="store_true")
parser.add_argument('-l',
'--last',
help='Display the last successful import performed',
required=False,
action="store_true")
parser.add_argument('-L',
'--list',
help='List all successfully completed imports',
required=False,
action="store_true")
parser.add_argument('-c',
'--count',
help='Display all package counts after import',
required=False,
action="store_true")
parser.add_argument(
'-f',
'--force',
help='Force import of data if it has previously been done',
required=False,
action="store_true")
parser.add_argument(
'-u',
'--unattended',
help='Answer any prompts safely, allowing automated usage',
required=False,
action="store_true")
parser.add_argument('--fixhistory',
help='Force import history to match export history',
required=False,
action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dataset = args.dataset
if args.fixhistory:
fixhistory = True
else:
fixhistory = False
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
imports = []
# Read the last imports data
if os.path.exists(vardir + '/imports.pkl'):
imports = pickle.load(open(vardir + '/imports.pkl', 'rb'))
# If we have a string we convert to a list. This should only occur the first time we
# migrate from the original string version of the pickle.
if type(imports) is str:
imports = imports.split()
last_import = imports[-1]
# Display the last successful import(s)
if args.last or args.list:
if os.path.exists(vardir + '/imports.pkl'):
if args.last:
msg = "Last successful import was " + last_import
helpers.log_msg(msg, 'INFO')
print msg
if args.list:
print "Completed imports:\n----------------"
for item in imports:
print item
else:
msg = "Import has never been performed"
helpers.log_msg(msg, 'INFO')
print msg
sys.exit(0)
# If we got this far without -d being specified, error out cleanly
if args.dataset is None:
parser.error("--dataset is required")
# If we have already imported this dataset let the user know
if dataset in imports:
if not args.force:
msg = "Dataset " + dataset + " has already been imported. Use --force if you really want to do this."
helpers.log_msg(msg, 'WARNING')
sys.exit(2)
# Figure out if we have the specified input fileset
basename = get_inputfiles(dataset)
# Cleanup from any previous imports
os.system("rm -rf " + helpers.IMPORTDIR +
"/{content,custom,listing,*.pkl}")
# Extract the input files
extract_content(basename)
# Read in the export history from the input dataset
dsname = dataset.split('_')[1]
exports = pickle.load(
open(helpers.IMPORTDIR + '/exporthistory_' + dsname + '.pkl', 'rb'))
# Check for and let the user decide if they want to continue with missing imports
missing_imports = check_missing(imports, exports, dataset, fixhistory,
vardir)
if missing_imports:
msg = "Run sat_import with the --fixhistory flag to reset the import history to this export"
helpers.log_msg(msg, 'INFO')
if not args.unattended:
answer = helpers.query_yes_no("Continue with import?", "no")
if not answer:
msg = "Import Aborted"
helpers.log_msg(msg, 'ERROR')
sys.exit(3)
else:
msg = "Import continued by user"
helpers.log_msg(msg, 'INFO')
else:
msg = "Import Aborted"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FI, output)
sys.exit(3)
# Trigger a sync of the content into the Library
if args.nosync:
#print helpers.GREEN + "Import complete.\n" + helpers.ENDC
msg = "Repository sync was requested to be skipped"
helpers.log_msg(msg, 'WARNING')
print 'Please synchronise all repositories to make new content available for publishing.'
delete_override = True
else:
# We need to figure out which repos to sync. This comes to us via a pickle containing
# a list of repositories that were exported
imported_repos = pickle.load(open('exported_repos.pkl', 'rb'))
package_count = pickle.load(open('package_count.pkl', 'rb'))
# Run a repo sync on each imported repo
(delete_override, newrepos) = sync_content(org_id, imported_repos)
print helpers.GREEN + "Import complete.\n" + helpers.ENDC
print 'Please publish content views to make new content available.'
# Verify the repository package/erratum counts match the sync host
check_counts(org_id, package_count, args.count)
if os.path.exists(helpers.IMPORTDIR + '/puppetforge'):
print 'Offline puppet-forge-server bundle is available to import seperately in '\
+ helpers.IMPORTDIR + '/puppetforge\n'
if args.remove and not delete_override:
msg = "Removing input files from " + helpers.IMPORTDIR
helpers.log_msg(msg, 'INFO')
print msg
os.system("rm -f " + helpers.IMPORTDIR + "/sat6_export_" + dataset +
"*")
os.system("rm -rf " + helpers.IMPORTDIR +
"/{content,custom,listing,*.pkl}")
excode = 0
elif delete_override:
msg = "* Not removing input files due to incomplete sync *"
helpers.log_msg(msg, 'INFO')
print msg
excode = 2
else:
msg = " (Removal of input files was not requested)"
helpers.log_msg(msg, 'INFO')
print msg
excode = 0
msg = "Import Complete"
helpers.log_msg(msg, 'INFO')
# Save the last completed import data (append to existing pickle)
os.chdir(script_dir)
if not os.path.exists(vardir):
os.makedirs(vardir)
imports.append(dataset)
pickle.dump(imports, open(vardir + '/imports.pkl', "wb"))
# Run the mailout
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
if missing_imports:
message = "Import of dataset " + dataset + " completed successfully.\n\n \
Missing datasets were detected during the import - please check the logs\n\n" + output
subject = "Satellite 6 import completed: Missing datasets"
elif newrepos:
message = "Import of dataset " + dataset + " completed successfully.\n\n \
New repos found that need to be imported manually - please check the logs \n\n" + output
subject = "Satellite 6 import completed: New repos require manual intervention"
else:
message = "Import of dataset " + dataset + " completed successfully\n\n" + output
subject = "Satellite 6 import completed"
helpers.mailout(subject, message)
# And exit.
sys.exit(excode)
def main(args):
"""
Main routine
"""
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Check for sane input
parser = argparse.ArgumentParser(
description='Promotes content views for specified organization to the target environment.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-e', '--env', help='Target Environment (e.g. Development, Quality, Production)',
required=False)
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
group.add_argument('-a', '--all', help='Promote ALL content views', required=False,
action="store_true")
parser.add_argument('-d', '--dryrun', help='Dry Run - Only show what will be promoted',
required=False, action="store_true")
parser.add_argument('-l', '--last', help='Display last promotions', required=False,
action="store_true")
parser.add_argument('-q', '--quiet', help="Suppress progress output updates", required=False,
action="store_true")
args = parser.parse_args()
# Log the fact we are starting
msg = "-------- Content view promotion started by " + runuser + " -----------"
helpers.log_msg(msg, 'INFO')
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
target_env = args.env
dry_run = args.dryrun
# Load the promotion history
if not os.path.exists(vardir + '/promotions.pkl'):
if not os.path.exists(vardir):
os.makedirs(vardir)
phistory = {}
else:
phistory = pickle.load(open(vardir + '/promotions.pkl', 'rb'))
# Read the promotion history if --last requested
if args.last:
if phistory:
print 'Last promotions:'
for lenv, time in phistory.iteritems():
print lenv, time
else:
print 'No promotions recorded'
sys.exit(0)
# Error if no environment to promote to is given
if args.env is None:
parser.error('--env is required')
promote_list = []
if not args.all:
for x in helpers.CONFIG['promotion']:
if x == 'batch':
continue
if helpers.CONFIG['promotion'][x]['name'] == target_env:
promote_list = helpers.CONFIG['promotion'][x]['content_views']
if not promote_list:
msg = "Cannot find promotion configuration for '" + target_env + "'"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FP, output)
sys.exit(1)
msg = "Config found for CV's " + str(promote_list)
helpers.log_msg(msg, 'DEBUG')
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Now, let's fetch all available lifecycle environments for this org...
(env_list, prior_list) = get_envs(org_id)
# Get the list of Content Views along with the latest view version in each environment
(ver_list, ver_descr, ver_version) = get_cv(org_id, target_env, env_list, prior_list,
promote_list)
# Promote to the given environment. Returns a list of task IDs.
promote(target_env, ver_list, ver_descr, ver_version, env_list, prior_list, dry_run, args.quiet)
# Add/Update the promotion history dictionary so we can check when we last promoted
phistory[target_env] = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
pickle.dump(phistory, open(vardir + '/promotions.pkl', 'wb'))
# Run the mailout
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
message = "Promotion completed successfully\n\n" + output
subject = "Satellite 6 promotion completed"
helpers.mailout(subject, message)
# Exit cleanly
sys.exit(0)
def main(args):
"""
Main routine
"""
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
confdir = os.path.join(dir, 'config')
# Check for sane input
parser = argparse.ArgumentParser(
description='Publishes content views for specified organization.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
group.add_argument('-a', '--all', help='Publish ALL content views', required=False,
action="store_true")
parser.add_argument('-d', '--dryrun', help='Dry Run - Only show what will be published',
required=False, action="store_true")
parser.add_argument('-l', '--last', help='Display last promotions', required=False,
action="store_true")
parser.add_argument('-c', '--comment', help="Add a custom description", required=False)
parser.add_argument('-q', '--quiet', help="Suppress progress output updates", required=False,
action="store_true")
parser.add_argument('-m', '--forcemeta', help="Force metadata regeneration", required=False,
action="store_true")
args = parser.parse_args()
# Log the fact we are starting
if not args.last:
msg = "-------- Content view publish started by " + runuser + " -----------"
helpers.log_msg(msg, 'INFO')
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dry_run = args.dryrun
# Set up the description that will be added to the published version
if args.comment:
description = args.comment
else:
description = "Published by " + runuser + "\n via API script"
# Load the promotion history
if not os.path.exists(vardir + '/promotions.pkl'):
if not os.path.exists(vardir):
os.makedirs(vardir)
phistory = {}
else:
phistory = pickle.load(open(vardir + '/promotions.pkl', 'rb'))
# Read the promotion history if --last requested
if args.last:
if phistory:
print 'Last promotions:'
for lenv, time in phistory.iteritems():
print lenv, time
else:
print 'No promotions recorded'
sys.exit(0)
publish_list = []
if not args.all:
publish_list = helpers.CONFIG['publish']['content_views']
if not publish_list:
msg = "Cannot find publish configuration"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FP, output)
sys.exit(1)
msg = "Config found for CV's " + str(publish_list)
helpers.log_msg(msg, 'DEBUG')
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the list of Content Views along with the latest view version in each environment
(ver_list, ver_descr, ver_version) = get_cv(org_id, publish_list)
# Publish the content views. Returns a list of task IDs.
publish(ver_list, ver_descr, ver_version, dry_run, runuser, description, args.quiet,
args.forcemeta)
# Add/Update the promotion history dictionary so we can check when we last promoted
phistory['Library'] = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
pickle.dump(phistory, open(vardir + '/promotions.pkl', 'wb'))
# Run the mailout
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
message = "Publish completed successfully\n\n" + output
subject = "Satellite 6 publish completed"
helpers.mailout(subject, message)
# Exit cleanly
sys.exit(0)
def main(args):
"""
Main routine
"""
# Who is running this script?
runuser = helpers.who_is_running()
# Check for sane input
parser = argparse.ArgumentParser(
description='Cleans content views for specified organization.')
group = parser.add_mutually_exclusive_group()
# pylint: disable=bad-continuation
parser.add_argument('-o',
'--org',
help='Organization (Uses default if not specified)',
required=False)
parser.add_argument(
'-k',
'--keep',
help='How many old versions to keep (only used with -a)',
required=False)
group.add_argument('-a',
'--all',
help='Clean ALL content views',
required=False,
action="store_true")
parser.add_argument('-c',
'--cleanall',
help='Remove orphan versions between in-use views',
required=False,
action="store_true")
parser.add_argument(
'-i',
'--ignorefirstpromoted',
help=
'Version to keep count starts from first CV, not first promoted CV',
required=False,
action="store_true")
parser.add_argument('-d',
'--dryrun',
help='Dry Run - Only show what will be cleaned',
required=False,
action="store_true")
args = parser.parse_args()
# Log the fact we are starting
msg = "-------- Content view cleanup started by " + runuser + " -----------"
helpers.log_msg(msg, 'INFO')
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dry_run = args.dryrun
cleanall = args.cleanall
ignorefirstpromoted = args.ignorefirstpromoted
if args.keep:
keep = args.keep
else:
keep = "0"
cleanup_list = []
if not args.all:
cleanup_list = helpers.CONFIG['cleanup']['content_views']
if not cleanup_list:
msg = "Cannot find cleanup configuration"
helpers.log_msg(msg, 'ERROR')
sys.exit(1)
msg = "Config found for CV's " + str(cleanup_list)
helpers.log_msg(msg, 'DEBUG')
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
# Get the list of Content Views along with the latest view version in each environment
(ver_list, ver_descr, ver_keep) = get_cv(org_id, cleanup_list, keep)
# Clean the content views. Returns a list of task IDs.
cleanup(ver_list, ver_descr, dry_run, runuser, ver_keep, cleanall,
ignorefirstpromoted)
# Exit cleanly
sys.exit(0)
def main(args):
"""
Main Routine
"""
#pylint: disable-msg=R0912,R0914,R0915
if not helpers.DISCONNECTED:
msg = "Import cannot be run on the connected Satellite (Sync) host"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FI, output)
sys.exit(1)
# Who is running this script?
runuser = helpers.who_is_running()
# Set the base dir of the script and where the var data is
global dir
global vardir
dir = os.path.dirname(__file__)
vardir = os.path.join(dir, 'var')
# Log the fact we are starting
msg = "------------- Content import started by " + runuser + " ----------------"
helpers.log_msg(msg, 'INFO')
# Check for sane input
parser = argparse.ArgumentParser(description='Performs Import of Default Content View.')
# pylint: disable=bad-continuation
parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)',
required=False)
parser.add_argument('-d', '--dataset', \
help='Date/name of Import dataset to process (YYYY-MM-DD_NAME)', required=False)
parser.add_argument('-n', '--nosync', help='Do not trigger a sync after extracting content',
required=False, action="store_true")
parser.add_argument('-r', '--remove', help='Remove input files after import has completed',
required=False, action="store_true")
parser.add_argument('-l', '--last', help='Display the last successful import performed',
required=False, action="store_true")
parser.add_argument('-L', '--list', help='List all successfully completed imports',
required=False, action="store_true")
parser.add_argument('-c', '--count', help='Display all package counts after import',
required=False, action="store_true")
parser.add_argument('-f', '--force', help='Force import of data if it has previously been done',
required=False, action="store_true")
parser.add_argument('-u', '--unattended', help='Answer any prompts safely, allowing automated usage',
required=False, action="store_true")
parser.add_argument('--fixhistory', help='Force import history to match export history',
required=False, action="store_true")
args = parser.parse_args()
# Set our script variables from the input args
if args.org:
org_name = args.org
else:
org_name = helpers.ORG_NAME
dataset = args.dataset
if args.fixhistory:
fixhistory = True
else:
fixhistory = False
# Record where we are running from
script_dir = str(os.getcwd())
# Get the org_id (Validates our connection to the API)
org_id = helpers.get_org_id(org_name)
imports = []
# Read the last imports data
if os.path.exists(vardir + '/imports.pkl'):
imports = pickle.load(open(vardir + '/imports.pkl', 'rb'))
# If we have a string we convert to a list. This should only occur the first time we
# migrate from the original string version of the pickle.
if type(imports) is str:
imports = imports.split()
last_import = imports[-1]
# Display the last successful import(s)
if args.last or args.list:
if os.path.exists(vardir + '/imports.pkl'):
if args.last:
msg = "Last successful import was " + last_import
helpers.log_msg(msg, 'INFO')
print msg
if args.list:
print "Completed imports:\n----------------"
for item in imports: print item
else:
msg = "Import has never been performed"
helpers.log_msg(msg, 'INFO')
print msg
sys.exit(0)
# If we got this far without -d being specified, error out cleanly
if args.dataset is None:
parser.error("--dataset is required")
# If we have already imported this dataset let the user know
if dataset in imports:
if not args.force:
msg = "Dataset " + dataset + " has already been imported. Use --force if you really want to do this."
helpers.log_msg(msg, 'WARNING')
sys.exit(2)
# Figure out if we have the specified input fileset
basename = get_inputfiles(dataset)
# Cleanup from any previous imports
os.system("rm -rf " + helpers.IMPORTDIR + "/{content,custom,listing,*.pkl}")
# Extract the input files
extract_content(basename)
# Read in the export history from the input dataset
dsname = dataset.split('_')[1]
exports = pickle.load(open(helpers.IMPORTDIR + '/exporthistory_' + dsname + '.pkl', 'rb'))
# Check for and let the user decide if they want to continue with missing imports
missing_imports = check_missing(imports, exports, dataset, fixhistory, vardir)
if missing_imports:
msg = "Run sat_import with the --fixhistory flag to reset the import history to this export"
helpers.log_msg(msg, 'INFO')
if not args.unattended:
answer = helpers.query_yes_no("Continue with import?", "no")
if not answer:
msg = "Import Aborted"
helpers.log_msg(msg, 'ERROR')
sys.exit(3)
else:
msg = "Import continued by user"
helpers.log_msg(msg, 'INFO')
else:
msg = "Import Aborted"
helpers.log_msg(msg, 'ERROR')
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
helpers.mailout(helpers.MAILSUBJ_FI, output)
sys.exit(3)
# Trigger a sync of the content into the Library
if args.nosync:
#print helpers.GREEN + "Import complete.\n" + helpers.ENDC
msg = "Repository sync was requested to be skipped"
helpers.log_msg(msg, 'WARNING')
print 'Please synchronise all repositories to make new content available for publishing.'
delete_override = True
else:
# We need to figure out which repos to sync. This comes to us via a pickle containing
# a list of repositories that were exported
imported_repos = pickle.load(open('exported_repos.pkl', 'rb'))
package_count = pickle.load(open('package_count.pkl', 'rb'))
# Run a repo sync on each imported repo
(delete_override, newrepos) = sync_content(org_id, imported_repos)
print helpers.GREEN + "Import complete.\n" + helpers.ENDC
print 'Please publish content views to make new content available.'
# Verify the repository package/erratum counts match the sync host
check_counts(org_id, package_count, args.count)
if os.path.exists(helpers.IMPORTDIR + '/puppetforge'):
print 'Offline puppet-forge-server bundle is available to import seperately in '\
+ helpers.IMPORTDIR + '/puppetforge\n'
if args.remove and not delete_override:
msg = "Removing input files from " + helpers.IMPORTDIR
helpers.log_msg(msg, 'INFO')
print msg
os.system("rm -f " + helpers.IMPORTDIR + "/sat6_export_" + dataset + "*")
os.system("rm -rf " + helpers.IMPORTDIR + "/{content,custom,listing,*.pkl}")
excode = 0
elif delete_override:
msg = "* Not removing input files due to incomplete sync *"
helpers.log_msg(msg, 'INFO')
print msg
excode = 2
else:
msg = " (Removal of input files was not requested)"
helpers.log_msg(msg, 'INFO')
print msg
excode = 0
msg = "Import Complete"
helpers.log_msg(msg, 'INFO')
# Save the last completed import data (append to existing pickle)
os.chdir(script_dir)
if not os.path.exists(vardir):
os.makedirs(vardir)
imports.append(dataset)
pickle.dump(imports, open(vardir + '/imports.pkl', "wb"))
# Run the mailout
if helpers.MAILOUT:
helpers.tf.seek(0)
output = "{}".format(helpers.tf.read())
if missing_imports:
message = "Import of dataset " + dataset + " completed successfully.\n\n \
Missing datasets were detected during the import - please check the logs\n\n" + output
subject = "Satellite 6 import completed: Missing datasets"
elif newrepos:
message = "Import of dataset " + dataset + " completed successfully.\n\n \
New repos found that need to be imported manually - please check the logs \n\n" + output
subject = "Satellite 6 import completed: New repos require manual intervention"
else:
message = "Import of dataset " + dataset + " completed successfully\n\n" + output
subject = "Satellite 6 import completed"
helpers.mailout(subject, message)
# And exit.
sys.exit(excode)