def update_version_and_merge_for_component(component, opts):
project_dir = builder.clone_branch(component)
try:
git_branch = promote.get_current_git_upstream_branch(project_dir)
parent_branch = component.get('parent_branch')
except subprocess.CalledProcessError:
# most likely, git branch is a tag. In that event, there's nothing to update or
# merge forward. The script was either called with the wrong release config, or is
# being used as an expedient to check out the git repos for a given release config.
# Either way, nothing can be done with this branch.
print(("Unable to determine git branch for git repo in {}, HEAD is probably a tag."
" Moving to next component.").format(project_dir))
return
if opts.update_version:
if git_branch.endswith('-release'):
# Even if update_version was requested, the only way versions should get updated on an
# x.y-release branch is through the merging of a released tag.
print("Not updating version on release branch, only merging branches forward.")
else:
promotion_chain = promote.get_promotion_chain(project_dir, git_branch,
parent_branch=parent_branch)
promote.check_merge_forward(project_dir, promotion_chain)
update_version = os.path.join(CI_DIR, 'update-version.py')
# Update the version to the one specified in the config
command = ['./update-version.py', '--version', component['version'], project_dir]
subprocess.call(command, cwd=CI_DIR)
command = ['git', 'commit', '-a', '-m', 'Bumping version to %s' % component['version']]
subprocess.call(command, cwd=project_dir)
if opts.push:
command = ['git', 'push', '-v']
subprocess.call(command, cwd=project_dir)
else:
print("Skipping version update, only merging branches forward.")
promote.merge_forward(project_dir, push=opts.push, parent_branch=parent_branch)
def main():
# Parse the args
parser = argparse.ArgumentParser()
parser.add_argument("--release",
required=True,
help="Build the docs for a given release.")
opts = parser.parse_args()
is_pulp3 = opts.release.startswith('3')
configuration = builder.load_config(opts.release)
# Get platform build version
repo_list = builder.components(configuration)
try:
pulp_dict = list(filter(lambda x: x['name'] == 'pulp', repo_list))[0]
except IndexError:
raise RuntimeError("config file does not have an entry for 'pulp'")
version = pulp_dict['version']
if version.endswith('alpha'):
build_type = 'nightly'
elif version.endswith('beta'):
build_type = 'testing'
elif version.endswith('rc'):
build_type = 'testing'
else:
build_type = 'ga'
x_y_version = '.'.join(version.split('.')[:2])
builder.ensure_dir(WORKING_DIR, clean=True)
# use the version update scripts to check out git repos and ensure correct versions
for component in repo_list:
builder.clone_branch(component)
# install any apidoc dependencies that exist for pulp 3 docs
if is_pulp3:
for repo, packages in APIDOC_PACKAGES.items():
for package in packages:
package_dir = os.path.join(WORKING_DIR, repo, package)
if os.path.exists(package_dir):
subprocess.check_call(['python', 'setup.py', 'develop'],
cwd=package_dir)
plugins_dir = os.sep.join([WORKING_DIR, 'pulp', 'docs', 'plugins'])
builder.ensure_dir(plugins_dir, clean=False)
for component in repo_list:
if component['name'] == 'pulp':
promote.update_versions(os.path.join(WORKING_DIR, 'pulp'),
*version.split('-'))
continue
if component['name'] == 'pulp_deb':
continue
src = os.sep.join([WORKING_DIR, component['name'], 'docs'])
dst = os.sep.join([plugins_dir, component['name']])
os.symlink(src, dst)
if is_pulp3:
src_index_path = 'docs/pulp_index_pulp3.rst'
src_all_content_path = 'docs/all_content_index_pulp3.rst'
else:
src_index_path = 'docs/pulp_index.rst'
src_all_content_path = 'docs/all_content_index.rst'
# copy in the plugin_index.rst file for Pulp 2 only
# (currently Pulp 3 has its own plugins/index.rst without a need of managing it here,
# outside of platform code)
plugin_index_rst = os.sep.join([plugins_dir, 'index.rst'])
copyfile('docs/plugin_index.rst', plugin_index_rst)
# copy in the pulp_index.rst file
pulp_index_rst = os.sep.join([WORKING_DIR, 'pulp', 'docs', 'index.rst'])
copyfile(src_index_path, pulp_index_rst)
# copy in the all_content_index.rst file
all_content_index_rst = os.sep.join(
[WORKING_DIR, 'pulp', 'docs', 'all_content_index.rst'])
copyfile(src_all_content_path, all_content_index_rst)
# make the _templates dir
layout_dir = os.sep.join([WORKING_DIR, 'pulp', 'docs', '_templates'])
os.makedirs(layout_dir)
# copy in the layout.html file for analytics
layout_html_path = os.sep.join(
[WORKING_DIR, 'pulp', 'docs', '_templates', 'layout.html'])
copyfile('docs/layout.html', layout_html_path)
# build the docs via the Pulp project itself
print("Building the docs")
docs_directory = os.sep.join([WORKING_DIR, 'pulp', 'docs'])
make_command = ['make', 'html']
exit_code = subprocess.call(make_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError('An error occurred while building the docs.')
# rsync the docs to the root if it's GA of latest
if build_type == 'ga' and x_y_version == LATEST:
local_path_arg = os.sep.join([docs_directory, '_build', 'html'
]) + os.sep
remote_path_arg = '%s@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
rsync_command = [
'rsync', '-avzh', '--delete', '--exclude', 'en', local_path_arg,
remote_path_arg
]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError(
'An error occurred while pushing latest docs to OpenShift.')
# rsync the nightly "master" docs to an unversioned "nightly" dir for
# easy linking to in-development docs: /en/nightly/
if build_type == 'nightly' and opts.release == 'master':
local_path_arg = os.sep.join([docs_directory, '_build', 'html'
]) + os.sep
remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT,
build_type)
path_option_arg = 'mkdir -p %sen/%s/ && rsync' % (SITE_ROOT,
build_type)
rsync_command = [
'rsync', '-avzh', '--rsync-path', path_option_arg, '--delete',
local_path_arg, remote_path_arg
]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError(
'An error occurred while pushing nightly docs to OpenShift.')
# rsync the docs to OpenShift
local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT,
x_y_version)
if build_type != 'ga':
remote_path_arg += build_type + '/'
path_option_arg = 'mkdir -p %sen/%s/%s/ && rsync' % (
SITE_ROOT, x_y_version, build_type)
rsync_command = [
'rsync', '-avzh', '--rsync-path', path_option_arg, '--delete',
local_path_arg, remote_path_arg
]
else:
path_option_arg = 'mkdir -p %sen/%s/ && rsync' % (SITE_ROOT,
x_y_version)
rsync_command = [
'rsync', '-avzh', '--rsync-path', path_option_arg, '--delete',
'--exclude', 'nightly', '--exclude', 'testing', local_path_arg,
remote_path_arg
]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError(
'An error occurred while pushing docs to OpenShift.')
# rsync the robots.txt to OpenShift
local_path_arg = 'docs/robots.txt'
remote_path_arg = '%s@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
scp_command = ['scp', local_path_arg, remote_path_arg]
exit_code = subprocess.call(scp_command)
if exit_code != 0:
raise RuntimeError(
'An error occurred while pushing robots.txt to OpenShift.')
# rsync the testrubyserver.rb to OpenShift
local_path_arg = 'docs/testrubyserver.rb'
remote_path_arg = '%s@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
scp_command = ['scp', local_path_arg, remote_path_arg]
exit_code = subprocess.call(scp_command)
if exit_code != 0:
raise RuntimeError(
'An error occurred while pushing testrubyserver.rb to OpenShift.')
# add symlink for latest
symlink_cmd = [
'ssh',
'%s@%s' % (USERNAME, HOSTNAME),
'ln -sfn %sen/%s %sen/latest' % (SITE_ROOT, LATEST, SITE_ROOT)
]
exit_code = subprocess.call(symlink_cmd)
if exit_code != 0:
raise RuntimeError(
"An error occurred while creating the 'latest' symlink "
"testrubyserver.rb to OpenShift.")
builder.ensure_dir(TITO_DIR, clean=True)
builder.ensure_dir(MASH_DIR, clean=True)
# Initialize our connection to koji
builder.init_koji()
# Build our WORKING_DIR
builder.ensure_dir(WORKING_DIR, clean=True)
# Get the project to build from git by spoofing a release config component
component = {
'name': opts.project,
'git_url': "[email protected]:pulp/{}.git".format(opts.project),
'git_branch': opts.branch,
}
project_path = builder.clone_branch(component)
print("Building list of things to build")
download_list = []
build_list = []
# Get all spec files
for spec in builder.find_all_spec_files(project_path):
spec_nvr = builder.get_package_nvr_from_spec(spec)
package_dists = builder.get_dists_for_spec(spec)
print("%s %s" % (spec_nvr, package_dists))
for package_nevra in builder.get_package_nevra(spec_nvr, package_dists):
info = builder.mysession.getBuild(package_nevra)
if info:
download_list.extend(builder.get_urls_for_build(builder.mysession, package_nevra))
# Source extract all the components
parent_branches = {}
merge_forward = {}
component_list = []
spec_project_map = {}
# these get set in the "for component" loop a few lines down
platform_version = None
# el6 is supported when the platform version is 2.11 or lower
el6_supported = None
print("Getting git repos")
for component in builder.components(configuration):
project_dir = builder.clone_branch(component)
branch_name = component['git_branch']
parent_branch = component.get('parent_branch', None)
parent_branches['origin/%s' % branch_name] = parent_branch
if component['name'] == 'pulp':
platform_version = parse_version(promote.to_python_version(component['version']))
el6_supported = platform_version <= parse_version('2.11')
# Check if this is a branch or a tag
tag_exists = builder.does_git_tag_exist(branch_name, project_dir)
component_list.append(component['name'])
merge_forward[component['name']] = False
if not tag_exists:
merge_forward[component['name']] = True
# Check if everything is merged forward
# Source extract all the components
parent_branches = {}
merge_forward = {}
component_list = []
spec_project_map = {}
# these get set in the "for component" loop a few lines down
platform_version = None
# el6 is supported when the platform version is 2.11 or lower
el6_supported = None
print("Getting git repos")
for component in builder.components(configuration):
project_dir = builder.clone_branch(component)
branch_name = component['git_branch']
parent_branch = component.get('parent_branch', None)
parent_branches['origin/%s' % branch_name] = parent_branch
if component['name'] == 'pulp':
platform_version = parse_version(
promote.to_python_version(component['version']))
el6_supported = platform_version <= parse_version('2.11')
# Check if this is a branch or a tag
tag_exists = builder.does_git_tag_exist(branch_name, project_dir)
component_list.append(component['name'])
merge_forward[component['name']] = False
if not tag_exists:
merge_forward[component['name']] = True
def main():
# Parse the args
parser = argparse.ArgumentParser()
parser.add_argument("--release", required=True, help="Build the docs for a given release.")
opts = parser.parse_args()
is_pulp3 = opts.release.startswith('3')
configuration = builder.load_config(opts.release)
# Get platform build version
repo_list = builder.components(configuration)
try:
pulp_dict = list(filter(lambda x: x['name'] == 'pulp', repo_list))[0]
except IndexError:
raise RuntimeError("config file does not have an entry for 'pulp'")
version = pulp_dict['version']
if version.endswith('alpha') or is_pulp3:
build_type = 'nightly'
elif version.endswith('beta'):
build_type = 'testing'
elif version.endswith('rc'):
build_type = 'testing'
else:
build_type = 'ga'
x_y_version = '.'.join(version.split('.')[:2])
builder.ensure_dir(WORKING_DIR, clean=True)
# use the version update scripts to check out git repos and ensure correct versions
for component in repo_list:
builder.clone_branch(component)
# install any apidoc dependencies that exist for pulp 3 docs
if is_pulp3:
for repo, packages in APIDOC_PACKAGES.items():
for package in packages:
package_dir = os.path.join(WORKING_DIR, repo, package)
if os.path.exists(package_dir):
subprocess.check_call(['pip', 'install', '-e', '.'], cwd=package_dir)
plugins_dir = os.sep.join([WORKING_DIR, 'pulp', 'docs', 'plugins'])
builder.ensure_dir(plugins_dir, clean=False)
for component in repo_list:
if component['name'] == 'pulp':
promote.update_versions(os.path.join(WORKING_DIR, 'pulp'), *version.split('-'))
continue
if component['name'] == 'pulp_deb':
continue
src = os.sep.join([WORKING_DIR, component['name'], 'docs'])
dst = os.sep.join([plugins_dir, component['name']])
os.symlink(src, dst)
if is_pulp3:
src_index_path = 'docs/pulp_index_pulp3.rst'
src_all_content_path = 'docs/all_content_index_pulp3.rst'
else:
src_index_path = 'docs/pulp_index.rst'
src_all_content_path = 'docs/all_content_index.rst'
# copy in the plugin_index.rst file for Pulp 2 only
# (currently Pulp 3 has its own plugins/index.rst without a need of managing it here,
# outside of platform code)
plugin_index_rst = os.sep.join([plugins_dir, 'index.rst'])
copyfile('docs/plugin_index.rst', plugin_index_rst)
# copy in the pulp_index.rst file
pulp_index_rst = os.sep.join([WORKING_DIR, 'pulp', 'docs', 'index.rst'])
copyfile(src_index_path, pulp_index_rst)
# copy in the all_content_index.rst file
all_content_index_rst = os.sep.join([WORKING_DIR, 'pulp', 'docs', 'all_content_index.rst'])
copyfile(src_all_content_path, all_content_index_rst)
# make the _templates dir
layout_dir = os.sep.join([WORKING_DIR, 'pulp', 'docs', '_templates'])
os.makedirs(layout_dir)
# copy in the layout.html file for analytics
layout_html_path = os.sep.join([WORKING_DIR, 'pulp', 'docs', '_templates', 'layout.html'])
copyfile('docs/layout.html', layout_html_path)
# build the docs via the Pulp project itself
print("Building the docs")
docs_directory = os.sep.join([WORKING_DIR, 'pulp', 'docs'])
# Get the latest api.yaml file to build the rest api docs
if is_pulp3:
with urllib.request.urlopen("http://*****:*****@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
rsync_command = ['rsync', '-avzh', '--delete', '--exclude', 'en',
'--omit-dir-times', local_path_arg, remote_path_arg]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError('An error occurred while pushing latest docs.')
# Also publish to the /en/latest/ directory
make_directory_with_rsync(['en', 'latest'])
local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
remote_path_arg = '%s@%s:%sen/latest/' % (USERNAME, HOSTNAME, SITE_ROOT)
rsync_command = ['rsync', '-avzh', '--delete', local_path_arg, remote_path_arg]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError("An error occurred while pushing the 'latest' directory.")
# rsync the nightly "2-master" docs to an unversioned "nightly" dir for
# easy linking to in-development docs: /en/nightly/
if build_type == 'nightly' and opts.release == '2-master':
local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT, build_type)
make_directory_with_rsync(['en', build_type])
rsync_command = ['rsync', '-avzh', '--delete', local_path_arg, remote_path_arg]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError('An error occurred while pushing nightly docs.')
# rsync the docs
local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT, x_y_version)
if build_type != 'ga':
remote_path_arg += build_type + '/'
make_directory_with_rsync(['en', x_y_version, build_type])
rsync_command = ['rsync', '-avzh', '--delete', local_path_arg, remote_path_arg]
else:
make_directory_with_rsync(['en', x_y_version])
rsync_command = ['rsync', '-avzh', '--delete', '--exclude', 'nightly', '--exclude',
'testing', local_path_arg, remote_path_arg]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError('An error occurred while pushing docs.')
def main():
# Parse the args
parser = argparse.ArgumentParser()
parser.add_argument("--release",
required=True,
help="Build the docs for a given release.")
opts = parser.parse_args()
is_pulp3 = opts.release.startswith('3')
configuration = builder.load_config(opts.release)
# Get platform build version
repo_list = builder.components(configuration)
try:
pulp_dict = list(filter(lambda x: x['name'] == 'pulp', repo_list))[0]
except IndexError:
raise RuntimeError("config file does not have an entry for 'pulp'")
version = pulp_dict['version']
if version.endswith('alpha') or is_pulp3:
build_type = 'nightly'
elif version.endswith('beta'):
build_type = 'testing'
elif version.endswith('rc'):
build_type = 'testing'
else:
build_type = 'ga'
x_y_version = '.'.join(version.split('.')[:2])
builder.ensure_dir(WORKING_DIR, clean=True)
# use the version update scripts to check out git repos and ensure correct versions
for component in repo_list:
builder.clone_branch(component)
# install any apidoc dependencies that exist for pulp 3 docs
if is_pulp3:
for repo, packages in APIDOC_PACKAGES.items():
for package in packages:
package_dir = os.path.join(WORKING_DIR, repo, package)
if os.path.exists(package_dir):
subprocess.check_call(['pip', 'install', '-e', '.'],
cwd=package_dir)
plugins_dir = os.sep.join([WORKING_DIR, 'pulp', 'docs', 'plugins'])
builder.ensure_dir(plugins_dir, clean=False)
for component in repo_list:
if component['name'] == 'pulp':
promote.update_versions(os.path.join(WORKING_DIR, 'pulp'),
*version.split('-'))
continue
if component['name'] == 'pulp_deb':
continue
src = os.sep.join([WORKING_DIR, component['name'], 'docs'])
dst = os.sep.join([plugins_dir, component['name']])
os.symlink(src, dst)
if is_pulp3:
src_index_path = 'docs/pulp_index_pulp3.rst'
src_all_content_path = 'docs/all_content_index_pulp3.rst'
else:
src_index_path = 'docs/pulp_index.rst'
src_all_content_path = 'docs/all_content_index.rst'
# copy in the plugin_index.rst file for Pulp 2 only
# (currently Pulp 3 has its own plugins/index.rst without a need of managing it here,
# outside of platform code)
plugin_index_rst = os.sep.join([plugins_dir, 'index.rst'])
copyfile('docs/plugin_index.rst', plugin_index_rst)
# copy in the pulp_index.rst file
pulp_index_rst = os.sep.join([WORKING_DIR, 'pulp', 'docs', 'index.rst'])
copyfile(src_index_path, pulp_index_rst)
# copy in the all_content_index.rst file
all_content_index_rst = os.sep.join(
[WORKING_DIR, 'pulp', 'docs', 'all_content_index.rst'])
copyfile(src_all_content_path, all_content_index_rst)
# make the _templates dir
layout_dir = os.sep.join([WORKING_DIR, 'pulp', 'docs', '_templates'])
os.makedirs(layout_dir)
# copy in the layout.html file for analytics
layout_html_path = os.sep.join(
[WORKING_DIR, 'pulp', 'docs', '_templates', 'layout.html'])
copyfile('docs/layout.html', layout_html_path)
# build the docs via the Pulp project itself
print("Building the docs")
docs_directory = os.sep.join([WORKING_DIR, 'pulp', 'docs'])
# Get the latest api.yaml file to build the rest api docs
if is_pulp3:
with urllib.request.urlopen("http://*****:*****@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
rsync_command = [
'rsync', '-avzh', '--delete', '--exclude', 'en',
'--omit-dir-times', local_path_arg, remote_path_arg
]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError('An error occurred while pushing latest docs.')
# Also publish to the /en/latest/ directory
make_directory_with_rsync(['en', 'latest'])
local_path_arg = os.sep.join([docs_directory, '_build', 'html'
]) + os.sep
remote_path_arg = '%s@%s:%sen/latest/' % (USERNAME, HOSTNAME,
SITE_ROOT)
rsync_command = [
'rsync', '-avzh', '--delete', local_path_arg, remote_path_arg
]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError(
"An error occurred while pushing the 'latest' directory.")
# rsync the nightly "2-master" docs to an unversioned "nightly" dir for
# easy linking to in-development docs: /en/nightly/
if build_type == 'nightly' and opts.release == '2-master':
local_path_arg = os.sep.join([docs_directory, '_build', 'html'
]) + os.sep
remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT,
build_type)
make_directory_with_rsync(['en', build_type])
rsync_command = [
'rsync', '-avzh', '--delete', local_path_arg, remote_path_arg
]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError('An error occurred while pushing nightly docs.')
# rsync the docs
local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT,
x_y_version)
if build_type != 'ga':
remote_path_arg += build_type + '/'
make_directory_with_rsync(['en', x_y_version, build_type])
rsync_command = [
'rsync', '-avzh', '--delete', local_path_arg, remote_path_arg
]
else:
make_directory_with_rsync(['en', x_y_version])
rsync_command = [
'rsync', '-avzh', '--delete', '--exclude', 'nightly', '--exclude',
'testing', local_path_arg, remote_path_arg
]
exit_code = subprocess.call(rsync_command, cwd=docs_directory)
if exit_code != 0:
raise RuntimeError('An error occurred while pushing docs.')