def verify_sls(pdc_url, sl_dict):
"""
Verifies that the service levels are properly formatted and exist in PDC
:param pdc_url: a string of the URL to PDC
:param sl_dict: a dictionary with the SLs of the request
:return: None or ValidationError
"""
# Make sure the EOL date is in the format of 2020-12-01
eol_date_regex = re.compile(r'\d{4}-\d{2}-\d{2}')
for sl, eol in sl_dict.items():
if re.match(eol_date_regex, eol):
eol_date = datetime.strptime(eol, '%Y-%m-%d').date()
today = datetime.utcnow().date()
if eol_date < today:
raise rpkgError(
'The SL "{0}" is already expired'.format(eol))
elif eol_date.month not in [6, 12] or eol_date.day != 1:
raise rpkgError(
'The SL "{0}" must expire on June 1st or December 1st'
.format(eol))
else:
raise rpkgError(
'The EOL date "{0}" is in an invalid format'.format(eol))
sl_obj = get_sl_type(pdc_url, sl)
if not sl_obj:
raise rpkgError('The SL "{0}" is not in PDC'.format(sl))
def query_pdc(server_url, endpoint, params, timeout=60):
api_url = '{0}/rest_api/v1/{1}/'.format(
server_url.rstrip('/'), endpoint.strip('/'))
query_args = params
while True:
try:
rv = requests.get(api_url, params=query_args, timeout=60)
except ConnectionError as error:
error_msg = ('The connection to PDC failed while trying to get '
'the active release branches. The error was: {0}'
.format(str(error)))
raise rpkgError(error_msg)
if not rv.ok:
base_error_msg = ('The following error occurred while trying to '
'get the active release branches in PDC: {0}')
raise rpkgError(base_error_msg.format(rv.text))
rv_json = rv.json()
for item in rv_json['results']:
yield item
if rv_json['next']:
# Clear the query_args because they are baked into the "next" URL
query_args = {}
api_url = rv_json['next']
else:
# We've gone through every page, so we can return the found
# branches
break
def push(self):
"""Push changes to the remote repository"""
# First check that we are not pushing to Fedora
# FIXME: Ugly screen scraping
push_remote = self.repo.git.config('--get',
'branch.%s.remote' % self.branch_merge)
if push_remote != self.remote:
raise pyrpkg.rpkgError('Can only push to the Network Box ' + \
'infrastructure')
# Then only push the relevant branches on the appropriate remote
cmd = ['git', 'push', self.remote]
# FIXME: Ugly screen scraping, functional programming style
merged = map(lambda x: x.strip(),
filter(lambda x: re.match(self.branchre, x),
git.Git().branch('--merged',
self.repo.active_branch.name
).split()))
# Move the active branch to the end, so that merged upstream branches
# get pushed first
if self.repo.active_branch.name in merged:
merged.remove(self.repo.active_branch.name)
merged.append(self.repo.active_branch.name)
if not merged:
raise pyrpkg.rpkgError('Could not find any local branch to push')
cmd.extend(merged)
self._run_command(cmd, cwd=self.path)
def get_fedora_release_state(config, cli_name, release):
"""
Queries service page for release state. Query result is returned as json dict.
:param config: ConfigParser object
:param cli_name: string of the CLI's name (e.g. fedpkg)
:param str release: short release name. Example: F29, F30, F29M, F30C, ...
:return: state of the release or None if there is no such release
:rtype: str
"""
try:
# url of the release service. It needs to be expanded by release name
releases_service_url = config.get('{0}.bodhi'.format(cli_name),
'releases_service',
vars={'release': release})
except (ValueError, NoOptionError, NoSectionError) as e:
raise rpkgError('Could not get release state for Fedora '
'({0}): {1}.'.format(release, str(e)))
try:
rv = requests.get(releases_service_url, timeout=60)
except ConnectionError as error:
error_msg = ('The connection to Bodhi failed while trying to get '
'release state. The error was: {0}'.format(str(error)))
raise rpkgError(error_msg)
if rv.status_code == 404:
# release wasn't found
return None
elif not rv.ok:
base_error_msg = ('The following error occurred while trying to '
'get the release state in Bodhi: {0}')
raise rpkgError(base_error_msg.format(rv.text))
return rv.json().get('state')
def get_sl_type(url, sl_name):
"""
Gets the service level (SL) type from PDC
:param url: a string of the URL to PDC
:param sl_name: a string of the SL name
:return: a dictionary representing the SL type or None
"""
api_url = '{0}/rest_api/v1/component-sla-types/'.format(url.rstrip('/'))
api_url_w_args = '{0}?{1}'.format(api_url, urlencode({'name': sl_name}))
try:
rv = requests.get(api_url_w_args, timeout=60)
except ConnectionError as error:
error_msg = ('The connection to PDC failed while trying to validate '
'the passed in service level. The error was: {0}'
.format(str(error)))
raise rpkgError(error_msg)
if not rv.ok:
base_error_msg = ('The following error occurred while validating the '
'passed in service level in PDC: {0}')
raise rpkgError(base_error_msg.format(rv.text))
rv_json = rv.json()
if rv_json['count'] == 1:
return rv_json['results'][0]
else:
return None
def new_pagure_issue(logger, url, token, title, body, cli_name):
"""
Posts a new Pagure issue
:param logger: A logger object
:param url: a string of the URL to Pagure
:param token: a string of the Pagure API token that has rights to create
a ticket
:param title: a string of the issue's title
:param body: a string of the issue's body
:return: a string of the URL to the created issue in the UI
"""
api_url = '{0}/api/0'.format(url.rstrip('/'))
new_issue_url = '{0}/releng/fedora-scm-requests/new_issue'.format(api_url)
headers = {
'Authorization': 'token {0}'.format(token),
'Accept': 'application/json',
'Content-Type': 'application/json'
}
payload = json.dumps({
'title': title,
'issue_content': body
})
try:
rv = requests.post(
new_issue_url, headers=headers, data=payload, timeout=60)
except ConnectionError as error:
error_msg = ('The connection to Pagure failed while trying to '
'create a new issue. The error was: {0}'.format(
str(error)))
raise rpkgError(error_msg)
base_error_msg = ('The following error occurred while creating a new '
'issue in Pagure: {0}')
try:
# Extract response error text (and debug data)
rv_json = rv.json()
logger.debug("Pagure API response: '{0}'".format(rv_json))
rv_error = rv_json.get('error')
except (ValueError, AttributeError):
rv_error = rv.text
if not rv.ok:
# Lets see if the API returned an error message in JSON that we can
# show the user
# show hint for expired token
if re.search(r"Invalid or expired token", rv_error, re.IGNORECASE):
base_error_msg += '\nFor invalid or expired token refer to ' \
'"{0} request-repo -h" to set a token in your user ' \
'configuration.'.format(cli_name)
raise rpkgError(base_error_msg.format(rv_error))
return '{0}/releng/fedora-scm-requests/issue/{1}'.format(
url.rstrip('/'), rv.json()['issue']['id'])
def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
"""
Creates a fork of the project.
:param logger: A logger object
:param base_url: a string of the URL repository
:param token: a string of the API token that has rights to make a fork
:param repo_name: a string of the repository name
:param namespace: a string determines a type of the repository
:param cli_name: string of the CLI's name (e.g. fedpkg)
:return: a bool; True when fork was created, False when already exists
"""
api_url = '{0}/api/0'.format(base_url.rstrip('/'))
fork_url = '{0}/fork'.format(api_url)
headers = {
'Authorization': 'token {0}'.format(token),
'Accept': 'application/json',
'Content-Type': 'application/json'
}
payload = json.dumps({
'wait': True,
'namespace': namespace,
'repo': repo_name,
})
try:
rv = requests.post(
fork_url, headers=headers, data=payload, timeout=60)
except ConnectionError as error:
error_msg = ('The connection to API failed while trying to '
'create a new fork. The error was: {0}'.format(str(error)))
raise rpkgError(error_msg)
try:
# Extract response error text (and debug data)
rv_json = rv.json()
logger.debug("Pagure API response: '{0}'".format(rv_json))
rv_error = rv_json.get('error')
except (ValueError, AttributeError):
rv_error = rv.text
base_error_msg = ('The following error occurred while creating a new fork: {0}')
if not rv.ok:
# Lets see if the API returned an error message in JSON that we can
# show the user
if re.search(r"Repo .+ already exists", rv_error, re.IGNORECASE):
return False
# show hint for expired token
if re.search(r"Invalid or expired token", rv_error, re.IGNORECASE):
base_error_msg += '\nFor invalid or expired token refer to ' \
'"{0} fork -h" to set a token in your user ' \
'configuration.'.format(cli_name)
raise rpkgError(base_error_msg.format(rv_error))
return True
def new_ticket(self, passwd, desc, build=None):
"""Open a new ticket on Rel-Eng trac instance.
Get ticket component and assignee from current branch
Create a new task ticket using username/password/desc
Discover build nvr from module or optional build argument
Return ticket number on success
"""
override = self.override
if not override:
raise pyrpkg.rpkgError('Override tag is not required for %s' %
self.branch_merge)
uri = self.tracbaseurl % {'user': self.user, 'password': passwd}
self.trac = offtrac.TracServer(uri)
# Set trac's component and assignee from related distvar
if self.distvar == 'fedora':
component = 'koji'
#owner = '*****@*****.**'
elif self.distvar == 'rhel':
component = 'epel'
#owner = '*****@*****.**'
# Raise if people request a tag against something that self updates
build_target = self.anon_kojisession.getBuildTarget(self.target)
if not build_target:
raise pyrpkg.rpkgError('Unknown build target: %s' % self.target)
dest_tag = self.anon_kojisession.getTag(build_target['dest_tag_name'])
ancestors = self.anon_kojisession.getFullInheritance(
build_target['build_tag'])
if dest_tag['id'] in [build_target['build_tag']] + \
[ancestor['parent_id'] for
ancestor in ancestors]:
raise pyrpkg.rpkgError('Override tag is not required for %s' %
self.branch_merge)
if not build:
build = self.nvr
summary = 'Tag request %s for %s' % (build, override)
type = 'task'
try:
ticket = self.trac.create_ticket(summary, desc,
component=component,
notify=True)
except Exception, e:
raise pyrpkg.rpkgError('Could not request tag %s: %s' % (build, e))
def checky(self, *args, **kwargs):
# First only work on the remotes we care about
fedpkg = 'pkgs.*\.fedoraproject\.org\/'
# Do this in a try in case we're not in a repo
try:
remotes = [remote.name for remote in self.repo.remotes if
re.search(fedpkg, remote.url)]
except:
self.log.debug("Not in a repo, don't care about remotes")
return func(self, *args, **kwargs)
# Now loop through the remotes and see if any of them have
# old style branch names
for remote in remotes:
# Check to see if the remote data matches the old style
# This regex looks at the ref name which should be
# "origin/f15/master or simliar. This regex fills in the remote
# name we care about and attempts to find any fedora/epel/olpc
# branch that has the old style /master tail.
refsre = r'%s/(f\d\d/master|f\d/master|fc\d/master|' % remote
refsre += r'el\d/master|olpc\d/master)'
for ref in self.repo.refs:
if type(ref) == git.RemoteReference and \
re.match(refsre, ref.name):
self.log.error('This repo has old style branches but '
'upstream has converted to new style.\n'
'Please run /usr/libexec/fedpkg-fixbranches '
'to fix your repo.')
raise pyrpkg.rpkgError('Unconverted branches')
return func(self, *args, **kwargs)
def _findmasterbranch(self):
"""Find the right "rpmfusion" for master"""
# If we already have a koji session, just get data from the source
if self._kojisession:
rawhidetarget = self.kojisession.getBuildTarget('rawhide-free')
desttag = rawhidetarget['dest_tag_name']
desttag=desttag.split('-')
desttag.remove('free')
desttag=''.join(desttag)
return desttag.replace('f', '')
else:
# We may not have Fedoras. Find out what rawhide target does.
try:
rawhidetarget = self.anon_kojisession.getBuildTarget(
'rawhide-free')
except:
# We couldn't hit koji, bail.
raise pyrpkg.rpkgError('Unable to query koji to find rawhide \
target')
desttag = rawhidetarget['dest_tag_name']
desttag=desttag.split('-')
desttag.remove('free')
desttag=''.join(desttag)
return desttag.replace('f', '')
def do_add_remote(base_url, remote_base_url, username, repo, repo_name,
namespace):
"""
Adds remote tracked repository
:param base_url: a string of the URL repository
:param remote_base_url: a string of the remote tracked repository
:param username: a string of the (FAS) user name
:param repo: object, current project git repository
:param repo_name: a string of the repository name
:param namespace: a string determines a type of the repository
:return: a bool; True if remote was created, False when already exists
"""
parsed_url = urlparse(remote_base_url)
remote_url = '{0}://{1}/forks/{2}/{3}/{4}.git'.format(
parsed_url.scheme,
parsed_url.netloc,
username,
namespace,
repo_name,
)
# check already existing remote
for remote in repo.remotes:
if remote.name == username:
return False
try:
# create remote with username as its name
repo.create_remote(username, url=remote_url)
except git.exc.GitCommandError as e:
error_msg = "During create remote:\n {0}\n {1}".format(
" ".join(e.command), e.stderr)
raise rpkgError(error_msg)
return True
def assert_valid_epel_package(name, branch):
"""
Determines if the package is allowed to have an EPEL branch. If it can't,
an rpkgError will be raised.
:param name: a string of the package name
:param branch: a string of the EPEL branch name (e.g. epel7)
:return: None or rpkgError
"""
# Extract any digits in the branch name to determine the EL version
version = ''.join([i for i in branch if re.match(r'\d', i)])
url = ('https://infrastructure.fedoraproject.org/repo/json/pkg_el{0}.json'
.format(version))
error_msg = ('The connection to infrastructure.fedoraproject.org failed '
'while trying to determine if this is a valid EPEL package.')
try:
rv = requests.get(url, timeout=60)
except ConnectionError as error:
error_msg += ' The error was: {0}'.format(str(error))
raise rpkgError(error_msg)
if not rv.ok:
raise rpkgError(error_msg + ' The status code was: {0}'.format(
rv.status_code))
rv_json = rv.json()
# Remove noarch from this because noarch is treated specially
all_arches = set(rv_json['arches']) - set(['noarch'])
# On EL6, also remove ppc and i386 as many packages will
# have these arches missing and cause false positives
if int(version) == 6:
all_arches = all_arches - set(['ppc', 'i386'])
# On EL7 and later, also remove ppc and i686 as many packages will
# have these arches missing and cause false positives
elif int(version) >= 7:
all_arches = all_arches - set(['ppc', 'i686'])
error_msg_two = (
'This package is already an EL package and is built on all supported '
'arches, therefore, it cannot be in EPEL. If this is a mistake or you '
'have an exception, please contact the Release Engineering team.')
for pkg_name, pkg_info in rv_json['packages'].items():
# If the EL package is noarch only or is available on all supported
# arches, then don't allow an EPEL branch
if pkg_name == name:
pkg_arches = set(pkg_info['arch'])
if pkg_arches == set(['noarch']) or not (all_arches - pkg_arches):
raise rpkgError(error_msg_two)
def load_rpmdefines(self):
"""Populate rpmdefines based on branch data.
We need to overload this as we don't use the same branch names.
"""
# We only match the top level branch name exactly.
# Anything else is too dangerous and --dist should be used
if re.match(r'nb\d\.\d$', self.branch_merge):
# E.g: 'networkbox/nb5.0'
self._distval = self.branch_merge.split('nb')[1]
self._distvar = 'nbrs'
self.dist = 'nb%s' % self.distval
elif re.match(r'nbplayground$', self.branch_merge):
# E.g: 'networkbox-nonfree/nbplayground'
self._distval = self._findmasterbranch()
self._distvar = 'nbrs'
self.dist = 'nb%s' % self.distval
elif re.match(r'nb-fedora\d\d$', self.branch_merge):
self._distval = self.branch_merge.split("nb-fedora")[1]
self._distvar = "fedora"
self.dist = "fc%s" % self.distval
self.mockconfig = "fedora-%s-%s" % (self._distval, self.localarch)
elif re.match(r'nb-rhel\d$', self.branch_merge):
self._distval = self.branch_merge.split('nb-rhel')[1]
self._distvar = 'rhel'
self.dist = 'el%s' % self.distval
self.mockconfig = 'epel-%s-%s' % (self.distval, self.localarch)
elif re.match(r'nb-epel\d$', self.branch_merge):
self._distval = self.branch_merge.split('nb-epel')[1]
self._distvar = 'rhel'
self.dist = 'el%s' % self.distval
self.mockconfig = 'epel-%s-%s' % (self.distval, self.localarch)
else:
raise pyrpkg.rpkgError('Could not find the dist from branch name '
'%s\nPlease specify with --dist' %
self.branch_merge)
short_distval = self.distval.replace('.', '')
self._rpmdefines = ["--define '_sourcedir %s'" % self.path,
"--define '_specdir %s'" % self.path,
"--define '_builddir %s'" % self.path,
"--define '_srcrpmdir %s'" % self.path,
"--define '_rpmdir %s'" % self.path,
"--define 'dist .%s'" % self.dist,
"--define '%s %s'" % (self.distvar, short_distval),
# Note: Contrary to Fedora, we do not define the following:
# "--define '%s 1'" % self.dist,
# This is because it is completely broken in our case, because our
# dist is 'nb5.0', and rpm chokes on the dot when evaluating the macro.
# It becomes a choice between having a defined value that will never
# work or not having it defined at all, and I chose the the latter to
# avoid confusion (we never even tried to use it anyway).
]
def fetchfedora(self):
"""Synchronise with the Fedora dist-git module."""
try:
self.fedora_remote.fetch('--no-tags')
except git.cmd.GitCommandError as e:
raise pyrpkg.rpkgError("%s\n(did you forget about the --name "
"option?)" % e)
def load_rpmdefines(self):
"""Populate rpmdefines based on branch data"""
# Determine runtime environment
self._runtime_disttag = self._determine_runtime_env()
# We only match the top level branch name exactly.
# Anything else is too dangerous and --dist should be used
# This regex works until after Fedora 99.
if re.match(r'f\d\d$', self.branch_merge):
self._distval = self.branch_merge.split('f')[1]
self._distvar = 'fedora'
self.dist = 'fc%s' % self._distval
self.mockconfig = 'fedora-%s-%s' % (self._distval, self.localarch)
self.override = 'f%s-override' % self._distval
self._distunset = 'rhel'
# Works until RHEL 10
elif re.match(r'el\d$', self.branch_merge) or re.match(r'epel\d$', self.branch_merge):
self._distval = self.branch_merge.split('el')[1]
self._distvar = 'rhel'
self.dist = 'el%s' % self._distval
self.mockconfig = 'epel-%s-%s' % (self._distval, self.localarch)
self.override = 'epel%s-override' % self._distval
self._distunset = 'fedora'
elif re.match(r'olpc\d$', self.branch_merge):
self._distval = self.branch_merge.split('olpc')[1]
self._distvar = 'olpc'
self.dist = 'olpc%s' % self._distval
self.override = 'dist-olpc%s-override' % self._distval
self._distunset = 'rhel'
# master
elif re.match(r'master$', self.branch_merge):
self._distval = self._findmasterbranch()
self._distvar = 'fedora'
self.dist = 'fc%s' % self._distval
self.mockconfig = 'fedora-devel-%s' % self.localarch
self.override = None
self._distunset = 'rhel'
# If we don't match one of the above, punt
else:
raise pyrpkg.rpkgError('Could not find the dist from branch name '
'%s\nPlease specify with --dist' %
self.branch_merge)
self._rpmdefines = ["--define '_sourcedir %s'" % self.path,
"--define '_specdir %s'" % self.path,
"--define '_builddir %s'" % self.path,
"--define '_srcrpmdir %s'" % self.path,
"--define '_rpmdir %s'" % self.path,
"--define 'dist .%s'" % self.dist,
"--define '%s %s'" % (self._distvar, self._distval),
"--eval '%%undefine %s'" % self._distunset,
"--define '%s 1'" % self.dist]
if self._runtime_disttag:
if self.dist != self._runtime_disttag:
# This means that the runtime is known, and is different from
# the target, so we need to unset the _runtime_disttag
self._rpmdefines.append("--eval '%%undefine %s'" %
self._runtime_disttag)
def load_rpmdefines(self):
"""Populate rpmdefines based on branch data"""
# Determine runtime environment
self._runtime_disttag = self._determine_runtime_env()
# We only match the top level branch name exactly.
# Anything else is too dangerous and --dist should be used
# This regex works until after Fedora 99.
if re.match(r'f\d\d$', self.branch_merge):
self._distval = self.branch_merge.split('f')[1]
self._distvar = 'fedora'
self.dist = 'fc%s' % self._distval
self.mockconfig = 'fedora-%s-%s' % (self._distval, self.localarch)
self.override = 'f%s-override' % self._distval
self._distunset = 'rhel'
# Works until RHEL 10
elif re.match(r'el\d$', self.branch_merge) or re.match(
r'epel\d$', self.branch_merge):
self._distval = self.branch_merge.split('el')[1]
self._distvar = 'rhel'
self.dist = 'el%s' % self._distval
self.mockconfig = 'epel-%s-%s' % (self._distval, self.localarch)
self.override = 'epel%s-override' % self._distval
self._distunset = 'fedora'
# master
elif re.match(r'master$', self.branch_merge):
self._distval = self._findmasterbranch()
self._distvar = 'fedora'
self.dist = 'fc%s' % self._distval
self.mockconfig = 'fedora-rawhide-%s' % self.localarch
self.override = None
self._distunset = 'rhel'
# If we don't match one of the above, punt
else:
raise pyrpkg.rpkgError('Could not find the dist from branch name '
'%s\nPlease specify with --dist' %
self.branch_merge)
self._rpmdefines = [
"--define '_sourcedir %s'" % self.path,
"--define '_specdir %s'" % self.path,
"--define '_builddir %s'" % self.path,
"--define '_srcrpmdir %s'" % self.path,
"--define '_rpmdir %s'" % self.path,
"--define 'dist .%s'" % self.dist,
"--define '%s %s'" % (self._distvar, self._distval),
"--eval '%%undefine %s'" % self._distunset,
"--define '%s 1'" % self.dist
]
if self._runtime_disttag:
if self.dist != self._runtime_disttag:
# This means that the runtime is known, and is different from
# the target, so we need to unset the _runtime_disttag
self._rpmdefines.append("--eval '%%undefine %s'" %
self._runtime_disttag)
def assert_new_tests_repo(name, dist_git_url):
"""
Asserts that the tests repository name is new. Note that the repository name
can be any arbitrary string, so just check if the repository already exists.
:param name: a string with the package name
:return: None or rpkgError
"""
url = '{0}/tests/{1}'.format(dist_git_url, name)
error_msg = (
'The connection to dist-git failed '
'trying to determine if this is a valid new tests '
'repository name.')
try:
rv = requests.get(url, timeout=60)
except ConnectionError as error:
error_msg += ' The error was: {0}'.format(str(error))
raise rpkgError(error_msg)
if rv.ok:
raise rpkgError("Repository {0} already exists".format(url))
def config_get_safely(config, section, option):
"""
Returns option from the user's configuration file. In case of missing
section or option method throws an exception with a human-readable
warning and a possible hint.
The method should be used especially in situations when there are newly
added sections/options into the config. In this case, there is a risk that
the user's config wasn't properly upgraded.
:param config: ConfigParser object
:param section: section name in the config
:param option: name of the option
:return: option value from the right section
:rtype: str
"""
hint = (
"First (if possible), refer to the help of the current command "
"(-h/--help).\n"
"There also might be a new version of the config after upgrade.\n"
"Hint: you can check if you have 'fedpkg.conf.rpmnew' or "
"'fedpkg.conf.rpmsave' in the config directory. If yes, try to merge "
"your changes to the config with the maintainer provided version "
"(or replace fedpkg.conf file with 'fedpkg.conf.rpmnew')."
)
try:
return config.get(section, option)
except NoSectionError:
msg = "Missing section '{0}' in the config file.".format(section)
raise rpkgError("{0}\n{1}".format(msg, hint))
except NoOptionError:
msg = "Missing option '{0}' in the section '{1}' of the config file.".format(
option, section
)
raise rpkgError("{0}\n{1}".format(msg, hint))
except Exception:
raise
def load_spec(self):
"""This sets the spec attribute"""
# We are not using the upstream load_spec because the file structure is
# hard-coded
# Get a list of files in the path we're looking at
files = os.listdir(os.path.join(self.path, 'SPECS'))
# Search the files for the first one that ends with ".spec"
for __f in files:
if __f.endswith('.spec') and not __f.startswith('.'):
self._spec = os.path.join('SPECS', __f)
return
raise pyrpkg.rpkgError('No spec file found.')
def load_fedora_cert_files(self):
"""This loads the fedora_cert_file and fedora_ca_cert attributes"""
import ConfigParser
with open(self.fedora_kojiconfig) as f:
config = ConfigParser.ConfigParser()
config.readfp(f)
if not config.has_section(os.path.basename(self.build_client)):
raise pyrpkg.rpkgError("Can't find the [%s] section in the "
"Koji config" % self.build_client)
self._fedora_cert_file = os.path.expanduser(
config.get(self.build_client, "cert"))
self._fedora_ca_cert = os.path.expanduser(
config.get(self.build_client, "serverca"))
def _findmasterbranch(self):
"""Find the right "nbrs" for master"""
# Create a list of "nbrses"
nbrses = []
# Create a regex to find branches that exactly match nb#.#. Should not
# catch branches such as nb5.0-foobar
branchre = r'nb\d$'
# Find the repo refs
for ref in self.repo.refs:
# Only find the remote refs
if type(ref) == git.refs.RemoteReference:
# Search for branch name by splitting off the remote
# part of the ref name and returning the rest. This may
# fail if somebody names a remote with / in the name...
if re.match(branchre, ref.name.split('/', 1)[1]):
# Add just the simple nb#.# part to the list
nbrses.append(ref.name.split('/')[1])
if nbrses:
# Sort the list...
nbrses.sort()
# ... so we can take the last one and strip it from its 'nb'...
latest_distval = nbrses[-1].strip('nb')
# ... so we can add 1 to the last one and recreate the new dist
return "%d" % (int(latest_distval)+1)
else:
# We may not have NBRSes. Find out what experimental target does.
try:
experimentaltarget = self.anon_kojisession.getBuildTarget(self.target)
except:
# We couldn't hit koji, bail.
raise pyrpkg.rpkgError("Unable to query koji to find " \
"experimental target")
desttag = experimentaltarget['dest_tag_name']
# Remove the trailing '-free' or '-nonfree'
desttag = desttag.split('-')[0]
return desttag.replace('nb', '')
def _findmasterbranch(self):
"""Find the right "fedora" for master"""
# If we already have a koji session, just get data from the source
if self._kojisession:
rawhidetarget = self.kojisession.getBuildTarget('rawhide')
desttag = rawhidetarget['dest_tag_name']
return desttag.replace('f', '')
# Create a list of "fedoras"
fedoras = []
# Create a regex to find branches that exactly match f##. Should not
# catch branches such as f14-foobar
branchre = 'f\d\d$'
# Find the repo refs
for ref in self.repo.refs:
# Only find the remote refs
if type(ref) == git.RemoteReference:
# Search for branch name by splitting off the remote
# part of the ref name and returning the rest. This may
# fail if somebody names a remote with / in the name...
if re.match(branchre, ref.name.split('/', 1)[1]):
# Add just the simple f## part to the list
fedoras.append(ref.name.split('/')[1])
if fedoras:
# Sort the list
fedoras.sort()
# Start with the last item, strip the f, add 1, return it.
return(int(fedoras[-1].strip('f')) + 1)
else:
# We may not have Fedoras. Find out what rawhide target does.
try:
rawhidetarget = self.anon_kojisession.getBuildTarget(
'rawhide')
except:
# We couldn't hit koji, bail.
raise pyrpkg.rpkgError('Unable to query koji to find rawhide \
target')
desttag = rawhidetarget['dest_tag_name']
return desttag.replace('f', '')
def _findmasterbranch(self):
"""Find the right "fedora" for master"""
# If we already have a koji session, just get data from the source
if self._kojisession:
rawhidetarget = self.kojisession.getBuildTarget('rawhide')
desttag = rawhidetarget['dest_tag_name']
return desttag.replace('f', '')
# Create a list of "fedoras"
fedoras = []
# Create a regex to find branches that exactly match f##. Should not
# catch branches such as f14-foobar
branchre = 'f\d\d$'
# Find the repo refs
for ref in self.repo.refs:
# Only find the remote refs
if type(ref) == git.RemoteReference:
# Search for branch name by splitting off the remote
# part of the ref name and returning the rest. This may
# fail if somebody names a remote with / in the name...
if re.match(branchre, ref.name.split('/', 1)[1]):
# Add just the simple f## part to the list
fedoras.append(ref.name.split('/')[1])
if fedoras:
# Sort the list
fedoras.sort()
# Start with the last item, strip the f, add 1, return it.
return (int(fedoras[-1].strip('f')) + 1)
else:
# We may not have Fedoras. Find out what rawhide target does.
try:
rawhidetarget = self.anon_kojisession.getBuildTarget('rawhide')
except:
# We couldn't hit koji, bail.
raise pyrpkg.rpkgError('Unable to query koji to find rawhide \
target')
desttag = rawhidetarget['dest_tag_name']
return desttag.replace('f', '')
def sl_list_to_dict(sls):
"""
Takes a list of SLs and returns them in a dictionary format. Any errors in
the SLs will be raised as an rpkgError.
:param sls: list of SLs in the format of sl_name:2017-12-25
:return: dictionary in the format of {'sl_name': '2017-12-25'}
"""
sl_dict = {}
# Ensures the SL is in the format "security_fixes:2020-01-01"
sl_regex = re.compile(r'^(.+)(?:\:)(\d{4}-\d{2}-\d{2})$')
for sl in sls:
sl_match = re.match(sl_regex, sl)
if sl_match:
sl_name = sl_match.groups()[0]
sl_date = sl_match.groups()[1]
sl_dict[sl_name] = sl_date
else:
raise rpkgError(
'The SL "{0}" is in an invalid format'.format(sl))
return sl_dict
def load_rpmdefines(self):
'''
Populate rpmdefines based on branch data
'''
try:
osver = re.search(r'\d.*$', self.branch_merge).group()
except AttributeError:
raise pyrpkg.rpkgError(
'Could not find the base OS ver from branch name'
' %s' % self.branch_merge)
self._distval = osver
self._distval = self._distval.replace('.', '_')
self._disttag = 'el%s' % self._distval
self._rpmdefines = [
"--define '_topdir {0}'".format(self.path),
"--define '_srcrpmdir {0}'".format(self.path),
"--define '_rpmdir {0}'".format(self.path),
"--define 'dist .{0}'".format(self._disttag),
# int and float this to remove the decimal
"--define '{0} 1'".format(self._disttag)
]
def get_review_bug(self, bug_id, namespace, pkg):
"""
Gets a Bugzilla bug representing a Fedora package review and does as
much validation as it can without authenticating to Bugzilla. This
function was inspired by:
https://github.com/fedora-infra/pkgdb2/blob/master/pkgdb2/api/extras.py
https://pagure.io/pkgdb-cli/blob/master/f/pkgdb2client/utils.py
:param bug_id: string or integer of the Bugzilla bug ID
:param namespace: string of the dist-git namespace
:param pkg: string of the package name
"""
try:
bug = self.client.getbug(bug_id)
except Exception as error:
raise rpkgError(
'The Bugzilla bug could not be verified. The following '
'error was encountered: {0}'.format(str(error)))
# Do some basic validation on the bug
pagure_namespace_to_component = {
'rpms': 'Package Review',
'container': 'Container Review',
'modules': 'Module Review',
'test-modules': 'Module Review'
}
pagure_namespace_to_product = {
'rpms': ['Fedora', 'Fedora EPEL'],
'container': ['Fedora Container Images'],
'modules': ['Fedora Modules'],
'test-modules': ['Fedora']
}
bz_proper_component = pagure_namespace_to_component.get(namespace)
bz_proper_products = pagure_namespace_to_product.get(namespace)
if bz_proper_component is None or bug.component != bz_proper_component:
raise rpkgError('The Bugzilla bug provided is not the proper type')
elif bug.product not in bz_proper_products:
raise rpkgError(
'The Bugzilla bug provided is not for "{0}"'.format(
'" or "'.join(bz_proper_products)))
elif bug.assigned_to in ['', None, '*****@*****.**']:
raise rpkgError(
'The Bugzilla bug provided is not assigned to anyone')
# Check if the review was approved
flag_set = False
for flag in bug.flags:
name, status = flag.get('name'), flag.get('status')
if name == 'fedora-review' and status == '+':
flag_set = True
update_dt = flag.get('modification_date')
if update_dt:
dt = datetime.strptime(update_dt.value, '%Y%m%dT%H:%M:%S')
delta = datetime.utcnow().date() - dt.date()
if delta.days > 60:
raise rpkgError('The Bugzilla bug\'s review was '
'approved over 60 days ago')
break
if not flag_set:
raise rpkgError('The Bugzilla bug is not approved yet')
# Check the format of the Bugzilla bug title
tmp_summary = bug.summary.partition(':')[2]
if not tmp_summary:
raise rpkgError(
'Invalid title for this Bugzilla bug (no ":" present)')
if ' - ' not in tmp_summary:
raise rpkgError(
'Invalid title for this Bugzilla bug (no "-" present)')
pkg_in_bug = tmp_summary.split(' - ', 1)[0].strip()
if pkg != pkg_in_bug:
error = ('The package in the Bugzilla bug "{0}" doesn\'t match '
'the one provided "{1}"'.format(pkg_in_bug, pkg))
raise rpkgError(error)
return bug
def sources(self, outdir=None):
"""Download source files"""
# See also:
# https://lists.fedoraproject.org/pipermail/buildsys/2014-July/004313.html
#
# in 'super' the sources function expects a file named 'sources' to be in the base directory.
# A patch has been sent to upstream to allow a more flexible location.
#
# This code doesn't work due to:
# archive.strip().split(' ', 1) # patch provided to upstream to fix
#
# url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module_name,
# file.replace(' ', '%20'),
# csum, file.replace(' ', '%20'))
#
#os.symlink(os.path.join(self.path, '.{0}.metadata'.format(self.module_name)), os.path.join(self.path, 'sources'))
#super(Commands, self).sources(outdir=None)
#os.unlink(os.path.join(self.path, 'sources'))
# The following is copied from rpkg/__init__.py:sources with minor changes
try:
archives = open(
os.path.join(self.path,
'.{0}.metadata'.format(self.module_name)),
'r').readlines()
except IOError as e:
raise pyrpkg.rpkgError('%s is not a valid repo: %s' %
(self.path, e))
# Default to putting the files where the module is
if not outdir:
outdir = self.path
for archive in archives:
try:
# This strip / split is kind a ugly, but checksums shouldn't have
# two spaces in them. sources file might need more structure in the
# future
csum, file = archive.strip().split(None, 1)
except ValueError:
raise pyrpkg.rpkgError('Malformed sources file.')
# The default lookaside hash is stored in centpkg.conf, but there is
# a mix of md5 and sha sums in the CentOS lookaside, here we divine
# which one we are using
sum_lengths = {
128: 'sha512',
64: 'sha256',
40: 'sha1',
32: 'md5',
}
self.lookasidehash = sum_lengths[len(csum)]
# If a directory is specified in the metadata file, append it to
# outdir
if os.path.dirname(file):
outdir = os.path.join(self.path, os.path.dirname(file))
file = os.path.basename(file)
# Create the output directory if it's not checked into git
if not os.path.exists(outdir):
self.log.info("Creating OUTDIR: {0}".format(outdir))
os.makedirs(outdir)
outfile = os.path.join(outdir, file)
# See if we already have a valid copy downloaded
if os.path.exists(outfile):
if self._verify_file(outfile, csum, self.lookasidehash):
continue
self.log.info("Downloading %s" % (file))
url = '%s/%s/%s/%s' % (
self.lookaside,
self.module_name,
self.branch_merge,
csum,
)
command = [
'curl', '-H', 'Pragma:', '-o', outfile, '-R', '-S', '--fail'
]
if self.quiet:
command.append('-s')
command.append(url)
self._run_command(command)
if not self._verify_file(outfile, csum, self.lookasidehash):
raise pyrpkg.rpkgError('%s failed checksum' % file)
return
