def parseSourceForgeLikeURL(self, scheme, host, path, query):
"""Extract the SourceForge-like base URLs and bug IDs.
Both path and hostname are considered. If the hostname
corresponds to one of the aliases for the SourceForge celebrity,
that celebrity will be returned (there can be only one
SourceForge instance in Launchpad).
"""
# We're only interested in URLs that look like they come from a
# *Forge bugtracker. The valid URL schemes are:
# * /support/tracker.php
# * /tracker/(index.php) (index.php part is optional)
# * /tracker2/(index.php) (index.php part is optional)
sf_path_re = re.compile(
'^\/(support\/tracker\.php|tracker2?\/(index\.php)?)$')
if (sf_path_re.match(path) is None):
return None
if not query.get('aid'):
return None
remote_bug = query['aid']
if remote_bug is None or not remote_bug.isdigit():
return None
# There's only one global SF instance registered in Launchpad,
# so we return that if the hostnames match.
sf_tracker = getUtility(ILaunchpadCelebrities).sourceforge_tracker
sf_hosts = [urlsplit(alias)[1] for alias in sf_tracker.aliases]
sf_hosts.append(urlsplit(sf_tracker.baseurl)[2])
if host in sf_hosts:
return sf_tracker.baseurl, remote_bug
else:
base_url = urlunsplit((scheme, host, '/', '', ''))
return base_url, remote_bug
def test_aliases(self):
"""Test that parsing SourceForge URLs works with the SF aliases."""
original_bug_url = self.bug_url
original_base_url = self.base_url
url_bits = urlsplit(original_bug_url)
sf_bugtracker = self.bugtracker_set.getByName(name='sf')
# Carry out all the applicable tests for each alias.
for alias in sf_bugtracker.aliases:
alias_bits = urlsplit(alias)
self.base_url = alias
bug_url_bits = (
alias_bits[0],
alias_bits[1],
url_bits[2],
url_bits[3],
url_bits[4],
)
self.bug_url = urlunsplit(bug_url_bits)
self.test_registered_tracker_url()
self.test_unknown_baseurl()
self.bug_url = original_bug_url
self.base_url = original_base_url
def parseSavaneURL(self, scheme, host, path, query):
"""Extract Savane base URL and bug ID."""
# We're only interested in URLs that look like they come from a
# Savane bugtracker. We currently accept URL paths /bugs/ or
# /bugs/index.php, and accept query strings that are just the bug ID
# or that have an item_id parameter containing the bug ID.
if path not in ('/bugs/', '/bugs/index.php'):
return None
if len(query) == 1 and query.values()[0] is None:
# The query string is just a bare ID.
remote_bug = query.keys()[0]
elif 'item_id' in query:
remote_bug = query['item_id']
else:
return None
if not remote_bug.isdigit():
return None
# There's only one global Savannah bugtracker registered with
# Launchpad, so we return that one if the hostname matches.
savannah_tracker = getUtility(ILaunchpadCelebrities).savannah_tracker
savannah_hosts = [
urlsplit(alias)[1] for alias in savannah_tracker.aliases
]
savannah_hosts.append(urlsplit(savannah_tracker.baseurl)[1])
if host in savannah_hosts:
return savannah_tracker.baseurl, remote_bug
else:
base_url = urlunsplit((scheme, host, '/', '', ''))
return base_url, remote_bug
def test_aliases(self):
"""Test that parsing SourceForge URLs works with the SF aliases."""
original_bug_url = self.bug_url
original_base_url = self.base_url
url_bits = urlsplit(original_bug_url)
sf_bugtracker = self.bugtracker_set.getByName(name='sf')
# Carry out all the applicable tests for each alias.
for alias in sf_bugtracker.aliases:
alias_bits = urlsplit(alias)
self.base_url = alias
bug_url_bits = (
alias_bits[0],
alias_bits[1],
url_bits[2],
url_bits[3],
url_bits[4],
)
self.bug_url = urlunsplit(bug_url_bits)
self.test_registered_tracker_url()
self.test_unknown_baseurl()
self.bug_url = original_bug_url
self.base_url = original_base_url
def parseSavaneURL(self, scheme, host, path, query):
"""Extract Savane base URL and bug ID."""
# Savane bugs URLs are in the form /bugs/?<bug-id>, so we
# exclude any path that isn't '/bugs/'. We also exclude query
# string that have a length of more or less than one, since in
# such cases we'd be taking a guess at the bug ID, which would
# probably be wrong.
if path != '/bugs/' or len(query) != 1:
return None
# There's only one global Savannah bugtracker registered with
# Launchpad, so we return that one if the hostname matches.
savannah_tracker = getUtility(ILaunchpadCelebrities).savannah_tracker
savannah_hosts = [
urlsplit(alias)[1] for alias in savannah_tracker.aliases
]
savannah_hosts.append(urlsplit(savannah_tracker.baseurl)[1])
# The remote bug is actually a key in the query dict rather than
# a value, so we simply use the first and only key we come
# across as a best-effort guess.
remote_bug = query.popitem()[0]
if remote_bug is None or not remote_bug.isdigit():
return None
if host in savannah_hosts:
return savannah_tracker.baseurl, remote_bug
else:
base_url = urlunsplit((scheme, host, '/', '', ''))
return base_url, remote_bug
def parseSavaneURL(self, scheme, host, path, query):
"""Extract Savane base URL and bug ID."""
# Savane bugs URLs are in the form /bugs/?<bug-id>, so we
# exclude any path that isn't '/bugs/'. We also exclude query
# string that have a length of more or less than one, since in
# such cases we'd be taking a guess at the bug ID, which would
# probably be wrong.
if path != '/bugs/' or len(query) != 1:
return None
# There's only one global Savannah bugtracker registered with
# Launchpad, so we return that one if the hostname matches.
savannah_tracker = getUtility(ILaunchpadCelebrities).savannah_tracker
savannah_hosts = [
urlsplit(alias)[1] for alias in savannah_tracker.aliases]
savannah_hosts.append(urlsplit(savannah_tracker.baseurl)[1])
# The remote bug is actually a key in the query dict rather than
# a value, so we simply use the first and only key we come
# across as a best-effort guess.
remote_bug = query.popitem()[0]
if remote_bug is None or not remote_bug.isdigit():
return None
if host in savannah_hosts:
return savannah_tracker.baseurl, remote_bug
else:
base_url = urlunsplit((scheme, host, '/', '', ''))
return base_url, remote_bug
def parseSourceForgeLikeURL(self, scheme, host, path, query):
"""Extract the SourceForge-like base URLs and bug IDs.
Both path and hostname are considered. If the hostname
corresponds to one of the aliases for the SourceForge celebrity,
that celebrity will be returned (there can be only one
SourceForge instance in Launchpad).
"""
# We're only interested in URLs that look like they come from a
# *Forge bugtracker. The valid URL schemes are:
# * /support/tracker.php
# * /tracker/(index.php) (index.php part is optional)
# * /tracker2/(index.php) (index.php part is optional)
sf_path_re = re.compile(
'^\/(support\/tracker\.php|tracker2?\/(index\.php)?)$')
if (sf_path_re.match(path) is None):
return None
if not query.get('aid'):
return None
remote_bug = query['aid']
if remote_bug is None or not remote_bug.isdigit():
return None
# There's only one global SF instance registered in Launchpad,
# so we return that if the hostnames match.
sf_tracker = getUtility(ILaunchpadCelebrities).sourceforge_tracker
sf_hosts = [urlsplit(alias)[1] for alias in sf_tracker.aliases]
sf_hosts.append(urlsplit(sf_tracker.baseurl)[2])
if host in sf_hosts:
return sf_tracker.baseurl, remote_bug
else:
base_url = urlunsplit((scheme, host, '/', '', ''))
return base_url, remote_bug
def addRemoteComment(self, remote_bug, comment_body, rfc822msgid):
"""Push a comment to the remote DebBugs instance.
See `ISupportsCommentPushing`.
"""
debian_bug = self._findBug(remote_bug)
# We set the subject to "Re: <bug subject>" in the same way that
# a mail client would.
subject = "Re: %s" % debian_bug.subject
host_name = urlsplit(self.baseurl)[1]
to_addr = "%s@%s" % (remote_bug, host_name)
headers = {'Message-Id': rfc822msgid}
# We str()ify to_addr since simple_sendmail expects ASCII
# strings and gets awfully upset when it gets a unicode one.
sent_msg_id = simple_sendmail('*****@*****.**',
[str(to_addr)],
subject,
comment_body,
headers=headers)
# We add angle-brackets to the sent_msg_id because
# simple_sendmail strips them out. We want to remain consistent
# with debbugs, which uses angle-brackets in its message IDS (as
# does Launchpad).
return "<%s>" % sent_msg_id
def addRemoteComment(self, remote_bug, comment_body, rfc822msgid):
"""Push a comment to the remote DebBugs instance.
See `ISupportsCommentPushing`.
"""
debian_bug = self._findBug(remote_bug)
# We set the subject to "Re: <bug subject>" in the same way that
# a mail client would.
subject = "Re: %s" % debian_bug.subject
host_name = urlsplit(self.baseurl)[1]
to_addr = "%s@%s" % (remote_bug, host_name)
headers = {'Message-Id': rfc822msgid}
# We str()ify to_addr since simple_sendmail expects ASCII
# strings and gets awfully upset when it gets a unicode one.
sent_msg_id = simple_sendmail(
'*****@*****.**', [str(to_addr)], subject,
comment_body, headers=headers)
# We add angle-brackets to the sent_msg_id because
# simple_sendmail strips them out. We want to remain consistent
# with debbugs, which uses angle-brackets in its message IDS (as
# does Launchpad).
return "<%s>" % sent_msg_id
def test_traverse(url):
"""Traverse the url in the same way normal publishing occurs.
Returns a tuple of (object, view, request) where:
object is the last model object in the traversal chain
view is the defined view for the object at the specified url (if
the url didn't directly specify a view, then the view is the
default view for the object.
request is the request object resulting from the traversal. This
contains a populated traversed_objects list just as a browser
request would from a normal call into the app servers.
This call uses the currently logged in user, and does not start a new
transaction.
"""
url_parts = urlsplit(url)
server_url = '://'.join(url_parts[0:2])
path_info = url_parts[2]
request, publication = get_request_and_publication(
host=url_parts[1], extra_environment={
'SERVER_URL': server_url,
'PATH_INFO': path_info})
request.setPublication(publication)
# We avoid calling publication.beforePublication because this starts a new
# transaction, which causes an abort of the existing transaction, and the
# removal of any created and uncommitted objects.
# Set the default layer.
adapters = getGlobalSiteManager().adapters
layer = adapters.lookup((providedBy(request),), IDefaultSkin, '')
if layer is not None:
layers.setAdditionalLayer(request, layer)
principal = get_current_principal()
if IUnauthenticatedPrincipal.providedBy(principal):
login = None
else:
login = principal.person
setupInteraction(principal, login, request)
getUtility(IOpenLaunchBag).clear()
app = publication.getApplication(request)
view = request.traverse(app)
# Find the object from the view instead on relying that it stays
# in the traversed_objects stack. That doesn't apply to the web
# service for example.
try:
obj = removeSecurityProxy(view).context
except AttributeError:
# But sometime the view didn't store the context...
# Use the last traversed object in these cases.
obj = request.traversed_objects[-2]
restoreInteraction()
return obj, view, request
def make_fake_request(url, traversed_objects=None):
"""Return a fake request object for menu testing.
:param traversed_objects: A list of objects that becomes the request's
traversed_objects attribute.
"""
url_parts = urlsplit(url)
server_url = '://'.join(url_parts[0:2])
path_info = url_parts[2]
request = LaunchpadTestRequest(
SERVER_URL=server_url,
PATH_INFO=path_info)
request._traversed_names = path_info.split('/')[1:]
if traversed_objects is not None:
request.traversed_objects = traversed_objects[:]
# After making the request, setup a new interaction.
endInteraction()
newInteraction(request)
return request
def _getDateForComment(self, parsed_comment):
"""Return the correct date for a comment.
:param parsed_comment: An `email.Message.Message` instance
containing a parsed DebBugs comment.
:return: The correct date to use for the comment contained in
`parsed_comment`. If a date is specified in a Received
header on `parsed_comment` that we can use, return that.
Otherwise, return the Date field of `parsed_comment`.
"""
# Check for a Received: header on the comment and use
# that to get the date, if possible. We only use the
# date received by this host (nominally bugs.debian.org)
# since that's the one that's likely to be correct.
received_headers = parsed_comment.get_all('received')
if received_headers is not None:
host_name = urlsplit(self.baseurl)[1]
received_headers = [
header for header in received_headers if host_name in header
]
# If there are too many - or too few - received headers then
# something's gone wrong and we default back to using
# the Date field.
if received_headers is not None and len(received_headers) == 1:
received_string = received_headers[0]
received_by, date_string = received_string.split(';', 2)
else:
date_string = parsed_comment['date']
# We parse the date_string if we can, otherwise we just return
# None.
if date_string is not None:
date_with_tz = parsedate_tz(date_string)
timestamp = mktime_tz(date_with_tz)
msg_date = datetime.fromtimestamp(timestamp,
tz=pytz.timezone('UTC'))
else:
msg_date = None
return msg_date
def _getDateForComment(self, parsed_comment):
"""Return the correct date for a comment.
:param parsed_comment: An `email.Message.Message` instance
containing a parsed DebBugs comment.
:return: The correct date to use for the comment contained in
`parsed_comment`. If a date is specified in a Received
header on `parsed_comment` that we can use, return that.
Otherwise, return the Date field of `parsed_comment`.
"""
# Check for a Received: header on the comment and use
# that to get the date, if possible. We only use the
# date received by this host (nominally bugs.debian.org)
# since that's the one that's likely to be correct.
received_headers = parsed_comment.get_all('received')
if received_headers is not None:
host_name = urlsplit(self.baseurl)[1]
received_headers = [
header for header in received_headers
if host_name in header]
# If there are too many - or too few - received headers then
# something's gone wrong and we default back to using
# the Date field.
if received_headers is not None and len(received_headers) == 1:
received_string = received_headers[0]
received_by, date_string = received_string.split(';', 2)
else:
date_string = parsed_comment['date']
# We parse the date_string if we can, otherwise we just return
# None.
if date_string is not None:
date_with_tz = parsedate_tz(date_string)
timestamp = mktime_tz(date_with_tz)
msg_date = datetime.fromtimestamp(timestamp,
tz=pytz.timezone('UTC'))
else:
msg_date = None
return msg_date
def extractBugTrackerAndBug(self, url):
"""See `IBugWatchSet`."""
for trackertype, parse_func in (
self.bugtracker_parse_functions.items()):
scheme, host, path, query_string, frag = urlsplit(url)
query = {}
for query_part in query_string.split('&'):
key, value = urllib.splitvalue(query_part)
query[key] = value
bugtracker_data = parse_func(scheme, host, path, query)
if not bugtracker_data:
continue
base_url, remote_bug = bugtracker_data
# Check whether we have a registered bug tracker already.
bugtracker = getUtility(IBugTrackerSet).queryByBaseURL(base_url)
if bugtracker is not None:
return bugtracker, remote_bug
else:
raise NoBugTrackerFound(base_url, remote_bug, trackertype)
raise UnrecognizedBugTrackerURL(url)
def extractBugTrackerAndBug(self, url):
"""See `IBugWatchSet`."""
for trackertype, parse_func in (
self.bugtracker_parse_functions.items()):
scheme, host, path, query_string, frag = urlsplit(url)
query = {}
for query_part in query_string.split('&'):
key, value = urllib.splitvalue(query_part)
query[key] = value
bugtracker_data = parse_func(scheme, host, path, query)
if not bugtracker_data:
continue
base_url, remote_bug = bugtracker_data
# Check whether we have a registered bug tracker already.
bugtracker = getUtility(IBugTrackerSet).queryByBaseURL(base_url)
if bugtracker is not None:
return bugtracker, remote_bug
else:
raise NoBugTrackerFound(base_url, remote_bug, trackertype)
raise UnrecognizedBugTrackerURL(url)
def initializeRemoteBugDB(self, bug_ids):
"""See `ExternalBugTracker`.
We override this method because SourceForge does not provide a
nice way for us to export bug statuses en masse. Instead, we
resort to screen-scraping on a per-bug basis. Therefore the
usual choice of batch vs. single export does not apply here and
we only perform single exports.
"""
self.bugs = {}
for bug_id in bug_ids:
query_url = self.export_url % bug_id
page_data = self._getPage(query_url)
soup = BeautifulSoup(page_data)
status_tag = soup.find(text=re.compile("Status:"))
status = None
private = False
if status_tag:
# We can extract the status by finding the grandparent tag.
# Happily, BeautifulSoup will turn the contents of this tag
# into a newline-delimited list from which we can then
# extract the requisite data.
status_row = status_tag.findParent().findParent()
status = status_row.contents[-1]
status = status.strip()
else:
error_message = self._extractErrorMessage(page_data)
# If the error message suggests that the bug is private,
# set the bug's private field to True.
# XXX 2008-05-01 gmb bug=225354:
# We should know more about possible errors and deal
# with them accordingly.
if error_message and "private" in error_message.lower():
private = True
else:
# If we can't find a status line in the output from
# SourceForge there's little point in continuing.
raise UnparsableBugData("Remote bug %s does not define a status." % bug_id)
# We need to do the same for Resolution, though if we can't
# find it it's not critical.
resolution_tag = soup.find(text=re.compile("Resolution:"))
if resolution_tag:
resolution_row = resolution_tag.findParent().findParent()
resolution = resolution_row.contents[-1]
resolution = resolution.strip()
else:
resolution = None
# We save the group_id and atid parameters from the
# query_url. They'll be returned by getRemoteProduct().
query_dict = {}
bugtracker_link = soup.find("a", text="Bugs")
if bugtracker_link:
href = bugtracker_link.findParent()["href"]
# We need to replace encoded ampersands in the URL since
# SourceForge occasionally encodes them.
href = href.replace("&", "&")
schema, host, path, query, fragment = urlsplit(href)
query_bits = query.split("&")
for bit in query_bits:
key, value = urllib.splitvalue(bit)
query_dict[key] = value
try:
atid = int(query_dict.get("atid", None))
group_id = int(query_dict.get("group_id", None))
except ValueError:
atid = None
group_id = None
else:
group_id = None
atid = None
self.bugs[int(bug_id)] = {
"id": int(bug_id),
"private": private,
"status": status,
"resolution": resolution,
"group_id": group_id,
"atid": atid,
}
def getRemoteProductFromSourceForge(self, sf_project):
"""Return the remote product of a SourceForge project.
:return: The group_id and atid of the SourceForge project's bug
tracker as an ampersand-separated string in the form
'group_id&atid'.
"""
# First, fetch the project page.
try:
soup = BeautifulSoup(self._getPage("projects/%s" % sf_project))
except HTTPError as error:
self.logger.error(
"Error fetching project %s: %s" %
(sf_project, error))
return None
# Find the Tracker link and fetch that.
tracker_link = soup.find('a', text='Tracker')
if tracker_link is None:
self.logger.error(
"No tracker link for project '%s'" % sf_project)
return None
tracker_url = tracker_link.findParent()['href']
# Clean any leading '/' from tracker_url so that urlappend
# doesn't choke on it.
tracker_url = tracker_url.lstrip('/')
try:
soup = BeautifulSoup(self._getPage(tracker_url))
except HTTPError as error:
self.logger.error(
"Error fetching project %s: %s" %
(sf_project, error))
return None
# Extract the group_id and atid from the bug tracker link.
bugtracker_link = soup.find('a', text='Bugs')
if bugtracker_link is None:
self.logger.error(
"No bug tracker link for project '%s'" % sf_project)
return None
bugtracker_url = bugtracker_link.findParent()['href']
# We need to replace encoded ampersands in the URL since
# SourceForge usually encodes them.
bugtracker_url = bugtracker_url.replace('&', '&')
schema, host, path, query, fragment = urlsplit(bugtracker_url)
query_dict = {}
query_bits = query.split('&')
for bit in query_bits:
key, value = urllib.splitvalue(bit)
query_dict[key] = value
try:
atid = int(query_dict.get('atid', None))
group_id = int(query_dict.get('group_id', None))
except ValueError:
# If anything goes wrong when int()ing the IDs, just return
# None.
return None
return u'%s&%s' % (group_id, atid)
def test_traverse(url):
"""Traverse the url in the same way normal publishing occurs.
Returns a tuple of (object, view, request) where:
object is the last model object in the traversal chain
view is the defined view for the object at the specified url (if
the url didn't directly specify a view, then the view is the
default view for the object.
request is the request object resulting from the traversal. This
contains a populated traversed_objects list just as a browser
request would from a normal call into the app servers.
This call uses the currently logged in user, and does not start a new
transaction.
"""
url_parts = urlsplit(url)
server_url = '://'.join(url_parts[0:2])
path_info = url_parts[2]
request, publication = get_request_and_publication(host=url_parts[1],
extra_environment={
'SERVER_URL':
server_url,
'PATH_INFO':
path_info
})
request.setPublication(publication)
# We avoid calling publication.beforePublication because this starts a new
# transaction, which causes an abort of the existing transaction, and the
# removal of any created and uncommitted objects.
# Set the default layer.
adapters = getGlobalSiteManager().adapters
layer = adapters.lookup((providedBy(request), ), IDefaultSkin, '')
if layer is not None:
layers.setAdditionalLayer(request, layer)
principal = get_current_principal()
if IUnauthenticatedPrincipal.providedBy(principal):
login = None
else:
login = principal.person
setupInteraction(principal, login, request)
getUtility(IOpenLaunchBag).clear()
app = publication.getApplication(request)
view = request.traverse(app)
# Find the object from the view instead on relying that it stays
# in the traversed_objects stack. That doesn't apply to the web
# service for example.
try:
obj = removeSecurityProxy(view).context
except AttributeError:
# But sometime the view didn't store the context...
# Use the last traversed object in these cases.
obj = request.traversed_objects[-2]
restoreInteraction()
return obj, view, request
def initializeRemoteBugDB(self, bug_ids):
"""See `ExternalBugTracker`.
We override this method because SourceForge does not provide a
nice way for us to export bug statuses en masse. Instead, we
resort to screen-scraping on a per-bug basis. Therefore the
usual choice of batch vs. single export does not apply here and
we only perform single exports.
"""
self.bugs = {}
for bug_id in bug_ids:
query_url = self.export_url % bug_id
page_data = self._getPage(query_url)
soup = BeautifulSoup(page_data)
status_tag = soup.find(text=re.compile('Status:'))
status = None
private = False
if status_tag:
# We can extract the status by finding the grandparent tag.
# Happily, BeautifulSoup will turn the contents of this tag
# into a newline-delimited list from which we can then
# extract the requisite data.
status_row = status_tag.findParent().findParent()
status = status_row.contents[-1]
status = status.strip()
else:
error_message = self._extractErrorMessage(page_data)
# If the error message suggests that the bug is private,
# set the bug's private field to True.
# XXX 2008-05-01 gmb bug=225354:
# We should know more about possible errors and deal
# with them accordingly.
if error_message and 'private' in error_message.lower():
private = True
else:
# If we can't find a status line in the output from
# SourceForge there's little point in continuing.
raise UnparsableBugData(
'Remote bug %s does not define a status.' % bug_id)
# We need to do the same for Resolution, though if we can't
# find it it's not critical.
resolution_tag = soup.find(text=re.compile('Resolution:'))
if resolution_tag:
resolution_row = resolution_tag.findParent().findParent()
resolution = resolution_row.contents[-1]
resolution = resolution.strip()
else:
resolution = None
# We save the group_id and atid parameters from the
# query_url. They'll be returned by getRemoteProduct().
query_dict = {}
bugtracker_link = soup.find('a', text='Bugs')
if bugtracker_link:
href = bugtracker_link.findParent()['href']
# We need to replace encoded ampersands in the URL since
# SourceForge occasionally encodes them.
href = href.replace('&', '&')
schema, host, path, query, fragment = urlsplit(href)
query_bits = query.split('&')
for bit in query_bits:
key, value = urllib.splitvalue(bit)
query_dict[key] = value
try:
atid = int(query_dict.get('atid', None))
group_id = int(query_dict.get('group_id', None))
except ValueError:
atid = None
group_id = None
else:
group_id = None
atid = None
self.bugs[int(bug_id)] = {
'id': int(bug_id),
'private': private,
'status': status,
'resolution': resolution,
'group_id': group_id,
'atid': atid,
}
def getRemoteProductFromSourceForge(self, sf_project):
"""Return the remote product of a SourceForge project.
:return: The group_id and atid of the SourceForge project's bug
tracker as an ampersand-separated string in the form
'group_id&atid'.
"""
# First, fetch the project page.
try:
soup = BeautifulSoup(self._getPage("projects/%s" % sf_project))
except requests.HTTPError as error:
self.logger.error("Error fetching project %s: %s" %
(sf_project, error))
return None
# Find the Tracker link and fetch that.
tracker_link = soup.find('a', text='Tracker')
if tracker_link is None:
self.logger.error("No tracker link for project '%s'" % sf_project)
return None
tracker_url = tracker_link.findParent()['href']
# Clean any leading '/' from tracker_url so that urlappend
# doesn't choke on it.
tracker_url = tracker_url.lstrip('/')
try:
soup = BeautifulSoup(self._getPage(tracker_url))
except requests.HTTPError as error:
self.logger.error("Error fetching project %s: %s" %
(sf_project, error))
return None
# Extract the group_id and atid from the bug tracker link.
bugtracker_link = soup.find('a', text='Bugs')
if bugtracker_link is None:
self.logger.error("No bug tracker link for project '%s'" %
sf_project)
return None
bugtracker_url = bugtracker_link.findParent()['href']
# We need to replace encoded ampersands in the URL since
# SourceForge usually encodes them.
bugtracker_url = bugtracker_url.replace('&', '&')
schema, host, path, query, fragment = urlsplit(bugtracker_url)
query_dict = {}
query_bits = query.split('&')
for bit in query_bits:
key, value = urllib.splitvalue(bit)
query_dict[key] = value
try:
atid = int(query_dict.get('atid', None))
group_id = int(query_dict.get('group_id', None))
except ValueError:
# If anything goes wrong when int()ing the IDs, just return
# None.
return None
return u'%s&%s' % (group_id, atid)
