def xml_tree_equal_to_xml_string(et, xml_s: str) -> bool:
xml_actual = tostring(et, encoding="unicode")
actions = diff_texts(xml_actual, xml_s)
if actions:
print(actions)
# No actions indicates that xmldiff thinks these two XML documents are equal.
return len(actions) == 0
def test_recipe_set_to_job_whst(self):
"""Ensure __recipe_set_to_job() works with hostname."""
# pylint: disable=W0212,E1101
beaker_xml = """<recipeSet><recipe><hostRequires>
<hostname op="!=" value="hst1"/></hostRequires></recipe></recipeSet>"""
xml_parsed = fromstring(beaker_xml)
result = self.myrunner._BeakerRunner__recipe_set_to_job(xml_parsed)
# check that <hostname op="!=" value="hst1"/> wasn't removed
self.assertEqual(len(result.findall('.//hostname')), 1)
self.assertEqual(
tostring(result.find('.//hostname')).decode(),
'<hostname op="!=" value="hst1" />')
def get_text_content_from_message(message: V4Message) -> str:
"""Get text content from PresentationML on incoming messages
:param message: message containing the PresentationML to be parsed
:return: the message text content extracted from the given PresentationML
"""
try:
escaped_text_content = tostring(fromstring(message.message),
method="text").decode().strip()
return html.unescape(escaped_text_content)
except ParseError as exc:
raise MessageParserError(
"Unable to parse the PresentationML, it is not in the correct format."
) from exc
def test_blacklist_test_force(self):
""" Ensure blacklist_hreq does not override when force= is set."""
# pylint: disable=W0212,E1101
initial = """<hostRequires force="srv" />"""
hreq_node = fromstring(initial)
# load blacklist ['host1', 'host2']
self.test_load_blacklist()
etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
# result must be the same as initial
result = tostring(etree_result).decode('utf-8')
self.assertEqual(re.sub(r'[\s]+', '', initial),
re.sub(r'[\s]+', '', result))
def test_blacklist_hreq_nohostnames(self):
""" Ensure blacklist_hreq works without hostnames."""
# pylint: disable=W0212,E1101
initial = """<hostRequires><system_type value="Machine"/><and>
<hypervisor op="=" value=""/></and></hostRequires>"""
exp_result = """<hostRequires><system_type value="Machine"/><and>
<hypervisor op="=" value=""/></and></hostRequires>"""
hreq_node = fromstring(initial)
self.myrunner.blacklisted = []
etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
result = tostring(etree_result).decode('utf-8')
self.assertEqual(re.sub(r'[\s]+', '', exp_result),
re.sub(r'[\s]+', '', result))
def test_blacklist_hreq_noand(self):
""" Ensure blacklist_hreq works without <and> element."""
# pylint: disable=W0212,E1101
initial = """<hostRequires></hostRequires>"""
exp_result = """<hostRequires><and>
<hostname op="!=" value="host1"/>
<hostname op="!=" value="host2"/></and></hostRequires>"""
hreq_node = fromstring(initial)
# load blacklist ['host1', 'host2']
self.test_load_blacklist()
etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
result = tostring(etree_result).decode('utf-8')
self.assertEqual(re.sub(r'[\s]+', '', exp_result),
re.sub(r'[\s]+', '', result))
def _redact_any_type(xml: T,
sensitive_word: T,
replacement: T,
encoding=None) -> T:
try:
root = fromstring(xml)
matches = root.findall(".//*[@password]")
for item in matches:
item.attrib["password"] = "******"
matches = root.findall(".//password")
for item in matches:
item.text = "********"
# tostring returns bytes unless an encoding value is passed
return tostring(root, encoding=encoding)
except Exception:
# something about the xml handling failed. Just cut off the text at the first occurrence of "password"
location = xml.find(sensitive_word)
return xml[:location] + replacement
def test_blacklist_hreq_whnames(self):
""" Ensure blacklist_hreq works with hostnames."""
# pylint: disable=W0212,E1101
initial = """<hostRequires><system_type value="Machine"/><and>
<hypervisor op="=" value=""/></and></hostRequires>"""
exp_result = """<hostRequires><system_type value="Machine"/><and>
<hypervisor op="=" value=""/><hostname op="!=" value="host1"/>
<hostname op="!=" value="host2"/></and></hostRequires>"""
hreq_node = fromstring(initial)
# load blacklist ['host1', 'host2']
self.test_load_blacklist()
etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
result = tostring(etree_result)
self.assertEqual(re.sub(r'[\s]+', '', exp_result),
re.sub(r'[\s]+', '', result))
def __handle_test_abort(self, recipe, recipe_id, recipe_set_id, root):
if self._not_booting(recipe):
return
retval, _ = self.decide_run_result_by_task(recipe, recipe_id)
if retval == SKT_SUCCESS:
# A task that is waived aborted or panicked. Waived tasks are
# appended to the end of the recipe, so we should be able to
# safely ignore this.
return
logging.warning('%s from %s aborted!', recipe_id, recipe_set_id)
self.aborted_count += 1
if self.aborted_count < self.max_aborted:
logging.warning('Resubmitting aborted %s', recipe_set_id)
newjob = self.__recipe_set_to_job(root)
newjobid = self.__jobsubmit(tostring(newjob))
self.__add_to_watchlist(newjobid)
self.watchlist.discard(recipe_set_id)
def encode(self, root, charset=None, mimetype=None): return tostring(root, charset)
def run(self,
url,
max_aborted,
release,
wait=False,
arch=platform.machine()):
"""
Run tests in Beaker.
Args:
url: URL pointing to kernel tarball.
max_aborted: Maximum number of allowed aborted jobs. Abort the
whole stage if the number is reached.
release: NVR of the tested kernel.
wait: False if skt should exit after submitting the jobs,
True if it should wait for them to finish.
arch: Architecture of the machine the tests should run on,
in a format accepted by Beaker. Defaults to
architecture of the current machine skt is running on
if not specified.
Returns:
ret where ret can be
SKT_SUCCESS if everything passed
SKT_FAIL if testing failed
SKT_ERROR in case of infrastructure error (exceptions are
logged)
SKT_BOOT if the boot test failed
"""
# pylint: disable=too-many-arguments
self.watchlist = set()
self.job_to_recipe_set_map = {}
self.recipe_set_results = {}
self.completed_recipes = {}
self.aborted_count = 0
self.max_aborted = max_aborted
try:
text = pathlib.Path(self.template).read_text()
job_xml_tree = fromstring(text)
# add blacklist to all recipes
self.add_blacklist2recipes(job_xml_tree)
# convert etree to xml and submit the job to Beaker
jobid = self.__jobsubmit(tostring(job_xml_tree))
if wait:
# wait for completion, resubmit jobs as needed
self.wait(jobid)
# get return code and report it
self.retcode = self.__getresults()
logging.debug("Got return code when gathering results: %s",
self.retcode)
else:
# not waiting -> change retcode to success
self.retcode = SKT_SUCCESS
except (Exception, BaseException) as e:
if isinstance(e, SystemExit):
sys.stderr.write('SystemExit exception caught\n')
raise
else:
exc = sys.exc_info()
logging.error('\n'.join(traceback.format_exception(*exc)))
return self.retcode
def __watchloop(self):
while self.watchlist:
time.sleep(self.watchdelay)
if self.max_aborted <= self.aborted_count:
self.has_aborted = True
# Remove / cancel all the remaining recipe set IDs and abort
self.cancel_pending_jobs()
return
for recipe_set_id in self.watchlist.copy():
root = self.getresultstree(recipe_set_id)
recipes = root.findall('.//recipe')
for recipe in recipes:
result = recipe.attrib.get('result')
status = recipe.attrib.get('status')
recipe_id = 'R:' + recipe.attrib.get('id')
if status not in ['Completed', 'Aborted', 'Cancelled'] or \
recipe_id in self.completed_recipes[recipe_set_id]:
# continue watching unfinished recipes
continue
logging.info("%s status changed to %s", recipe_id, status)
self.completed_recipes[recipe_set_id].add(recipe_id)
if len(self.completed_recipes[recipe_set_id]) == \
len(recipes):
try:
self.watchlist.remove(recipe_set_id)
except KeyError:
pass
self.recipe_set_results[recipe_set_id] = root
if result == 'Pass':
# some recipe passed, nothing to do here
continue
if status == 'Cancelled':
# job got cancelled for some reason, there's probably
# an external reason
logging.error('Cancelled run detected! Cancelling the '
'rest of runs and aborting!')
self.cancel_pending_jobs()
return
if result == 'Warn' and status == 'Aborted':
self.__handle_test_abort(recipe, recipe_id,
recipe_set_id, root)
continue
# check for test failure
test_failure, waive_skip = \
self.__handle_test_fail(recipe, recipe_id)
if waive_skip:
logging.info("recipe %s waived task(s) failed",
recipe_id)
continue
if not test_failure:
# Recipe failed before the tested kernel was installed
self.__forget_taskspec(recipe_set_id)
self.aborted_count += 1
if self.aborted_count < self.max_aborted:
logging.warning(
'Infrastructure-related problem '
'found, resubmitting %s', recipe_set_id)
newjob = self.__recipe_set_to_job(root)
newjobid = self.__jobsubmit(tostring(newjob))
self.__add_to_watchlist(newjobid)
def create_medusa_textcard(card, display_state):
"""
All news and features are drawn here. For condensing content,
wrap any nested image inside a "read more" bracket that will appear
after the 1st paragraph of text. Hide images behind this link too.
Fill out the next/previous type links based on #anchors done with a
descending number. If the next page of content isn't loaded, make
the link also load the next page of content before the anchor tag
is followed.
"""
# TODO: VET title and topics for reasonable size restrictions
topic_header = ""
for topic in card.topics:
topic_link = """<a class="topicLink" href="javascript:">%s</a>""" % topic
if topic_header == "":
topic_header = topic_link
else:
topic_header = topic_header + ", " + topic_link
anchor = card.cfile.split('/').pop()
# The body of a text card consists of paragraphs of text, possibly
# with a starting image tag. Wrap all tags other than the first
# paragraph tag with a "Expand" style that will keep that text
# properly hidden. Then, after the end of the first paragraph, add
# a "Read More" link for expanding the other hidden items.
output = ""
output += """<div class="card %s" id="%s">\n""" % (card.ctype, anchor)
output += """ <div class="cardTitle">\n"""
if card.permalink is True:
output += """ <h2>%s</h2>\n""" % card.title
output += """ <p class="subject">%s</p>\n""" % card.cdate
else:
output += """ <h2><a href="#%s" onclick="revealToggle('%s');">%s</a></h2>\n""" % (
anchor, anchor, card.title)
output += """ <p class="subject">%s</p>\n""" % topic_header
output += """ </div>\n"""
passed = {}
body_lines = card.body.splitlines()
processed_lines = unroll_newlines(body_lines)
first_line = processed_lines[0]
# Count all the <p> tags. If there's less than three paragraphs, don't do the
# "Read More" logic for that item.
ptags = count_ptags(processed_lines)
for line in processed_lines:
# Parsing the whole page only works for full HTML pages
# Pass tags we don't care about
if line.find('<img') != 0 and line.find('<p') != 0:
output += line + "\n"
continue
# syslog.syslog(line)
e = fromstring(escape_amp(line))
if e.tag == 'img':
if (line == first_line) and ('img' not in passed):
# Check image size. If it's the first line in the body and
# it's relatively small, display with the first paragraph.
# Add the dot in front to let the URIs look absolute, but the
# Python directories be relative to the image folder in the
# private contents directory (not exposed when auth is used)
img = Image.open("../private" + e.attrib['src'])
if ((img.size[0] > 300) and (img.size[1] > 220)
and (card.permalink is False)
and (card.search_result is False) and (ptags >= 3)):
if 'class' in e.attrib:
e.attrib['class'] += " imgExpand"
else:
e.attrib['class'] = "imgExpand"
elif ((ptags >= 3) and (card.permalink is False)
and (card.search_result is False)):
# Add a showExtend tag to hide it
if 'class' in e.attrib:
e.attrib['class'] += " imgExpand"
else:
e.attrib['class'] = "imgExpand"
else:
pass
# Track that we saw an img tag, and write the tag out
output += unescape(tostring(e, encoding="unicode"))
passed.update({'img': True})
elif e.tag == 'p':
# If further than the first paragraph, write output
if 'p' in passed:
output += unescape(tostring(e, encoding="unicode"))
# If more than three paragraphs, and it's a news entry,
# start hiding extra paragraphs from view
elif ((ptags >= 3) and (card.permalink is False)
and (card.search_result is False)):
# First <p> is OK, but follow it with a (Read More) link, and a
# div with showExtend that hides all the other elements
read_more = """ <a href="#%s" class="showShort" onclick="revealToggle('%s');">(Read More...)</a>""" % (
anchor, anchor)
prep = unescape(tostring(e, encoding="unicode"))
output += prep.replace('</p>', read_more + '</p>')
output += """<div class="divExpand">\n"""
else:
output += unescape(tostring(e, encoding="unicode"))
# Track that we saw an img tag, and write the tag out
passed.update({'p': True})
else:
# pass all other tags unmodified
output += line + "\n"
# End loop. now close the showExtend div if we
# added it earlier during tag processing
if ((ptags >= 3) and (card.permalink is False)
and (card.search_result is False)):
output += """ </div>\n"""
# Convert the appearance value into a string for permalinks
# And close the textcard
if display_state is not None:
permanchor = "/?x" + card.ctype[0] + anchor + ":" + display_state
else:
permanchor = "/?x" + card.ctype[0] + anchor
if card.permalink is False:
output += """ <div class="cardFooter">\n"""
output += """ <div class="bottom">\n"""
output += """ <p class="cardNav"><a href="%s">Permalink</a></p>\n""" % permanchor
output += """ <p class="postDate">%s</p>\n""" % card.cdate
output += """ </div>\n"""
output += """ </div>\n"""
output += """</div>\n"""
return output
def _prettyprint(etree):
data = parseString(tostring(etree))
print(data.toprettyxml())