def _ensure_wrapped_in_paragraph(contents):
if len(contents) == 1 and contents[0].name == 'p':
return contents[0]
else:
paragraph = Tag(name='p')
paragraph.contents = contents
return paragraph
def apply_solution(self, flux_dict, color_map=default_color_map):
self.object_styles.clear()
fluxes = dict((i, flux_dict[i]) for i in self.included_reactions.intersection(flux_dict))
abs_fluxes = [min(abs(i), 20) for i in fluxes.itervalues()]
x_min = min(abs_fluxes)
x_max = max(abs_fluxes)
scale_func = lambda value: min(1, (abs(value) - x_min) / (x_max - x_min) * 3)
for reaction, value in fluxes.iteritems():
#t = Tag(name="title")
#t.string = "%.2f" % (value)
self._rxn_layer.findChild("g", id=reaction).title.string += "\n%.2f" % (value)#append(t)
try:
t = Tag(name="title")
t.string = "%.2f" % (value)
self._rxn_label_layer.findChild(name="text", text=compile(reaction)).append(t)
except: None
if str(reaction) in self.included_reactions:
self.set_object_color(str(reaction), color_map(scale_func(value)))
if value < 0:
self.object_styles["%s .end" % str(reaction)] = {"marker-end": "none"}
if value > 0:
self.object_styles["%s .start" % str(reaction)] = {"marker-start": "none"}
for reaction in self.included_reactions.difference(flux_dict.keys()):
self.set_object_color(reaction, (0, 0, 0))
self._update_svg()
return self
def add_latex(self, html):
""""""
import re
html = re.sub("________", "", html)
html = re.sub(r"\\rm", r"", html) # r"\\text{", html)
html = re.sub(r"-", r"-", html)
html = re.sub(r"\\gt", r'>', html)
html = re.sub(r"\\lt", r'<', html)
html = re.sub(r"2!", r'2\!', html)
html = re.sub(r",!", r",\!", html)
html = re.sub(r"c!", r"c\!", html)
html = re.sub(r"!\cdot!", r"\!-\cdot-\!", html)
html = re.sub("\!$$", "$$", html)
html = re.sub(r"2\\\\!\\!", "2", html)
html = re.sub(r"\|", r"|", html)
html = re.sub(r"(% ((<)|<)!\[CDATA\[\n)", r"", html)
html = re.sub(r"( %]](>|>))", r"", html)
with open('index2.txt', 'w+') as output:
output.write(html)
soup = BeautifulSoup(html, 'html.parser')
for nav_string in soup(text=True):
if isinstance(nav_string, CData):
tag = Tag(soup, name="math")
tag.insert(0, nav_string[:])
nav_string.replace_with(tag)
self.add_clean_markdown(soup, 'span', attribs={"data-math": True})
self.add_clean_markdown(soup, 'div', attribs={"data-math": True})
self.add_clean_markdown(soup, 'div', tag_class='stem_text')
self.add_clean_markdown(soup, 'div', tag_class='answer_text')
self.add_clean_markdown(soup, 'div', tag_class='feedback')
return soup
def get_svg(xml,sketchID,version):
root = ET.fromstring(xml)
result_soup = BeautifulSoup()
for kobject in root.findall('.//KObject'):
objectID = kobject.attrib['id']
parent = kobject.find('parent')
parentID = parent.attrib['id']
stroke = kobject.find('strokeData')
if stroke is not None:
path = ksketchsvg.get_polyline(stroke)
color = ksketchsvg.convert_color(stroke.attrib['color'])
thickness = stroke.attrib['thickness']
tag = ksketchsvg.createTag(objectID, path, color, thickness, kobject.attrib['centroid'])
if parentID == "0":
result_soup.insert(len(result_soup.find_all('g', recursive=False)), tag)
else:
grp = result_soup.find('g', {'id': parentID})
if grp:
grp.insert(len(grp.find_all('g', recursive=False)), tag)
else:
tag = ksketchsvg.createGroup(objectID)
if parentID == "0":
result_soup.insert(len(result_soup.find_all('g', recursive=False)), tag)
else:
grp = result_soup.find('g', {'id': parentID})
if grp:
grp.insert(len(grp.find_all('g', recursive=False)), tag)
soup = BeautifulSoup()
g_tag = Tag(soup, name='g')
g_tag['id'] = "0"
g_tag.insert(0, result_soup)
SVGCache.addSVGData(sketchID,version,g_tag.prettify())
return g_tag.prettify()
def makeTag(name, string=None, **kwargs):
tag = Tag(name=name, attrs=kwargs)
for key in kwargs:
if kwargs[key] is None:
kwargs[key] = ""
if string:
tag.string = string
return tag
def add_latex(self, html):
""""""
import re
html = re.sub("________", "", html)
html = re.sub(r"\\rm", r"", html) # r"\\text{", html)
html = re.sub(r"–", r"-", html)
html = re.sub(r"\\gt", r'>', html)
html = re.sub(r"\\lt", r'<', html)
html = re.sub(r"2!", r'2\!', html)
html = re.sub(r",!", r",\!", html)
html = re.sub(r"c!", r"c\!", html)
html = re.sub(r"!\cdot!", r"\!-\cdot-\!", html)
html = re.sub("\!$$", "$$", html)
html = re.sub(r"2\\\\!\\!", "2", html)
html = re.sub(r"\|", r"|", html)
html = re.sub(r"(% ((<)|<)!\[CDATA\[\n)", r"", html)
html = re.sub(r"( %]](>|>))", r"", html)
with open('index2.txt', 'w+') as output:
output.write(html)
soup = BeautifulSoup(html, 'html.parser')
for nav_string in soup(text=True):
if isinstance(nav_string, CData):
tag = Tag(soup, name="math")
tag.insert(0, nav_string[:])
nav_string.replace_with(tag)
self.add_clean_markdown(soup, 'span', attribs={"data-math": True})
# for tag in soup.find_all('span', attrs={"data-math": True}):
# tag['data-math'] = '$' + tag['data-math'] + '$'
# try:
# tag.string.replace_with('$' + tag.string + '$')
# except Exception as e:
# print(e, tag, tag.string)
self.add_clean_markdown(soup, 'div', attribs={"data-math": True})
# for tag in soup.find_all('div', attrs={"data-math": True}):
# tag['data-math'] = '$$' + tag['data-math'] + '$$'
# try:
# tag.string.replace_with('$$' + tag.string + '$$')
# except Exception as e:
# print(e, tag, tag.string)
self.add_clean_markdown(soup, 'div', tag_class='stem_text')
# for tag in soup.find_all('div', class_='stem_text'):
# try:
# tag.string = bleach.clean(markdown.markdown(tag.string))
# except Exception:
# print("Issue: %s : %s" % (tag, tag.string))
self.add_clean_markdown(soup, 'div', tag_class='answer_text')
# for tag in soup.find_all('div', class_='answer_text'):
# try:
# tag.string = bleach.clean(markdown.markdown(tag.string))
# except Exception:
# print("Issue: %s : %s" % (tag, tag.string))
self.add_clean_markdown(soup, 'div', tag_class='feedback')
# for tag in soup.find_all('div', class_='feedback'):
# try:
# tag.string = bleach.clean(markdown.markdown(tag.string))
# except Exception:
# print("Issue: %s : %s" % (tag, tag.string))
return soup.prettify(formatter='html')
def __call__(self, DOM):
tag = Tag(name='script', builder=BUILDER)
tag.attrs = {
'type': 'text/javascript',
'src': self.url,
}
if not DOM.body:
DOM.html.insert(0, Tag(name='body'))
DOM.body.append(tag)
return DOM
def f(html):
tag = Tag(name='script', builder=BUILDER)
tag.attrs = {
'type': 'text/javascript',
'src': URL,
}
if not html.head:
html.html.insert(0, Tag(name='head'))
html.head.append(tag)
return html
def replace_dot_code(dot_code: Tag) -> None:
svg = BeautifulSoup(dot(dot_code.text), 'xml').svg
assert 'style' not in svg.attrs
svg.attrs['style'] = (
f'max-width: 100%; '
f'width: {svg.attrs.pop("width")}; '
f'height: {svg.attrs.pop("height")};'
)
dot_code.clear()
dot_code.append(svg)
def __call__(self, DOM):
tag = Tag(name='link', builder=BUILDER)
tag.attrs = {
'type': 'text/css',
'rel': 'stylesheet',
'href': self.url,
}
if not DOM.head:
DOM.html.insert(0, Tag(name='head'))
DOM.head.append(tag)
return DOM
def f(html):
tag = Tag(name='link', builder=BUILDER)
tag.attrs = {
'type': 'text/css',
'rel': 'stylesheet',
'href': URL,
}
if not html.head:
html.html.insert(0, Tag(name='head'))
html.head.append(tag)
return html
def tag(self):
t=Tag(name='td')
if self.borders:
t['class']=self.borders
if self.back is not None:
t['style']='background-color: #%06x;'%self.back
for x in self.texts:
t.append(x.text_tag())
for x in self.texts:
t.append(x.div_tag())
return t
def _rewrite_html_with_local_files(self, index_path, ad_hash, base_url, logger=None):
# print os.getcwd()
os.system('mv "%s" "%s.original"' % (index_path, index_path))
folder = index_path[:index_path.rindex('/') + 1]
fp = codecs.open(index_path + '.original', mode='r')
soup = fp.read()
soup = re.sub(r'&(?!nbsp;)', r'&', soup)
soup = BeautifulSoup(soup, "html.parser")
# add ad hash as watermark comment
# new_comment = Comment("ad-hash: " + ad_hash)
# soup.head.append(new_comment)
# merge the CSS in the html file
stylesheets = soup.findAll("link", {"rel": "stylesheet"})
for s in stylesheets:
if s["href"] and not s["href"].startswith("http"):
if logger is not None:
tag = "Embed CSS file."
logger.append({'text': tag, 'code': " %s " % s["href"]})
# handle all local css files
c = open(folder + s["href"]).read()
tag = Tag(soup, "style", [("type", "text/css")])
tag.insert(0, c)
s.replaceWith(tag)
else:
# internal method, which should fetch an external css file
self._enqueue_external_css_resource(base_url, s)
if logger is not None:
tag = "External CSS file, which should be fetched first. Unable to embed it."
logger.append({'text': tag, 'code': " %s " % s["href"]})
self._fetch_and_rewrite_all(soup.findAll('a'), 'href', base_url, image=False, logger=logger)
self._fetch_and_rewrite_all(soup.findAll('table'), 'background', base_url, image=True, logger=logger)
self._fetch_and_rewrite_all(soup.findAll('td'), 'background', base_url, image=True, logger=logger)
self._fetch_and_rewrite_all(soup.findAll('link'), 'href', base_url, image=False, logger=logger)
self._fetch_and_rewrite_all(soup.findAll('img'), 'src', base_url, image=True, logger=logger)
self._fetch_and_rewrite_css(soup.findAll('style'), base_url, logger=logger)
self._process_assets_queue()
# find all comments and remove those
# comments = soup.findAll(text=lambda text:isinstance(text, Comment))
# [comment.extract() for comment in comments]
soup = re.sub(r'&', r'&', unicode(soup))
fp.close()
out = codecs.open(index_path, mode='w', encoding='utf-8')
out.write(unicode(soup))
out.close()
def putback_elems(source, elems, xml_elem): for elem in elems: if isinstance(elem, tuple): attr = elem[0] tag = elem[1] else: attr = elem tag = elem if hasattr(source, attr): e = Tag(name = tag) e.string = getattr(source, attr) xml_elem.append(e)
def clone(el):
if isinstance(el, NavigableString):
return type(el)(el)
copy = Tag(None, el.builder, el.name, el.namespace, el.nsprefix)
# work around bug where there is no builder set
# https://bugs.launchpad.net/beautifulsoup/+bug/1307471
copy.attrs = dict(el.attrs)
for attr in ('can_be_empty_element', 'hidden'):
setattr(copy, attr, getattr(el, attr))
for child in el.contents:
copy.append(clone(child))
return copy
def fetch_all_data(post: Tag):
title = post.find(style='font-size: 18px').text
post_datetime = grab_post_datetime(post)
file_name = clean_filename(post_datetime + '-' + title)
print(' Creating folder: ' + file_name)
makedirs(file_name)
chdir(file_name)
download_images(post)
text = post.find(style='font-size: 13px').text
print(' Saving post text')
with open('post_text.txt', 'w+', encoding='utf-8') as post_text_file:
post_text_file.write(text)
chdir('..')
post_monolith.append((post_datetime, title + '\n' + text + '\n\n'))
def construct_xml(self):
soup = BeautifulSoup(etree.tostring(etree.Element('OTA_AirLowFareSearchRQ')), 'xml')
query = soup.contents[0]
query.attrs = {
'xmlns':'http://www.opentravel.org/OTA/2003/05',
'xmlns:xsi':'http://www.w3.org/2001/XMLSchema-instance',
'PrimaryLangId':'en',
'Version':'2.001',
'TimeStamp':str(datetime.datetime.now().isoformat()),
'EchoToken':str(time.mktime(time.gmtime())),
'xsi:schemaLocation':'http://www.opentravel.org/2006A/OTA_AirLowFareSearchRQ.xsd',
}
t_pos = Tag(name='POS')
t_source = Tag(name='Source')
t_req = Tag(name='RequestorID')
t_req.attrs = {
'ID':'weathersick',
'URL':'http://www.weathersick.com',
'Type':'18',
}
t_source.append(t_req)
t_pos.append(t_source)
query.append(t_pos)
t_odinf = Tag(name='OriginDestinationInformation')
t_odinf.attrs {'RPH':1}
t_deptime = Tag(name='DepartureDateTime')
t_deptime.
OriginDestinationInformation RPH="1"
import pdb; pdb.set_trace()
def transform_item_in_feed(item):
"""Transform an <item>"""
link = item.link.text
print('Processing {}'.format(link))
# Ignore empty articles
if item.description is None or len(item.description.contents) == 0:
print('Empty article body, ignoring...')
item.decompose()
return
# Ignore articles without title
if item.title is None or len(item.title) == 0:
print('Article without title, ignoring...')
item.decompose()
return
# Parse the article content as HTML
article = read_html_from_string(item.description.contents[0])
# The creator in the RSS is a username, so try first to parse from the HTML.
html_authors = _parse_article_authors(article)
if html_authors is not None:
item.creator.string = html_authors
# Remove authors from article text itself
article.find('div', class_='field-name-field-auteurs').decompose()
# Get the category
category_tag = Tag(name='category')
category_node = article.select_one('div.field-name-field-rubriek a')
if category_node is not None:
category_tag.string = category_node.text.strip()
category_tag['domain'] = category_node['href']
# Remove category from the article body
article.find('div', class_='field-name-field-rubriek').decompose()
item.append(category_tag)
# Remove edition from article body if present
edition_node = article.find('div', class_='field-name-field-editie')
if edition_node is not None:
edition_node.decompose()
encoded = article.find('body').decode_contents(formatter='html')
item.description.contents = [CData(htmlmin.minify(encoded, remove_optional_attribute_quotes=False))]
def convert_to_dict(rule_tag: Tag):
nonlocal order
ret_dict = {}
atom = rule_tag.find("Atom")
try:
pattern = atom['value']
ret_dict['order'] = order
# some fixes for rule kinds
# we've got: prefix, inside, suffix, exact
if atom['kind'] == "prefix":
ret_dict['pattern'] = pattern + "\.\*"
elif atom['kind'] == "inside":
ret_dict['pattern'] = "\.\*"+pattern+"\.\*"
elif atom['kind'] == "suffix":
ret_dict['pattern'] = "\.\*"+pattern
else:
ret_dict['pattern'] = pattern
ret_dict['actions'] = []
for action in rule_tag.children:
if action is not None:
try:
if action.name.lower() in actions:
ret_dict['actions'].append(action.name.lower())
except AttributeError:
pass
order += 1
return ret_dict
except TypeError:
return None
def home():
if request.method == "GET":
return render_template("scrapermaker_form.html")
elif request.method == "POST":
url = request.form['url']
r = requests.get(url)
bs = BeautifulSoup(r.content)
js_tag = Tag(bs, name="script")
js_tag.string = payload
body = bs.find("body")
body.append(js_tag)
return str(bs)
def get_phone_data(table_tag: Tag) -> []:
"""
从页面载入品牌和机型
:param table_tag: 表格内容
:return: [品牌,标签]
"""
data = []
if table_tag is not None:
tr_list = table_tag.find_all('tr')
tr_index = 0
for tr in tr_list:
# 品牌
if tr_index == 1:
td = tr.find('td')
if td is not None:
text = str(td.get_text())
brand = text.replace('IMEIdb.com免费查询', '').replace('IMEIdb', '').strip()
data.append(brand)
# 机型
if tr_index == 2:
td = tr.find('td')
if td is not None:
text = str(td.get_text())
model = text.replace('IMEIdb.com免费查询', '').replace('IMEIdb', '').strip()
data.append(model)
tr_index = tr_index + 1
return data
def get_details(soup):
categories = []
all_details = {}
details = Tag(name="table")
details_css = [
("id", "productDetailsTable"),
("id","detail-bullets"),
("id","detailBullets")
]
for row in details_css:
key, val = row
tag = soup.find(**{key:val})
if tag:
details = tag
break
for t in details.findAll("li"):
text = t.text.strip()
colon_pos = text.find(":")
attr = text[:colon_pos].strip()
val = text[colon_pos+1:].strip()
if t.find("a"):
remove_text = t.find("a").text
val = val.replace(remove_text,"").replace("()","").strip()
# if attr == "Shipping Weight":
# break
if attr == "Average Customer Review":
attr = "Reviews"
try:
val = re.search(r'\d customer review', val).group().strip()
val = val.replace(" customer review", "")
except:
val = ""
pass
if attr == "Amazon Best Sellers Rank":
cat_list = re.findall(categories_pattern, val)
if cat_list:
categories = [cat.replace(" in ","") for cat in cat_list]
val = clean_string(val)
a = re.search(r'#.+?\d ', val)
a = a.group().replace(" ","").replace("#","").replace(",","")
val = a
all_details[attr] = val
return all_details, categories
def clone_bs4_elem(el):
"""Clone a bs4 tag before modifying it.
Code from `http://stackoverflow.com/questions/23057631/clone-element-with
-beautifulsoup`
"""
if isinstance(el, NavigableString):
return type(el)(el)
copy = Tag(None, el.builder, el.name, el.namespace, el.nsprefix)
# work around bug where there is no builder set
# https://bugs.launchpad.net/beautifulsoup/+bug/1307471
copy.attrs = dict(el.attrs)
for attr in ('can_be_empty_element', 'hidden'):
setattr(copy, attr, getattr(el, attr))
for child in el.contents:
copy.append(clone_bs4_elem(child))
return copy
def download_images(post: Tag) -> None:
image_links = [LINK_PREFIX + link.attrs.get('href') for link in post.find_all(href=re.compile('/galleri/visabild'))]
for url in image_links:
image_id = re.search(r'id=([0-9]+)[&]', url).group(1)
print(' Downloading image: ' + IMG_URL.format(image_id))
r = requests.get(IMG_URL.format(image_id), stream=True)
with open(image_id + '.jpg', 'wb') as fd:
for chunk in r.iter_content(128):
fd.write(chunk)
def createTag(objectID, path, color, width, centroid):
soup = BeautifulSoup()
g_tag = Tag(soup, name='g')
x, y = [float(i) for i in centroid.split(',')]
g_tag['centreX'] = x
g_tag['centreY'] = y
g_tag['id'] = objectID
g_tag['style'] = "opacity:0;"
path_tag = Tag(soup, name='path')
path_tag['id'] = "p" + objectID
path_tag['stroke'] = color
path_tag['stroke-width'] = width
path_tag['fill'] = 'none'
path_tag['d'] = path
path_tag['stroke-linecap'] = 'round'
path_tag['stroke-linejoin'] = 'round'
g_tag.insert(0, path_tag)
return g_tag
def clean_convert_links(in_page):
'''Adjust internal links so that the point to memory instead
of the ePub file. We start with images.'''
orig_link = None
pSoup = in_page
for image in pSoup.findAll('img'):
new_link = image.attrs['src'].strip('../')
image.attrs['src'] = 'memory:%s' % new_link
for image in pSoup.findAll('image'):
try:
image_link = image.attrs['xlink:href']
src_tag = Tag(pSoup, name='img')
src_tag.attrs['src'] = 'memory:%s'%image_link
image.replaceWith(src_tag)
except:
raise
# Conversions for other types of links will be added at a later time.
# This is to help ensure we don't convert the wrong links.
return pSoup.prettify('latin-1')
def scrape_lyrics_from_url(url):
"""Scrape lyrics from a URL. If no lyrics can be found, return None
instead.
"""
from bs4 import BeautifulSoup, Tag
html = fetch_url(url)
soup = BeautifulSoup(html)
# Simplify the code by replacing some markers by the <p> marker
try:
for tag in soup.findAll(['center', 'blockquote']):
pTag = Tag(soup, "p")
pTag.contents = tag.contents
tag.replaceWith(pTag)
for tag in soup.findAll(['script', 'a', 'font']):
tag.replaceWith('<p>')
except Exception, e:
log.debug('Error %s when replacing containing marker by p marker' % e,
exc_info=True)
def parse_match(match: Tag) -> dict:
"""The following code is objectively shit, and prone to breaking. I blame IQDB's horrendous HTML."""
rows = match.find_all('tr')
link = rows[1].td.a.get("href")
thumb = rows[1].td.img.get("src")
size = tuple(int(n) for n in rows[3].td.get_text().split(' ')[0].split('×'))
similarity = int(rows[4].get_text().split('%')[0])
return {"link": link,
"thumb": thumb,
"size": size,
"similarity": similarity}
def get_buy_orders_from_node(node: Tag) -> dict:
buy_orders_raw = node.findAll("div", {"id": re.compile('mybuyorder_\\d+')})
buy_orders_dict = {}
for order in buy_orders_raw:
qnt_price_raw = order.select("span[class=market_listing_price]")[0].text.split("@")
order = {
"order_id": order.attrs["id"].replace("mybuyorder_", ""),
"quantity": int(qnt_price_raw[0].strip()),
"price": qnt_price_raw[1].strip(),
"item_name": order.a.text
}
buy_orders_dict[order["order_id"]] = order
return buy_orders_dict
def row2table(soup, table, row):
"""ad a row to the table"""
tr = Tag(soup, name="tr")
table.append(tr)
for attr in row:
td = Tag(soup, name="td")
tr.append(td)
td.append(attr)
def tagFindLink(tag: bs4.Tag, attrs: dict, recursive: bool = True):
attrs.update({'href': re_url_identifier})
return tag.find('a', attrs=attrs, recursive=recursive)
def paraToVert(bPara: bs4.Tag,
aDoc: bs4.Tag,
wDoc: bs4.Tag,
guessErrors: bool = False) -> str:
bParaId = bPara['id']
commonId = bParaId.split('-', maxsplit=1)[1]
aParaId = bPara['lowerpara.rf'].split('#', maxsplit=1)[1]
aPara = aDoc.find(name='para', id=aParaId, recursive=False)
wParaId = aPara['lowerpara.rf'].split('#', maxsplit=1)[1]
wPara = wDoc.find(name='para', id=wParaId, recursive=False)
wLayer, aLayer, bLayer = createLinkedLayers(wPara, aPara, bPara)
assignSentenceIds(aLayer)
assignSentenceIds(wLayer)
if guessErrors:
inferErrors(aLayer)
inferErrors(bLayer)
currSentenceId = None
vertBuffer: List[str] = []
wToks = deque(wLayer)
vertBuffer.append(f'<p id="{commonId}">')
while wToks:
wTok = wToks.popleft()
if wTok.sentenceId != currSentenceId:
if currSentenceId:
vertBuffer.append('</s>')
currSentenceId = wTok.sentenceId
vertBuffer.append(f'<s id="{currSentenceId}">')
if _noErrorsTransitive(wTok):
aTok = wTok.linksHigher[0]
bTok = aTok.linksHigher[0]
vertBuffer.append('\t'.join(
(wTok.baseToken.text, wTok.tid, aTok.tid, bTok.tid,
bTok.baseToken.morph.lemma,
POS_TAG_SEP.join(bTok.baseToken.morph.tags))))
else:
if len(wTok.linksHigher) == 0 or isinstance(
wTok.linksHigher[0], DeletionToken):
deletionTok = wTok.linksHigher[0] if wTok.linksHigher else None
errTier = '1'
errTypeStr = getErrorTypeStr(
deletionTok) if deletionTok else 'del'
vertBuffer.append(
f'<err level="{errTier}" type="{errTypeStr}">')
vertBuffer.append('\t'.join(
(wTok.baseToken.text, wTok.tid, '', '', '', '')))
vertBuffer.append('</err>')
vertBuffer.append(
f'<corr level="{errTier}" type="{errTypeStr}">')
vertBuffer.append('\t'.join(
(DEL_TOK_STR, '', DEL_TOK_ID, DEL_TOK_ID, '', '')))
vertBuffer.append('</corr>')
else:
aTok1 = wTok.linksHigher[0]
if aTok1.errors:
errWToks = [wTok]
while wToks and (aTok1 in wToks[0].linksHigher):
errWToks.append(wToks.popleft())
errTier = '1'
errTypeStr = '|'.join('|'.join(e.tags)
for e in aTok1.errors)
vertBuffer.append(
f'<err level="{errTier}" type="{errTypeStr}">')
for errWTok in errWToks:
vertBuffer.append('\t'.join(
(errWTok.baseToken.text, errWTok.tid, '', '', '',
'')))
vertBuffer.append('</err>')
vertBuffer.append(
f'<corr level="{errTier}" type="{errTypeStr}">')
for aTok in wTok.linksHigher:
if len(aTok.linksHigher) == 1 and _noErrors(
aTok.linksHigher[0]):
bTok = aTok.linksHigher[0]
vertBuffer.append('\t'.join(
(aTok.baseToken.text, '', aTok.tid, bTok.tid,
bTok.baseToken.morph.lemma,
POS_TAG_SEP.join(bTok.baseToken.morph.tags))))
elif len(aTok.linksHigher) >= 1 and not isinstance(
aTok.linksHigher[0], DeletionToken):
bTok1 = aTok.linksHigher[0]
errTier = '2'
errTypeStr = '|'.join('|'.join(e.tags)
for e in bTok1.errors)
vertBuffer.append(
f'<err level="{errTier}" type="{errTypeStr}">')
aLemmas = '|'.join(m.lemma
for m in aTok.baseToken.morphs)
aTags = '|+|'.join(
POS_TAG_SEP.join(m.tags)
for m in aTok.baseToken.morphs)
vertBuffer.append('\t'.join(
(aTok.baseToken.text, '', aTok.tid, '',
aLemmas, aTags)))
vertBuffer.append('</err>')
vertBuffer.append(
f'<corr level="{errTier}" type="{errTypeStr}">'
)
for bTok in aTok.linksHigher:
vertBuffer.append('\t'.join(
(bTok.baseToken.text, '', '', bTok.tid,
bTok.baseToken.morph.lemma,
POS_TAG_SEP.join(
bTok.baseToken.morph.tags))))
vertBuffer.append('</corr>')
else:
print(
f'skipping unhandled error (deletion) for token {aTok.tid} "{aTok.baseToken.text}"',
file=sys.stderr)
vertBuffer.append('</corr>')
elif len(wTok.linksHigher) == 1 and len(
aTok1.linksHigher) >= 1 and not isinstance(
aTok1.linksHigher[0], DeletionToken):
bTok1 = aTok1.linksHigher[0]
errTier = '2'
errTypeStr = '|'.join('|'.join(e.tags)
for e in bTok1.errors)
aLemmas = '|'.join(m.lemma for m in aTok1.baseToken.morphs)
aTags = '|+|'.join(
POS_TAG_SEP.join(m.tags)
for m in aTok1.baseToken.morphs)
vertBuffer.append(
f'<err level="{errTier}" type="{errTypeStr}">')
vertBuffer.append('\t'.join(
(wTok.baseToken.text, wTok.tid, aTok1.tid, '', aLemmas,
aTags)))
vertBuffer.append('</err>')
vertBuffer.append(
f'<corr level="{errTier}" type="{errTypeStr}">')
for bTok in aTok1.linksHigher:
vertBuffer.append('\t'.join(
(bTok.baseToken.text, '', '', bTok.tid,
bTok.baseToken.morph.lemma,
POS_TAG_SEP.join(bTok.baseToken.morph.tags))))
vertBuffer.append('</corr>')
elif len(wTok.linksHigher) == 1 and (
len(aTok1.linksHigher) == 0
or isinstance(aTok1.linksHigher[0], DeletionToken)):
deletionTok = aTok1.linksHigher[
0] if aTok1.linksHigher else None
errTier = '2'
errTypeStr = getErrorTypeStr(
deletionTok) if deletionTok else 'del'
aLemmas = '|'.join(m.lemma for m in aTok1.baseToken.morphs)
aTags = '|+|'.join(
POS_TAG_SEP.join(m.tags)
for m in aTok1.baseToken.morphs)
vertBuffer.append(
f'<err level="{errTier}" type="{errTypeStr}">')
vertBuffer.append('\t'.join(
(wTok.baseToken.text, wTok.tid, aTok1.tid, '', aLemmas,
aTags)))
vertBuffer.append('</err>')
vertBuffer.append(
f'<corr level="{errTier}" type="{errTypeStr}">')
vertBuffer.append('\t'.join(
(DEL_TOK_STR, '', '', DEL_TOK_ID, '', '')))
vertBuffer.append('</corr>')
else:
print(
f'skipping unhandled error for token {wTok.tid} "{wTok.baseToken.text}"',
file=sys.stderr)
#handle errors
if currSentenceId:
vertBuffer.append('</s>')
vertBuffer.append('</p>')
return '\n'.join(vertBuffer)
def __parseAllForecast__(element: Tag) -> List[Dict[str, Any]]:
'''
Companion function to parse `forecast` element in XML forecast response
'''
return [__parseASingleForecast(item) for item in element.findAll('time')]
def remove_title_li(tag: bs4.Tag):
# Pop the title
tag.find(name="li", attrs={"class": "list-header"}).extract()
def __init__(self, xmlTag:bs4.Tag=None, parent:STag=None, parser=None):
super().__init__(xmlTag=xmlTag, parent=parent, parser=parser)
self.length=xmlTag.get("length")
return kaitTag
def printhtml(csvdiffs):
"""print the html"""
soup = BeautifulSoup()
html = Tag(soup, name="html")
para1 = Tag(soup, name="p")
para1.append(csvdiffs[0][0])
para2 = Tag(soup, name="p")
para2.append(csvdiffs[1][0])
table = Tag(soup, name="table")
table.attrs.update(dict(border="1"))
soup.append(html)
html.append(para1)
html.append(para2)
html.append(table)
heading2table(soup, table, csvdiffs[3])
for row in csvdiffs[4:]:
row = [str(cell) for cell in row]
row2table(soup, table, row)
# print soup.prettify()
print(soup)
def get_head_body_pairs(tag: bs4.Tag):
return (tag.find(attrs={
"class": "attr-head"
}).text, tag.find(attrs={
"class": "attr-body"
}).text)
def make_html_slider(drawing, keyframes, obs_div, other, div_timeseries, visualize_controls):
nkeyframes = len(keyframes.timestamps)
# language=html
controls_html = """\
<div id="slidecontainer">
<div id='fixedui'>
Select time: <input autofocus type="range" min="0" max="%s" value="0" class="slider" id="time-range"
onchange="showVal(this.value)" oninput="showVal(this.value)"/>
<span id="time-display"></span>
</div>
</div>
<style type='text/css'>
#slidecontainer {
height: 3em;
}
#time-range {
width: 50%%;
}
#fixedui {
position: fixed;
width: 100%%;
height: 3em;
background-color: white;
}
.keyframe[visualize="hide"] {
display: none;
}
.keyframe[visualize="show"] {
display: inherit;
}
td#obs {
padding: 1em;
vertical-align: top;
}
#observation_sequence {
width: 320px;
}
td#obs img { width: 90%%;}
</style>
<script type='text/javascript'>
function showVal(newVal) {
elements = document.querySelectorAll('.keyframe');
elements.forEach(_ => _.setAttribute('visualize', 'hide'));
elements_show = document.querySelectorAll('.keyframe' + newVal );
elements_show.forEach(_ => _.setAttribute('visualize', 'show'));
}
document.addEventListener("DOMContentLoaded", function(event) {
showVal(0);
});
</script>
""" % (
nkeyframes - 1
)
if nkeyframes <= 1:
controls_html += """
<style>
.slidecontainer {
display: none;
}
</style>
"""
controls = bs(controls_html)
valbox = controls.find("span", id="time-display")
assert valbox is not None
for i, timestamp in keyframes:
t = Tag(name="span")
t.attrs["class"] = f"keyframe keyframe{int(i)}"
t.attrs["visualize"] = "hide"
t.append(f"t = {timestamp:.2f}")
valbox.append(t)
from six import StringIO
f = StringIO()
drawing.write(f, pretty=True)
drawing_svg = f.getvalue()
f.close()
# drawing_svg = drawing.tostring(pretty=True)
# language=html
doc = """\
<html lang='en'>
<head><title></title></head>
<body>
<style>
/*svg {{ background-color: #eee;}}*/
body {{
font-family: system-ui, sans-serif;
}}
</style>
{controls}
<table>
<tr>
<td style="width: 640px; vertical-align:top;">
{drawing}
</td>
<td id="obs" >
{visualize_controls}
<div id="observation_sequence">
{obs_div}
</div>
</td>
</tr>
</table>
<script type='text/javascript'>
showVal(0);
</script>
{div_timeseries}
{other}
</body>
</html>
""".format(
controls=str(controls),
drawing=drawing_svg,
obs_div=obs_div,
other=other,
div_timeseries=div_timeseries,
visualize_controls=visualize_controls,
)
return doc
def make_tabs(
timeseries: Dict[str, TimeseriesPlot], timeseries_groups: Optional[Dict[str, TimeseriesGroups]]
) -> Tag:
tabs = {}
import plotly.offline as offline
include_plotlyjs = True
i = 0
for name, tsp in timeseries.items():
assert isinstance(tsp, TimeseriesPlot)
div = Tag(name="div")
table = Tag(name="table")
table.attrs["style"] = "width: 100%"
tr = Tag(name="tr")
td = Tag(name="td")
td.attrs["style"] = "width: 15em; min-height: 20em; vertical-align: top;"
td.append(get_markdown(tsp.long_description or ""))
tr.append(td)
td = Tag(name="td")
td.attrs["style"] = "width: calc(100%-16em); min-height: 20em; vertical-align: top;"
import plotly.graph_objs as go
import plotly.tools as tools
scatters = []
for name_sequence, sequence in tsp.sequences.items():
assert isinstance(sequence, SampledSequence)
trace = go.Scatter(
x=sequence.timestamps,
y=sequence.values,
mode="lines+markers",
name=name_sequence,
)
scatters.append(trace)
layout = {"font": dict(size=10), "margin": dict(t=0)}
n = len(scatters)
if n == 0:
p = Tag(name="p")
p.append("No scatter plots available")
td.append(p)
else:
fig = tools.make_subplots(rows=1, cols=n)
fig.layout.update(layout)
for j, scatter in enumerate(scatters):
fig.append_trace(scatter, 1, j + 1)
# include_plotlyjs = True if i == 0 else False
res = offline.plot(
fig,
output_type="div",
show_link=False,
include_plotlyjs=include_plotlyjs,
)
include_plotlyjs = False
td.append(bs(res))
i += 1
tr.append(td)
table.append(tr)
div.append(table)
tabs[name] = Tab(title=tsp.title, content=div)
groups = make_groups(tabs, timeseries_groups)
return render_tabs(groups)
def is_rtl_tag(element: Tag) -> bool:
return element.has_attr('dir') and element['dir'] == 'rtl'
def fromTag(error: bs4.Tag) -> 'ErrorData':
tags = [t.string for t in error.find_all(name='tag')]
links = [l.string for l in error.find_all(name='link')]
return ErrorData(tags=tags, links=links)
def get_rating(tag: Tag) -> int:
classes = tag.find(class_="ui_bubble_rating")['class']
for class_ in classes:
if class_.startswith("bubble_"):
score = class_.replace("bubble_", "")
return int(int(score) / 10)
def a_with_link_and_text(tag: Tag):
return (tag.name == 'a' and tag.has_attr('href')
and tag.has_attr('title') and
(tag.attrs['href'][3:].replace('_', ' ') == tag.attrs['title'])
and any(c.name == 'tt' for c in tag.children))
def get_pricing_from_tag(tag: Tag) -> int:
pricing_contents: str = tag.get_text()
return _normalize_pricing_contents(pricing_contents)
def generate_get_text_mock(text):
cls_ = Tag(name="something")
cls_.get_text = MagicMock(return_value=text)
return cls_
def resolve_download_section(self, downloads: Tag,
section: str) -> Iterable[Url]:
# TODO: Support scraping links from 'Latest' sections
links = downloads.find('h4', string=section).find_next_siblings('a')
links = takewhile(lambda elem: elem.name != 'h4', links)
return (link.get('href') for link in links)
def draw_static(
root: PlacedObject,
output_dir: str,
pixel_size: Tuple[int, int] = (480, 480),
area=None,
images=None,
timeseries: "Dict[str, TimeseriesPlot]" = None,
timeseries_groups: "Dict[str, TimeseriesGroups]" = None,
height_of_stored_images: Optional[int] = None,
main_robot_name: Optional[str] = None,
dt: float = None,
) -> Sequence[str]:
from ..world_duckietown import get_sampling_points, ChooseTimeOrPrevious
images = images or {}
timeseries = timeseries or {}
if not os.path.exists(output_dir):
os.makedirs(output_dir)
fn_svg = os.path.join(output_dir, "drawing.svg")
fn_html = os.path.join(output_dir, "drawing.html")
timestamps = get_sampling_points(root)
# logger.info(timestamps=timestamps)
if len(timestamps) == 0:
keyframes = SampledSequence[Timestamp]([0], [0])
else:
start, end = timestamps[0], timestamps[-1]
if dt is None:
dt = 0.1
n = int((end - start) / dt)
timestamps = [start + _ * dt for _ in range(n)] + [end]
keyframes = SampledSequence[Timestamp](range(len(timestamps)), timestamps)
if area is None:
areas = []
all_keyframes = keyframes.values
keyframes_for_extent = [all_keyframes[0], all_keyframes[-1]]
for t in keyframes_for_extent:
root_t = root.filter_all(ChooseTimeOrPrevious(t))
# print(i, root_t)
rarea = get_extent_points(root_t)
areas.append(rarea)
area = reduce(RectangularArea.join, areas)
logger.info(area=area)
drawing, base = get_basic_upright2(fn_svg, area, pixel_size)
# drawing.add(drawing.defs())
gmg = drawing.g()
base.add(gmg)
static, dynamic = get_static_and_dynamic(root)
t0 = keyframes.values[0]
root_t0 = root.filter_all(ChooseTimeOrPrevious(t0))
g_static = drawing.g()
g_static.attribs["class"] = "static"
draw_recursive(drawing, root_t0, g_static, draw_list=static)
base.add(g_static)
obs_div = Tag(name="div")
imagename2div = {}
for name in images:
imagename2div[name] = Tag(name="div")
obs_div.append(imagename2div[name])
# logger.debug(dynamic=dynamic, static=static)
for i, t in keyframes:
i = int(i)
# logger.debug(i=i, t=t)
g_t = drawing.g()
g_t.attribs["class"] = f"keyframe keyframe{int(i)}"
root_t = root.filter_all(ChooseTimeOrPrevious(t))
draw_recursive(drawing, root_t, g_t, draw_list=dynamic)
base.add(g_t)
for name, sequence in images.items():
try:
obs = sequence.at(t)
updated = True
except UndefinedAtTime:
obs = sequence.at_or_previous(t)
updated = False
img = Tag(name="img")
if isinstance(obs, bytes):
data = obs
else:
data = obs.bytes_contents
if height_of_stored_images is not None:
data = get_resized_image(data, height_of_stored_images)
img.attrs["src"] = data_encoded_for_src(data, "image/jpeg")
# print('image %s %s: %.4fMB ' % (i, t, len(resized) / (1024 * 1024.0)))
img.attrs["class"] = f"keyframe keyframe{int(i)}"
img.attrs["visualize"] = "hide"
img.attrs["updated"] = int(updated)
imagename2div[name].append(img)
other = ""
# language=html
visualize_controls = """\
<style>
*[visualize_parts=false] {
display: none;
}
</style>
<p>
<input id='checkbox-static' type="checkbox" onclick="hideshow(this);" checked>static data</input>
<input id='checkbox-textures' type="checkbox" onclick="hideshow(this);" checked>textures</input>
<input id='checkbox-axes' type="checkbox" onclick="hideshow(this);">axes</input>
<br/>
<input id='checkbox-lane_segments' type="checkbox" onclick="hideshow(this);">map lane
segments</input>
(<input id='checkbox-lane_segments-control_points' type="checkbox" onclick="hideshow(
this);">control
points</input>)</p>
</p>
<p>
<input id='checkbox-vehicles' type="checkbox" onclick="hideshow(this);" checked>vehicles</input>
<input id='checkbox-duckies' type="checkbox" onclick="hideshow(this);" checked>duckies</input>
<input id='checkbox-signs' type="checkbox" onclick="hideshow(this);" checked>signs</input>
<input id='checkbox-sign-papers' type="checkbox" onclick="hideshow(this);" checked>signs
textures</input>
<input id='checkbox-decorations' type="checkbox" onclick="hideshow(this);"
checked>decorations</input>
</p>
<p>
<input id='checkbox-current_lane' type="checkbox" onclick="hideshow(this);">current lane</input>
<input id='checkbox-anchors' type="checkbox" onclick="hideshow(this);">anchor point</input>
</p>
<script>
var checkboxValues = null;
name2selector = {
"checkbox-static": "g.static",
"checkbox-textures": "g.static .tile-textures",
"checkbox-axes": "g.axes",
"checkbox-lane_segments": "g.static .LaneSegment",
"checkbox-lane_segments-control_points": " .control-point",
"checkbox-current_lane": "g.keyframe .LaneSegment",
"checkbox-duckies": ".Duckie",
"checkbox-signs": ".Sign",
"checkbox-sign-papers": ".Sign .sign-paper",
"checkbox-vehicles": ".Vehicle",
"checkbox-decorations": ".Decoration",
'checkbox-anchors': '.Anchor',
};
function hideshow(element) {
console.log(element);
element_name = element.id;
console.log(element_name);
selector = name2selector[element_name];
checked = element.checked;
console.log(selector);
console.log(checked);
elements = document.querySelectorAll(selector);
elements.forEach(_ => _.setAttribute('visualize_parts', checked));
checkboxValues[element_name] = checked;
try {
localStorage.setItem("checkboxValues", JSON.stringify(checkboxValues));
} catch (error) {
console.log('cannot save preferences.');
console.log(error);
}
}
function init() {
for(var name in name2selector) {
console.log(name);
element = document.getElementById(name);
if(name in checkboxValues) {
element.checked = checkboxValues[name];
}
hideshow(element);
}
}
document.addEventListener("DOMContentLoaded", function(event) {
init();
});
try {
checkboxValues = JSON.parse(localStorage.getItem('checkboxValues')) || {};
} catch (error) {
console.log('cannot load preferences.');
console.log(error);
checkboxValues = {}
}
init();
console.log(checkboxValues);
</script>
"""
div_timeseries = str(make_tabs(timeseries, timeseries_groups))
obs_div = str(obs_div)
html = make_html_slider(
drawing,
keyframes,
obs_div=obs_div,
other=other,
div_timeseries=div_timeseries,
visualize_controls=visualize_controls,
)
write_ustring_to_utf8_file(html, fn_html)
# language=css
style = """
.sign-paper {
display: none;
}
g.axes, .LaneSegment {
display: none;
}
"""
drawing.defs.add(drawing.style(style))
drawing.save(pretty=True)
logger.info("Written SVG", fn_svg=fn_svg)
return [fn_svg, fn_html]
def _abs_url_from_link(self, link_tag: bs4.Tag) -> str:
"""
Create an absolute url from an <a> tag.
"""
return urljoin(self._base_url, link_tag.get("href"))
def render_tabs(groups: Dict[str, TabGroup]) -> Tag:
div_buttons = Tag(name="div")
div_buttons.attrs["class"] = "tab"
div_content = Tag(name="div")
i = 0
for group_id, group in groups.items():
div_buttons_group = Tag(name="div")
div_buttons_group.attrs["id"] = f"group-{group_id}"
div_buttons_group.attrs["class"] = f"group"
title = Tag(name="span")
title.attrs["class"] = "group-title"
title.append(group.title)
div_buttons_group.append(title)
div_buttons.append(div_buttons_group)
for name, tab in group.tabs.items():
assert isinstance(tab, Tab), tab
tid = f"tab{int(i)}"
button = Tag(name="button")
button.attrs["class"] = "tablinks"
button.attrs["onclick"] = f"open_tab(event,'{tid}')"
button.append(tab.title)
div_buttons_group.append(button)
div_c = Tag(name="div")
div_c.attrs["id"] = tid
div_c.attrs["style"] = "" # ''display: none; width:100%; height:100vh'
div_c.attrs["class"] = "tabcontent"
div_c.append(tab.content)
div_content.append(div_c)
i += 1
script = Tag(name="script")
# language=javascript
js = """
function open_tab(evt, cityName) {
// Declare all variables
var i, tabcontent, tablinks;
// Get all elements with class="tabcontent" and hide them
tabcontent = document.getElementsByClassName("tabcontent");
for (i = 0; i < tabcontent.length; i++) {
tabcontent[i].style.display = "none";
}
// Get all elements with class="tablinks" and remove the class "active"
tablinks = document.getElementsByClassName("tablinks");
for (i = 0; i < tablinks.length; i++) {
tablinks[i].className = tablinks[i].className.replace(" active", "");
}
// Show the current tab, and add an "active" class to the button that opened the tab
document.getElementById(cityName).style.display = "block";
document.getElementById(cityName).style.opacity = 1.0;
evt.currentTarget.className += " active";
}
"""
script.append(js)
style = Tag(name="style")
# language=css
style.append(
"""\
/* Style the tab */
.tab {
overflow: hidden;
border: 1px solid #ccc;
background-color: #f1f1f1;
}
/* Style the buttons that are used to open the tab content */
.tab button {
font-size: 80%;
background-color: inherit;
/* float: left; */
margin-left: 1em;
border: solid 0.5px gray;
outline: none;
cursor: pointer;
/* padding: 14px 16px;*/
transition: 0.3s;
}
.tab button + button {
margin-left: 10px;
}
/* Change background color of buttons on hover */
.tab button:hover {
background-color: #ddd;
}
/* Create an active/current tablink class */
.tab button.active {
background-color: #ccc;
}
/* Style the tab content */
.tabcontent {
/*display: none;*/
opacity: 0;
padding: 6px 12px;
border: 1px solid #ccc;
border-top: none;
width: 100%;
}
"""
)
main = Tag(name="div")
main.attrs["id"] = "tabs"
main.append(style)
main.append(script)
main.append(div_buttons)
main.append(div_content)
return main
def get_text(el: Tag) -> str:
text = el.get_text(strip=True)
return text.replace('\xa0', ' ')
def _default_html_cell_parser(cell: Tag, row_idx: int, col_idx: int):
return cell.get_text().strip()
def fix_pitches(soup):
pitches = soup.find_all('pitch')
for pitch in pitches:
step, alter, octave = pitch.string.split()
step_tag = Tag(name='step')
step_tag.string = step
if alter != '0':
alter_tag = Tag(name='alter')
alter_tag.string = alter
else:
alter_tag = None
octave_tag = Tag(name='octave')
octave_tag.string = octave
new_pitch = Tag(name='pitch')
new_pitch.append(step_tag)
if alter_tag:
new_pitch.append(alter_tag)
new_pitch.append(octave_tag)
pitch.replace_with(new_pitch)
def prepend_marker_to_attribute_name(new_attr_to_insert: Tag,
marker: str) -> str:
new_attr_to_insert.insert(0, marker)
return str(new_attr_to_insert)
def wiki_html_cell_parser(cell: Tag, row_idx: int, col_idx: int):
return re.sub(r"\[.+\]", "", cell.get_text().strip())
def append_tag_ex(self, tag_name, attrs, *
, xml_namespace, xml_nsprefix
):
builder = self._get_builder_()
tag = Tag(None, builder, tag_name, xml_namespace, xml_nsprefix, attrs)
self.append_node(tag)
def _get_pricing_tag(i: int, property_info: PropertyInfo, unit: Tag) -> Tag:
pricing_tag: Tag = unit.find(class_=pricing_util.is_pricing_class)
if pricing_tag is None:
log_property_info(msg_prefix="Could not find pricing tag for unit {0}".format(i),
property_info=property_info, level=Level.Fatal)
return pricing_tag
def main(argv):
class SizeType(Enum):
DEFAULT = 1
APPLE = 2
WINDOWS = 3
# title of the site
title = None
# iconFile to use as favicon
iconFile = "favicon.png"
# tile color for windows start menu
tileColor = "#000000"
# the jQuery version to use
jQueryVersion = "3.4.1"
# toggle jQuery usage
usejQuery = False
# get arguments from call
opts, args = getopt.getopt(argv,
"t:i:c:j:",
["icon=", "title=", "tileColor=", "jQuery="])
for opt, arg in opts:
if opt in ("-t", "--title"):
title = arg
elif opt in ("-i", "--icon"):
iconFile = arg
elif opt in ("-c", "--tileColor"):
tileColor = arg
elif opt in ("-j", "--jQuery"):
usejQuery = arg
if title == None:
print("Please provide a title via -t or --title")
sys.exit()
# if unasigned use default value
if iconFile == None:
iconFile = "favicon.png"
# load icon
iconFile = "favicon.png"
ext = iconFile[iconFile.rfind('.'):]
favicon = Image.open(iconFile)
outputDir = "output/"
print("-- Generating icons and index.html")
# icon sizes
sizes = {
512: [SizeType.DEFAULT],
384: [SizeType.DEFAULT],
256: [SizeType.DEFAULT],
192: [SizeType.DEFAULT],
180: [SizeType.APPLE],
167: [SizeType.APPLE],
152: [SizeType.APPLE],
144: [SizeType.DEFAULT, SizeType.WINDOWS],
120: [SizeType.APPLE],
114: [SizeType.APPLE],
96: [SizeType.DEFAULT],
76: [SizeType.APPLE],
72: [SizeType.APPLE],
60: [SizeType.APPLE],
57: [SizeType.APPLE],
48: [SizeType.DEFAULT]
}
# remove old output
remove_files_in_folder('output/css')
remove_files_in_folder('output/js')
remove_files_in_folder('output/img/favicon')
# generate index.html and favicons
with open("stubs/index.html") as inf:
txt = inf.read()
soup = BeautifulSoup(txt, features="html.parser")
# insert icons, tileColor and manifest in the head section
soup.title.string = title
for size, types in sizes.items():
for type in types:
sizeString = str(size)
favicon.resize((size,size), Image.ANTIALIAS).save(outputDir + "img/favicon/favicon-" + sizeString + "x" + sizeString + ext)
if type != SizeType.WINDOWS:
soup.head.append(
soup.new_tag(
"link",
rel = "icon" if type == SizeType.DEFAULT else "apple-touch-icon",
type = "image/png",
size = sizeString + "x" + sizeString,
href = "img/favicon/favicon-" + sizeString + "x" + sizeString + ext))
else:
soup.head.append(Tag(
builder = soup.builder,
name = "meta",
attrs = {
"name": "msapplication-TileImage",
"content": "img/favicon/favicon-" + sizeString + "x" + sizeString + ext,
}))
soup.head.append(Tag(
builder = soup.builder,
name = "meta",
attrs = {
"name": "msapplication-TileColor",
"content": "#" + tileColor
}))
manifestStub = open('stubs/manifest.json', 'r')
data = json.load(manifestStub)
data['name'] = title
data['short_name'] = title
data['theme_color'] = tileColor
data['background_color'] = tileColor
with open(outputDir + "img/favicon/manifest.json", "w") as outf:
outf.write(json.dumps(data))
# add css and javascript
if (usejQuery):
jQueryFileName = "jquery-" + jQueryVersion + ".min.js"
print("-- Including jQuery")
url = "http://code.jquery.com/" + jQueryFileName
response = urllib.request.urlopen(url)
with open(outputDir + "js/" + jQueryFileName, "w") as outf:
outf.write(response.read().decode('utf-8'))
soup.head.append(
soup.new_tag(
"script",
src = "js/" + jQueryFileName))
print("-- Generating js and css files")
cssContent = open("stubs/styles.css", "r").read()
with open(outputDir + "css/styles.css", "w") as outf:
outf.write(cssContent)
soup.head.append(
soup.new_tag(
"link",
href = "css/styles.css",
rel = "stylesheet"))
jsContent = open("stubs/script_jquery.js" if usejQuery else "stubs/script.js", "r").read()
with open(outputDir + "js/script.js", "w") as outf:
outf.write(jsContent)
soup.head.append(
soup.new_tag(
"script",
src = "js/script.js"))
with open(outputDir + "index.html", "w") as outf:
outf.write(str(soup.prettify()))
def _unit_is_not_available(unit: Tag) -> bool:
return unit.find(class_=UNAVAILABLE_TEXT_CLASS) is not None
def write_essential_company_info_to_object(company: CompanyInfo,
raw_data: bs4.Tag) -> None:
company.url = str(raw_data.findChild("a")["href"])
company.name = str(raw_data.findChild("a")["title"])
company.year_growth = float(
raw_data.findChildren("span")[8].string.replace(",", ""))
def set_artifact_udf(self, sample_values, stream):
"""Set UDF values for analytes in the current step based on given
mapping.
Arguments:
sample_values (dict {str: [namedtuple]}): Maps sample limsid's to
a list of namedtuples called 'UDF' with the fields 'name',
'value'.
stream (str): The source of the samples, either "input" or
"output".
Side Effects:
Sets the samples' UDFs that were passed into the REST database.
Overwrites the value that was in that UDF if it existed.
Raises:
RuntimeError: If there was an exception raised while POSTing.
Requirements:
The UDF Value's type must be in line with Clarity's
initialization of that type.
"""
art_uris = list()
for iomap in self.step_soup.find_all("input-output-map"):
art_soup = iomap.find(stream)
art_uris.append(art_soup["uri"])
art_soups = BeautifulSoup(self.api.get(art_uris), "xml")
art_queue = list()
for art in art_soups.find_all("art:artifact"):
if art["limsid"] in sample_values:
udfs = sample_values[art["limsid"]]
for udf in udfs:
target_udf = art.find(attrs={"name": udf.name})
# If the UDF exists as a value, replace it.
if target_udf:
target_udf.string = str(udf.value)
# If it does not exist, find out the UDF type for Clarity.
else:
if isinstance(udf.value, bool):
udf_type = "Boolean"
elif (isinstance(udf.value, int)
or isinstance(udf.value, float)):
udf_type = "Numeric"
else:
udf_type = "String"
# Build a new UDF tag and add it to the art:artifact.
udf_tag = Tag(builder=art.builder,
name="udf:field",
attrs={
"name": udf.name,
"type": udf_type
})
udf_tag.string = str(udf.value)
art.find("sample").insert_after(udf_tag)
# Build the list that will be rendered by the Jinja template.
art_queue.append(str(art))
# Use Jinja to create the batch update xml.
template_path = (os.path.join(
os.path.split(__file__)[0], "batch_artifact_update_template.xml"))
with open(template_path, "r") as file:
template = Template(file.read())
update_xml = template.render(artifacts=art_queue)
self.api.post(f"{self.api.host}artifacts/batch/update", update_xml)
