def add_videos(self, vids, txts, links, width=400):
"""add images to the HTML file
Parameters:
ims (str list) -- a list of image paths
txts (str list) -- a list of image names shown on the website
links (str list) -- a list of hyperref links; when you click an image, it will redirect you to a new page
"""
self.t = table(border=1,
style="table-layout: fixed;") # Insert a table
self.doc.add(self.t)
with self.t:
with tr():
for vid, txt, link in zip(vids, txts, links):
with td(style="word-wrap: break-word;",
halign="center",
valign="top"):
with p():
with a(href=os.path.join('videos', link)):
with video(style="width:%dpx" % width,
controls=True):
source(src=os.path.join('videos', vid),
type="video/mp4")
br()
p(txt)
def html_grid(grid, transpose=False):
if transpose:
grid = list(map(list, zip(*grid)))
with dtags.table().add(dtags.tbody()):
for row_idx, row in enumerate(grid):
with dtags.tr():
for cell_idx, cell in enumerate(row):
cell_type = cell["type"]
if cell_type == "txt":
if "text" not in cell:
raise ValueError(
"Expected grid cell of type 'txt'"
" to have field 'text'"
)
dtags.td().add(dtags.p(cell["text"]))
elif cell_type == "video":
if "path" not in cell:
raise ValueError(
"Expected grid cell of type 'video'"
" to have field 'path'"
)
video_path = cell["path"]
if str(video_path).lower().endswith("webm"):
vid_type = "video/webm"
if str(video_path).lower().endswith("mp4"):
vid_type = "video/mp4"
with dtags.td():
dtags.video(controls=True, autoplay=False).add(
dtags.source(src=video_path, type=vid_type)
)
def add_videos(self, vids, txts, links, width=400, hidden_tag="hidden"):
"""add images to the HTML file
Parameters:
vids (str list) -- a list of image paths
txts (str list) -- a list of image names shown on the website
links (str list) -- a list of hyperref links; when you click an image,
it will redirect you to a new page
"""
self.t = table(border=1, style="table-layout: fixed;") # Insert a table
self.doc.add(self.t)
colors = ["red", "blue", "gold", "salman"]
with self.t:
with tr():
for vid, txt, link in zip(vids, txts, links):
td_style = "word-wrap: break-word; width:{}px".format(width)
with td(style=td_style, halign="center", valign="top"):
with p():
vid_path = str(vid)
if vid_path == hidden_tag:
p_style = "font-weight: bold; width:{}px;"
p_style = p_style.format(width * 3)
p("hidden video", style=p_style)
else:
with a(href=str(link)):
with video():
attr(controls="controls")
source(src=vid_path, type="video/mp4")
br()
rows = txt.split("<br>")
for idx, row in enumerate(rows):
color = colors[idx % len(colors)]
bold_tag = "<b>"
if not row.startswith(bold_tag):
s_style = "color:{};".format(color)
else:
s_style = "color:black; font-weight: bold;"
row = row[len(bold_tag):]
span(row, style=s_style)
def _resources(container, records, show_cw, subject_title, subject_directory,
add_record, audio_widgets):
with container:
with t.div(cls='resource_block'):
t.div(t.b(subject_title), cls='subject_title')
cycle_week = None
for record in records:
if cycle_week != (record['cycle'], record['week']):
# For each new week encountered, add the cycle and week numbers on rhs...
cycle_week = (record['cycle'], record['week'])
resource_div = t.div(cls='resource_record')
buttonstrip = t.div(cls='buttonstrip')
if audio_widgets:
filename_base = subject_directory + '/c%sw%s' % (
record['cycle'], record['week'])
with buttonstrip:
t.audio(t.source(src=_aurl(filename_base + '.mp3'),
type='audio/mpeg'),
id=filename_base) # invisible
t.button('>',
onclick='getElementById("%s").play();' %
filename_base),
t.button('$',
onclick='window.open("%s","_blank");' %
_aurl(filename_base + '.pdf')),
t.button('@', onclick='')
_add_cw(record, buttonstrip)
resource_div += buttonstrip
add_record(record, resource_div)
t.div(
cls='clear'
) # force resource_block container to be tall enough for all content
"We have randomly selected 12 video samples and show the synthesized audio for our baseline method (denoted by B in the paper) and for the work of <a href='https://www.isca-speech.org/archive/Interspeech_2019/pdfs/1445.pdf'>Vougioukas et al. (Interspeech, 2019)</a> (denoted by V2S GAN). "
"The videos are cropped around the lips, corresponding to the input to our network. "
"These results correspond to section 4.1 in our paper.</p>")
data1 = load_results_1()
for row in partition_all(6, data1):
with tags.div(cls="row mt-2 align-items-end"):
for col in row:
with tags.div(cls="col-2 text-center"):
with tags.div():
tags.span(col["speaker"], cls="text-muted")
tags.code(col["sample-id"], cls="ml-1")
tags.span(col["text"], cls="font-italic")
with tags.video(controls=True,
cls="embed-responsive"):
tags.source(src=col["video-path"],
type="video/webm")
tags.span("Ours")
with tags.audio(controls=True,
cls="embed-responsive"):
tags.source(src=col["audio-path-ours"],
type="audio/wav")
tags.span("V2S GAN")
with tags.audio(controls=True,
cls="embed-responsive"):
tags.source(src=col["audio-path-theirs"],
type="audio/wav")
tags.h2("Speaker control", id="speaker-control", cls="mt-3")
p = tags.p(
"In this experiment, we synthesize audio based on two inputs:")
def make_region_html(region, args):
"""Write region-specific HTML page."""
latest = max(
m.frame.index.max()
for m in region.metrics["covid"].values()
if m.emphasis >= 0
)
doc = dominate.document(title=f"{region.name} COVID-19 ({latest.date()})")
doc_url = urls.region_page(region)
def doc_link(url):
return urls.link(doc_url, url)
with doc.head:
style.add_head_style(doc_url)
with doc.body:
tags.attr(id="map_key_target", tabindex="-1")
with tags.h1():
def write_breadcrumbs(r):
if r is not None:
write_breadcrumbs(r.parent)
tags.a(r.short_name, href=doc_link(urls.region_page(r)))
util.text(" » ")
write_breadcrumbs(region.parent)
util.text(region.name)
with tags.div():
pop = region.totals["population"]
vax = region.totals.get("vaccinated", 0)
pos = region.totals.get("positives", 0)
dead = region.totals.get("deaths", 0)
nobreak = lambda t: tags.span(t, cls="nobreak")
nobreak(f"{pop:,.0f} pop; ")
if vax:
nobreak(f"{vax:,.0f} ({100 * vax / pop:.2g}%) vacc, ")
nobreak(f"{pos:,.0f} ({100 * pos / pop:.2g}%) pos, ")
nobreak(f"{dead:,.0f} ({100 * dead / pop:.2g}%) deaths ")
nobreak(f"as of {latest.date()}")
if urls.has_map(region):
with tags.div(cls="graphic"):
with tags.video(id="map", preload="auto"):
href = urls.link(doc_url, urls.map_video_maybe(region))
tags.source(type="video/webm", src=f"{href}#t=1000")
with tags.div(cls="map_controls"):
def i(n):
return tags.i(cls=f"fas fa-{n}")
tags.button(i("pause"), " ", i("play"), " P", id="map_play")
tags.button(i("repeat"), " L", id="map_loop")
tags.button(i("backward"), " R", id="map_rewind")
tags.button(i("step-backward"), " [", id="map_prev")
tags.input_(type="range", id="map_slider")
tags.button(i("step-forward"), " ]", id="map_next")
tags.button(i("forward"), " F", id="map_forward")
tags.img(cls="graphic", src=doc_link(urls.chart_image(region)))
notables = [p for p in region.policy_changes if p.score]
if notables:
tags.h2(
tags.span("Closing", cls="policy_close"),
" and ",
tags.span("Reopening", cls="policy_open"),
" policy changes",
)
with tags.div(cls="policies"):
last_date = None
for p in notables:
date, s = str(p.date.date()), p.score
if date != last_date:
tags.div(date, cls=f"date")
last_date = date
tags.div(p.emoji, cls=f"emoji")
tags.div(
p.text,
cls="text"
+ (" policy_close" if s < 0 else "")
+ (" policy_open" if s > 0 else "")
+ (" policy_major" if abs(s) >= 2 else ""),
)
subs = [
r
for r in region.subregions.values()
if r.matches_regex(args.region_regex)
]
if subs:
sub_pop = sum(s.totals["population"] for s in subs)
if len(subs) >= 10 and sub_pop > 0.9 * region.totals["population"]:
def pop(r):
return r.totals.get("population", 0)
def pos(r):
m = r.metrics["covid"].get("COVID positives / day / 100Kp")
return m.frame.value.iloc[-1] * pop(r) if m else 0
tags.h2("Top 5 by population")
for s in list(sorted(subs, key=pop, reverse=True))[:5]:
make_subregion_html(doc_url, s)
tags.h2("Top 5 by new positives")
for s in list(sorted(subs, key=pos, reverse=True))[:5]:
make_subregion_html(doc_url, s)
tags.h2(f'All {"divisions" if region.parent else "countries"}')
else:
tags.h2("Subdivisions")
for s in sorted(subs, key=lambda r: r.name):
make_subregion_html(doc_url, s)
r = region
credits = dict(c for p in r.policy_changes for c in p.credits.items())
for ms in r.metrics.values():
credits.update(c for m in ms.values() for c in m.credits.items())
with tags.p("Sources: ", cls="credits"):
for i, (url, text) in enumerate(credits.items()):
util.text(", ") if i > 0 else None
tags.a(text, href=url)
with open(urls.file(args.site_dir, doc_url), "w") as doc_file:
doc_file.write(doc.render())
def create_page_1(phone, data):
doc = dominate.document(title="Phone alignments – " + phone)
lexicon = load_lexicon()
with doc.head:
T.meta(**{'content': 'text/html;charset=utf-8', 'http-equiv': 'Content-Type'})
T.meta(**{'content': 'utf-8', 'http-equiv': 'encoding'})
T.link(rel='stylesheet', href='../style.css')
T.script(
src="https://code.jquery.com/jquery-3.4.1.min.js",
integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=",
crossorigin="anonymous",
)
T.script(type='text/javascript', src='../script.js')
with doc:
T.h3("Phone: ", phone)
with T.div(cls="hint"):
T.p("Notes:")
with T.ul():
T.li("hover over the images to play the gif and the audio")
T.li("below each clip we indicate the utterance id, speaker id and text")
T.li("the word in bold and brown indicates the \"location\" of the uttered phone")
for k, sample in data.items():
# Copy gif
for t in {"still", "anima"}:
src = os.path.join(DATA_DIR, "crops", k + f"_{t}.gif")
dst = os.path.join("www", DATA_DIR, k + f"_{t}.gif")
shutil.copyfile(src, dst)
# Copy audio
src = os.path.join(DATA_DIR, "crops", k + f".wav")
dst = os.path.join("www", DATA_DIR, k + f".wav")
shutil.copyfile(src, dst)
# Create item
with T.div(cls="sample"):
src = os.path.join("..", DATA_DIR, k + "_still.gif")
T.img(src=src, width=100, height=80)
with T.audio(controls=True):
src = os.path.join("..", DATA_DIR, k + ".wav")
T.source(src=src, type="audio/wav")
with T.div(cls="info"):
T.span(sample.key + " | " + sample.speaker)
with T.div(cls="text"):
text = [word for word in sample.sentence.split() if word != "sil"]
text = " ".join(text)
phone_id = k.split("_")[1]
phone_id = int(phone_id)
phones = [phone for phone, _, _ in sample.audio_alignment]
i = find_word_id(lexicon, text, phones, phone_id)
words = text.split()
T.span(" ".join(words[:i]))
T.span(T.b(words[i]), style="color:brown")
T.span(" ".join(words[i + 1:]))
directory = os.path.join("www", phone)
path = os.path.join(directory, "index.html")
os.makedirs(directory, exist_ok=True)
with open(path, "w") as f:
f.write(doc.render())
print(path)
def create_page(data, samples):
doc = dominate.document(title="Deep Lip Reading on the Grid data")
with doc.head:
T.meta(**{
'content': 'text/html;charset=utf-8',
'http-equiv': 'Content-Type'
})
T.meta(**{'content': 'utf-8', 'http-equiv': 'encoding'})
T.link(rel='stylesheet', href='style.css')
T.script(
src="https://code.jquery.com/jquery-3.4.1.min.js",
integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=",
crossorigin="anonymous",
)
T.script(type='text/javascript', src='script.js')
with doc:
T.h3("Deep Lip Reading on the Grid data")
T.p("""
We run the pre-trained transformer model from VGG on word-level video snippets (shown as GIFs);
these are taken from the Grid corpus (see also videos below).
For each word-level snippet we provide the following information:
""")
with T.ul():
T.li("the sample's name")
T.li("the input to the visual frontend")
T.li("the prediction")
T.li("the groundtruth")
T.p("""
The GIFs are at 4 frames per second and can be animated by hovering over.
""")
for k, group in itertools.groupby(data,
key=lambda t: t[0].split("_")[0]):
# Show video
sample, *_ = [s for s in samples if s.key == k]
src = sample.video_path
dst = os.path.join("www", "data-vgg", k + ".mp4")
if not os.path.exists(dst):
subprocess.run(["ffmpeg", "-i", src, dst])
with T.video(controls="controls"):
path = os.path.join("data-vgg", k + ".mp4")
T.source(src=path, **{"type": "video/mp4"})
T.br()
# Show crops
with T.div():
for key, pred, gt in group:
with T.div(cls="sample"):
src = os.path.join("data-vgg", key + "_still.gif")
with T.div(cls="info"):
T.span(key)
T.img(src=src, width=100, height=100)
with T.div():
T.span(pred)
with T.div():
T.span(gt)
path = os.path.join("www", "vgg.html")
with open(path, "w") as f:
f.write(doc.render())
print(path)
