def design(slides: Slides):
slide = slides.new_slide()
content = slide_header(slide, "Design by community")
list = content.box()
list_item(list).text("Open source")
list_item(list).text("RFC")
list.box(width=700, height=200, p_top=40).image("imgs/rust-rfc.png")
slide = slides.new_slide()
content = slide_header(slide, "Backwards compatibility")
small = s(size=28)
list = content.box()
list_item(list).text("Strong BC guarantees")
list_item(list, show="2+").text("New version every 6 weeks")
list_item(list, show="2+", level=1).text(
"Thousands of libraries tested to spot regressions", style=small)
list_item(list, show="3+").text("Big changes => new edition")
list_item(list, show="3+", level=1).text("Rust 2015 vs Rust 2018",
style=small)
slide = slides.new_slide()
slide.update_style("code", s(size=38))
content = slide_header(slide, "Unstable features")
code(content.box(), """
#![feature(async_await)]
async fn foo() {
...
}""")
content.box(height=20)
bash(content.box(show="2+"), "$ cargo +nightly build")
def reset_dir(self, pkgdirname=None, all=False, rmpkg=True):
if all:
logger.info('resetting %s', str(REPO_ROOT))
bash(GIT_RESET_SUBDIR, cwd=REPO_ROOT)
else:
if not pkgdirname:
return False
cwd = REPO_ROOT / pkgdirname
if cwd.exists():
logger.info('resetting %s', str(cwd))
try:
bash(GIT_RESET_SUBDIR, cwd=cwd)
except Exception:
logger.error(f'Unable to reset dir {cwd}')
print_exc_plus()
for fpath in [f for f in cwd.iterdir()]:
if fpath.is_dir() and \
fpath.name in ('pkg', 'src'):
if fpath.name == 'pkg':
fpath.chmod(0o0755)
rmtree(fpath)
elif rmpkg and fpath.is_file() and \
(fpath.name.endswith(PKG_SUFFIX) or \
fpath.name.endswith(PKG_SIG_SUFFIX)):
fpath.unlink()
else:
return False
return True
def cargo_line(parent, text, code, show="1+", **text_args):
wrapper = parent.box(width="fill", horizontal=True, show=show)
textbox = wrapper.box(width="50%")
textbox = textbox.text(text, **text_args)
codebox = wrapper.box(width="40%")
bash(codebox, code, x=0, width="fill")
return textbox
def download(self):
"""Download the datasource if not already downloaded"""
if not os.path.isfile(self.filepath):
utils.bash("wget -P "+self.download_dir+" "+self.config["download_url"])
self.unzip()
self.process()
else:
print self.filepath + " already exists"
def shp2pgsql(self, datafile, table):
"""Load shapefiles into a postgres database"""
print "Opening {shapefile}".format(shapefile=datafile)
query = "shp2pgsql -s {srs} -W LATIN1 -g geom {shapefile} {table} > temp.sql".format(srs=self.srs, shapefile=datafile, table=table)
utils.bash(query)
print "Sql schema generated"
self.sql2pgsql(open('temp.sql', 'r').read())
def raster2pgsql(self, datafile, table):
print "Opening {raster}".format(raster=datafile)
query = "raster2pgsql -s {srs} -I -C {raster} {table} > temp.sql".format(srs=self.srs, raster=datafile, table=table)
utils.bash(query)
print "Sql schema generated"
try:
self.sql2pgsql(open('temp.sql', 'r').read())
except psycopg2.ProgrammingError:
print "Rasters not supported in PostGIS 1.5 Please upgrade to 2.0"
def run_commands(step):
for data in step.hashes:
if 'noFail' == data['Expected'].strip():
step_assert(step).assert_true(
utils.bash(data['Command']).successful())
else:
step_assert(step).assert_equals(
utils.bash(data['Command']).output_text().strip(),
data['Expected'].strip())
def done(self, fnames, overwrite=False):
'''
return None means success
else returns an error string
'''
if [
f for f in fnames
if not (f.endswith(PKG_SUFFIX) or f.endswith(PKG_SIG_SUFFIX))
]:
return "files to upload are garbage"
filter_sig = lambda fnames: [
fname for fname in fnames if not fname.endswith(PKG_SIG_SUFFIX)
]
if sorted(filter_sig(fnames)) == sorted(filter_sig(self.fnames)):
try:
update_path = Path('updates')
for pkgfname in filter_sig(fnames):
pkg_found = False
sig_found = False
for fpath in update_path.iterdir():
if fpath.is_dir():
continue
if fpath.name == pkgfname:
pkg_found = fpath
elif fpath.name == f'{pkgfname}.sig':
sig_found = fpath
if pkg_found and sig_found:
try:
bash(f'{GPG_VERIFY_CMD} {sig_found} {pkg_found}')
except CalledProcessError:
ret = f'{pkg_found} GPG verify error'
logger.error(ret)
print_exc_plus()
return ret
else:
# gpg verify success
continue
else:
return f'file missing: pkg {pkg_found} sig {sig_found}'
raise RuntimeError("unexpected error")
else:
# gpg verified for all
try:
if update(overwrite=overwrite):
return None
else:
raise RuntimeError('update return false')
except Exception:
print_exc_plus()
return f'{pkg_found} update error'
raise RuntimeError("unexpected error")
finally:
self.__init__()
else:
return "Wrong file"
def tick(self):
'''
check for updates,
create new jobs
and run them
'''
if not self.__buildjobs:
# This part check for updates
if time() - self.last_updatecheck <= UPDATE_INTERVAL * 60:
if not self.idle:
logger.info('Buildbot is idling for package updates.')
self.idle = True
return 60
else:
self.last_updatecheck = time()
self.idle = False
# git pull repo
try:
bash(GIT_PULL, cwd=REPO_ROOT)
except Exception:
print_exc_plus()
self.pkgconfigs = load_all_yaml()
updates = updmgr.check_update()
for update in updates:
(pkgconfig, ver, buildarchs) = update
march = True if len(buildarchs) >= 2 else False
for arch in buildarchs:
newjob = Job(arch, pkgconfig, ver, multiarch=march)
self._new_buildjob(newjob)
return 0
else:
# This part does the job
self.idle = False
job = self.__get_job()
if not job:
logging.error('No job got')
return
if job.multiarch:
self.__clean(job, remove_pkg=True)
self.__makepkg(job)
self.__sign(job)
if self.__upload(job):
self.__clean(job, remove_pkg=True)
else:
self.__makepkg(job)
self.__sign(job)
if self.__upload(job):
if job.pkgconfig.cleanbuild:
self.__clean(job, remove_pkg=True)
else:
self.__clean(job, rm_src=False, remove_pkg=True)
self.__finish_job(job.pkgconfig.dirname)
return 0
def install(self):
dst_dir = os.path.join(self.args.installPrefixDir, 'bin')
os.makedirs(dst_dir, exist_ok=True)
retval = 0
try:
theApp = AN_APP
if self.args.target != '':
theApp = '{}-{}'.format(self.args.target, theApp)
theApp = os.path.join(dst_dir, theApp)
output = subprocess.check_output([theApp, '--version'],
stderr=subprocess.STDOUT)
if output is None:
output = b''
except BaseException as err:
output = b''
if not self.args.forceInstall and bytes(self.args.ver, 'utf-8') in output:
print('{app} {ver} is already installed'
.format(app=self.args.app, ver=self.args.ver))
else:
print('compiling {app} {ver}'
.format(app=self.args.app, ver=self.args.ver))
code_dir = self.args.codePrefixDir
if self.args.target != '':
code_dir = os.path.join(code_dir, self.args.target)
code_dir = os.path.join(code_dir, self.args.app)
if self.args.forceInstall:
shutil.rmtree(code_dir, ignore_errors=True)
os.makedirs(code_dir)
utils.git('clone', [URL, code_dir])
os.chdir(code_dir)
version = self.args.ver.replace('.','_')
utils.git('checkout', ['binutils-{}'.format(version)])
os.mkdir('build')
os.chdir('build')
configureCmd = '../configure --prefix={} --disable-nls'.format(
self.args.installPrefixDir)
if self.args.target != '':
configureCmd += ' --target={}'.format(self.args.target)
subprocess.run(configureCmd,
shell=True,
stdout=subprocess.DEVNULL) # Too much logging overflows 4MB travis-ci log limit
subprocess.run('make all -j {}'.format(multiprocessing.cpu_count()),
shell=True,
stdout=subprocess.DEVNULL) # Too much logging overflows 4MB travis-ci log limit
utils.bash('make install')
return retval
def align(self, sourceLang, targetLang, sourceText, targetText):
"""
Performs word alignment on sourceText to targetText using fast_align.
The output is in well known Pharaoh format.
It's ok to raise Exceptions here. They are handled upstream.
"""
file1, file2 = self.findRawData(sourceLang, targetLang)
out = formatParallel(file1, file2)
sourceText = tokenize(sourceText.replace('\n', ' '), sourceLang)
targetText = tokenize(targetText.replace('\n', ' '), targetLang)
out = ["{} ||| {}".format(sourceText, targetText)] + out
# Found relevant parallel corpus
print("Files found: {}, {}".format(file1, file2))
with open('tmp.parallel', 'w') as tmpFile:
print('\n'.join(out), file=tmpFile)
(output, _
) = bash("align/fast_align/build/fast_align -d -o -v -i tmp.parallel")
os.remove('tmp.parallel')
return {
'status':
'OK',
'alignment':
'\n'.join(output.split('\n')[:(targetText.count('\n') + 1)])
}
def false_sharing(slides: SlideDeck, backup: bool):
if backup:
slide = new_slide(slides)
content = slide_header(slide, "Code (backup)")
code(
content.box(), """// tid - [0, NO_OF_THREADS)
void thread_fn(int tid, double* data)
{
size_t repetitions = 1024 * 1024 * 1024UL;
for (size_t i = 0; i < repetitions; i++)
{
data[tid] *= i;
}
}""")
slide = new_slide(slides)
content = slide_header(slide, "Result (backup)")
content.box(height=600).image("images/example3-time.png")
slide = new_slide(slides)
content = slide_header(slide, "Cache system")
content.box(height=600).image("images/haswell-diagram.png")
content.box(width=230, height=40, x=816, y=530).rect(color=COLOR_BACKEND,
stroke_width=3)
content.box(width=226, height=40, x=240, y=570).rect(color=COLOR_BACKEND,
stroke_width=3)
slide = new_slide(slides)
content = slide_header(slide, "Cache coherency")
content.box(height=600).image("images/false-sharing.svg")
slide = new_slide(slides)
content = slide_header(slide, "False sharing")
content.box(width="90%").image("images/false-sharing-array.svg")
slide = new_slide(slides)
content = slide_header(slide, "How to measure?")
content.box().text("~tt{l2_rqsts.all_rfo}", s(size=48))
content.box(p_top=20).text(
"How many times some core invalidated data in other cores?")
if backup:
bash(content.box(p_top=40, show="next+"),
"""$ perf stat -e l2_rqsts.all_rfo ./example3
1 thread -> 59 711
2 threads -> 1 112 258 710""",
text_style=s(align="left"))
def hunalign(sourceText, targetText):
"""
Performs sentence alignment on sourceText to targetText using Hunalign.
It's ok to raise Exceptions here. They are handled upstream.
"""
sourceText = re.sub('\n', ' ', sourceText)
sourceText = re.sub('\.', '.\n', sourceText)
sourceText = re.sub('\?', '.\n', sourceText)
targetText = re.sub('\n', ' ', targetText)
targetText = re.sub('\.', '.\n', targetText)
targetText = re.sub('\?', '?\n', targetText)
sourceText = sourceText.split('\n')
targetText = targetText.split('\n')
sourceText = list(
filter(lambda x: not re.match('.?.?(\.|\?)', x), sourceText))
targetText = list(
filter(lambda x: not re.match('.?.?(\.|\?)', x), targetText))
with DirCrawler('align/hunalign'):
with open('tmp.src', 'w') as f:
f.write('\n'.join(sourceText) + '\n')
with open('tmp.trg', 'w') as f:
f.write('\n'.join(targetText) + '\n')
(output, _error) = bash(
"perl hunalignwrapper.pl --hunalign=./hunalign/src/hunalign/hunalign tmp.src tmp.trg"
)
os.remove('tmp.src')
os.remove('tmp.trg')
output = list(filter(lambda x: x.strip() != '', output.split('\n')))
align = [[int(y) for y in x.split('\t')[0].split('-')] for x in output]
# Process the ladder output of Hunalign
output = []
for l, r in align:
left = []
right = []
while True:
if l == 0 and r == 0:
output.append([' '.join(left), ' '.join(right)])
break
elif l > r:
left.append(sourceText.pop(0))
l -= 1
continue
elif l < r:
right.append(targetText.pop(0))
r -= 1
continue
else:
left.append(sourceText.pop(0))
right.append(targetText.pop(0))
l -= 1
r -= 1
return output
def repo_add(fpaths):
assert type(fpaths) is list
assert not [None for fpath in fpaths if fpath.parent != fpaths[0].parent]
for fpath in fpaths:
assert issubclass(type(fpath), os.PathLike) and \
fpath.name.endswith(PKG_SUFFIX)
dbpath = fpaths[0].parent / f'{REPO_NAME}.db.tar.gz'
return bash(
f'{REPO_CMD} {dbpath} {" ".join([str(fpath) for fpath in fpaths])}',
RUN_CMD_TIMEOUT=5 * 60)
def _execute(self, script, stdin, language):
"Generate stdout by executing the script with given stdin."
if language.lower() != 'python':
raise Exception('Language not supported: {}'.format(language))
if language == 'python':
fd, filename = tempfile.mktemp(suffix=".py")
os.write(fd, script)
stdout = bash("python " + filename, stdin=stdin)
os.close(fd)
return stdout
# TODO: Add support for more languages
return ''
def qe(self, sourceLang, targetLang, sourceText, targetText):
"""
Performs translation quality estimation on sourceText to targetText using OpenKiwi
It's ok to raise Exceptions here. They are handled upstream.
"""
if not [sourceLang, targetLang] in [['cs', 'de'], ['en', 'de']]:
raise Exception(
f'{sourceLang}-{targetLang} language pair not supported')
# Sanitize input
aligned = hunalign(sourceText, targetText)
sourceText = [tokenize(x[0], sourceLang, False) for x in aligned]
targetText = [tokenize(x[1], sourceLang, False) for x in aligned]
sourceTextPlain = '\n'.join([' '.join(x) for x in sourceText])
targetTextPlain = '\n'.join([' '.join(x) for x in targetText])
with DirCrawler('qe/openkiwi-config'):
fileSource = 'data/input.src'
with open(fileSource, 'w') as fileSourceW:
fileSourceW.write(sourceTextPlain)
fileTarget = 'data/input.trg'
with open(fileTarget, 'w') as fileTargetW:
fileTargetW.write(targetTextPlain)
(_output, _error) = bash(f"""
kiwi predict --config experiments/predict_estimator_{sourceLang}_{targetLang}.yaml
""")
#print(_output)
#print(_error)
fileOut = 'data/tags'
with open(fileOut, 'r') as f:
out = [
1 - float(x.rstrip('\n'))
for x in ' '.join(f.readlines()).split(' ')
]
# map the estimation from an interval to discrete values:
# >= 0.5 -> 1.0
# >= 0.4 -> 0.7
# >= 0.1 -> 0.3
# <= 0.1 -> 0.1
out = map(
lambda x: 1 if x >= 0.5 else 0.7 if x >= 0.4 else 0.3
if x >= 1 else 0.1, out)
out = list(out)
return {'status': 'OK', 'qe': out}
def disconnect(dc,vm_name):
logger=my_logger()
if dc == 'two':
vcent_user,vcent_pass,vcent_ip = settings.TVCENTER_USER, settings.TVCENTER_PASSWORD, settings.TVCENTER_IP
elif dc == 'san':
vcent_user,vcent_pass,vcent_ip = settings.SVCENTER_USER, settings.SVCENTER_PASSWORD, settings.SVCENTER_IP
else:
vcent_user,vcent_pass,vcent_ip = settings.FVCENTER_USER, settings.FVCENTER_PASSWORD, settings.FVCENTER_IP
dis='govc device.disconnect -u "%s":"%s"@"%s" -k -dc="%s" -vm "%s" cdrom-3000' \
%(vcent_user,vcent_pass,vcent_ip,dc,vm_name)
stdout, stderr = bash(dis)
if stderr:
logger.error('机房: %s VM_Name: %s cdrom断开连接失败 %s' % (dc, vm_name, stderr.decode('utf-8')))
else:
logger.info('机房: %s VM_Name: %s cdrom已断开连接' % (dc, vm_name))
def repo_remove(fpaths):
assert type(fpaths) is list
assert not [None for fpath in fpaths if fpath.parent != fpaths[0].parent]
for fpath in fpaths:
assert issubclass(type(fpath), os.PathLike) and \
fpath.name.endswith(PKG_SUFFIX)
dbpath = fpaths[0].parent / f'{REPO_NAME}.db.tar.gz'
for fpath in fpaths:
throw_away(fpath)
sigpath = fpath.parent / f'{fpath.name}.sig'
# there is a fscking problem that fscking pathlib always follow symlinks
if sigpath.exists() or sigpath.is_symlink():
throw_away(sigpath)
pkgnames = [
get_pkg_details_from_name(fpath.name).pkgname for fpath in fpaths
]
return bash(f'{REPO_REMOVE_CMD} {dbpath} {" ".join(pkgnames)}',
RUN_CMD_TIMEOUT=5 * 60)
def unzip(self):
"""Unpack the source"""
utils.bash("unzip {} -d {}".fromat(self.filepath, self.dir))
output = b''
except BaseException as err:
output = b''
if bytes(args.o.ver, 'utf-8') in output:
print('{app} {ver} is already installed'.format(app=args.o.src, ver=VER))
exit(0)
else:
print('compiling {app} {ver}'.format(app=args.o.src, ver=VER))
os.makedirs(args.o.src, exist_ok=True)
try:
url = 'git://sourceware.org/git/binutils-gdb.git'
utils.git('clone', [url, args.o.src])
os.chdir(args.o.src)
version = VER.replace('.','_')
utils.git('checkout', ['binutils-{ver}'.format(ver=version)])
os.mkdir('build')
os.chdir('build')
except BaseException as ex:
traceback.print_exc()
exit(1)
print('configure')
utils.bash('../configure --prefix={0} --target={1} --disable-nls'
.format(args.o.prefix, TARGET))
utils.bash('make all -j {}'.format(multiprocessing.cpu_count()))
utils.bash('make install')
exit(0)
def __sign(self, job):
logger.info('signing in %s %s', job.pkgconfig.dirname, job.arch)
cwd = REPO_ROOT / job.pkgconfig.dirname
for fpath in cwd.iterdir():
if fpath.name.endswith(PKG_SUFFIX):
bash(f'{GPG_SIGN_CMD} {fpath.name}', cwd=cwd)
def qe(self, sourceLang, targetLang, sourceText, targetText):
"""
Performs translation quality estimation on sourceText to targetText using QuEst++ and fast_align
It's ok to raise Exceptions here. They are handled upstream.
"""
os.makedirs('data/tmp', exist_ok=True)
if not [sourceLang, targetLang] in self.supportedPairs:
raise Exception("{}-{} language pair not supported".format(
sourceLang, targetLang))
# Sanitize input
aligned = hunalign(sourceText, targetText)
sourceText = [tokenize(x[0], sourceLang, False) for x in aligned]
targetText = [tokenize(x[1], sourceLang, False) for x in aligned]
sourceTextPlain = '\n'.join([' '.join(x) for x in sourceText])
targetTextPlain = '\n'.join([' '.join(x) for x in targetText])
alignments = fast_align.FastAlign().align(sourceLang, targetLang,
sourceTextPlain,
targetTextPlain)['alignment']
with open('data/tmp/alignments', 'w') as fileAlignments:
fileAlignments.write(alignments)
with open('data/tmp/source', 'w') as fileSource:
fileSource.write(sourceTextPlain)
with open('data/tmp/target', 'w') as fileTarget:
fileTarget.write(targetTextPlain)
with DirCrawler('qe/questplusplus'):
print("Extracting features")
(_output, _error) = bash("""
java -cp QuEst++.jar:lib/* shef.mt.WordLevelFeatureExtractor
-lang english spanish
-input ../../data/tmp/source ../../data/tmp/target
-alignments ../../data/tmp/alignments
-config ../questplusplus-config/config.word-level.properties
""")
outputFile = 'output/test/output.txt'
if not os.path.isfile(outputFile):
raise Exception('Server Processing Error')
with open(outputFile, 'r') as outputFileR:
features = outputFileR.readlines()
os.remove('data/tmp/alignments')
os.remove('data/tmp/source')
os.remove('data/tmp/target')
features = [[
x.split('=')[1] for x in line.rstrip('\n').rstrip('\t').split('\t')
] for line in features]
with open('data/tmp/features', 'w') as fileFeatures:
fileFeatures.write('\n'.join(['\t'.join(x) for x in features]))
with open('data/tmp/labels', 'w') as fileLabels:
fileLabels.write('\n'.join(['1'] * len(features)))
with DirCrawler('qe/questplusplus'):
print("Removing output directory structure for feature extractor")
os.remove(outputFile)
os.rmdir('output/test')
os.rmdir('output')
print("Machine Learning")
(_output, _error) = bash(f"""
python learning/src/learn_model.py ../questplusplus-config/svr_{sourceLang}_{targetLang}.cfg
""")
with open('predicted.csv', 'r') as predictedFile:
output = [
float(x.rstrip('\n').split('\t')[1])
for x in predictedFile.readlines()
]
os.remove('predicted.csv')
os.remove('data/tmp/features')
os.remove('data/tmp/labels')
os.rmdir('data/tmp')
return {'status': 'OK', 'qe': output}
def cache_conflicts(slides: SlideDeck, backup: bool):
if backup:
slide = new_slide(slides)
content = slide_header(slide, "Code (backup)")
code(
content.box(),
"""// Addresses of N integers, each `offset` bytes apart
std::vector<int*> data = ...;
for (auto ptr: data)
{
*ptr += 1;
}
// Offsets: 4, 64, 4000, 4096, 4128""")
slide = new_slide(slides)
content = slide_header(slide, "Result (backup)")
content.box(height=600).image("images/example1-time.png")
slide = new_slide(slides)
content = slide_header(slide, "Cache memory")
content.box(height=600).image("images/haswell-diagram.png")
content.box(width=230, height=40, x=816, y=530).rect(color=COLOR_BACKEND,
stroke_width=3)
slide = new_slide(slides)
content = slide_header(slide, "How are (L1) caches implemented")
list_wrapper = content.box()
list_item(list_wrapper).text("N-way set associative table")
list_item(list_wrapper, level=1, show="last+").text("Hardware hash table")
list_item(list_wrapper, show="next+").text("Key = address (8B)")
list_item(list_wrapper, show="next+").text("Entry = cache line (64B)")
slide = new_slide(slides)
content = slide_header(slide, "N-way set associative cache")
hash_size = 8
hash_dimension = 60
def table(wrapper: Box,
size,
dimension,
buckets=None,
bucket_indices=True):
htable = wrapper.box(horizontal=True)
items = []
for i in range(size):
cell = htable.box(width=dimension,
height=dimension,
horizontal=True).rect("black", stroke_width=2)
items.append(cell)
if buckets:
bucket_width = int((size / buckets) * dimension)
for i in range(buckets):
pos = i * bucket_width
htable.box(x=pos, y=0, width=bucket_width,
height=dimension).rect("black", stroke_width=6)
if bucket_indices:
htable.box(x=pos, y=dimension - 5,
width=bucket_width).text(str(i))
return (htable, items)
content.box().text("Size = {} cache lines".format(hash_size),
style="notice")
(htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension)
arrow_wrapper = content.box()
arrow = Arrow(20)
arrow_wrapper.box().line([
hitems[0].p("50%", 0).add(0, -20),
hitems[-1].p("50%", 0).add(0, -20),
],
start_arrow=arrow,
end_arrow=arrow,
stroke_width=5,
color=COLOR_NOTE)
content.box(
p_top=20,
show="next+").text("Associativity (N) - # of cache lines per bucket")
content.box(p_top=10, show="next+").text("# of buckets = Size / N")
row = content.box(horizontal=True, p_top=20)
lcol = row.box(y=0)
rcol = row.box(y=0, p_left=20)
def htable_row(text, block_count):
padding = 20
height = 110
lcol.box(show="next+", p_top=padding, height=height).text(text)
return table(rcol.box(show="last+", p_top=padding, height=height),
hash_size, hash_dimension, block_count)
htable_row("N = 1 (direct mapped)", hash_size)
htable_row("N = {} (fully associative)".format(hash_size), 1)
htable_row("N = 2", hash_size // 2)
slide = new_slide(slides)
content = slide_header(slide, "How are addresses hashed?")
content.box().text("64-bit address:")
row = content.box(horizontal=True, width=800)
widths = ["60%", "25%", "15%"]
address_colors = ["#B22222", "#007944", "#0018AE"]
labels = ["Tag", "Index", "Offset"]
for i in range(3):
wrapper = row.box(width=widths[i]).rect(color=address_colors[i],
stroke_width=4)
wrapper.box(padding=4).text(labels[i])
labelrow = content.box(horizontal=True, x=220)
labelrow.box().text("63")
labelrow.box(p_left=770).text("0")
list_wrapper = content.box(p_top=20)
list_item(list_wrapper, show="next+").text("Offset", "bold")
list_item(list_wrapper, level=1,
show="last+").text("Selects byte within a cache line")
list_item(list_wrapper, level=1,
show="last+").text("log2(cache line size) bits")
list_item(list_wrapper, show="next+").text("Index", "bold")
list_item(list_wrapper, level=1,
show="last+").text("Selects bucket within the cache")
list_item(list_wrapper, level=1,
show="last+").text("log2(bucket count) bits")
list_item(list_wrapper, show="next+").text("Tag", "bold")
list_item(list_wrapper, level=1, show="last+").text("Used for matching")
slide = new_slide(slides)
content = slide_header(slide, "N-way set associative cache")
queue = content.box(x="55%", y=40, horizontal=True)
queue.box(p_right=40).text("Cache lines:")
colors = ("#F0134D", "#FF6F5E", "#F0134D")
cacheline_labels = ("A", "B", "C")
for i in range(3):
queue.box(width=hash_dimension,
height=hash_dimension).rect(color="black",
bg_color=colors[i],
stroke_width=5).text(
cacheline_labels[i],
style=s(bold=True,
color="white"))
index = content.box(x="55%", y=90, horizontal=True)
index.box(p_right=75).text("Index bits:")
index_bits = (0, 1, 0)
for i in range(3):
index.box(width=hash_dimension,
height=hash_dimension).text(str(index_bits[i]))
def insert(slot, show, item):
wrapper = slot.overlay(show=show)
wrapper.rect(bg_color=colors[item])
wrapper.text(cacheline_labels[item], style=s(bold=True, color="white"))
row = content.box(horizontal=True, p_top=20)
lcol = row.box(y=0)
rcol = row.box(y=0, p_left=20)
def htable_row(text, block_count):
padding = 20
height = 140
lcol.box(show="next+", p_top=padding, height=height).text(text)
return table(rcol.box(show="last+", p_top=padding, height=height),
hash_size, hash_dimension, block_count)
(_, hitems) = htable_row("N = 1", hash_size)
insert(hitems[0], "next+", 0)
insert(hitems[1], "next+", 1)
insert(hitems[0], "next+", 2)
(_, hitems) = htable_row("N = {}".format(hash_size), 1)
insert(hitems[0], "next+", 0)
insert(hitems[1], "next+", 1)
insert(hitems[2], "next+", 2)
(_, hitems) = htable_row("N = 2", hash_size // 2)
insert(hitems[0], "next+", 0)
insert(hitems[2], "next+", 1)
insert(hitems[1], "next+", 2)
slide = new_slide(slides)
slide.update_style("default", s(size=46))
slide.update_style("bold", s(size=46))
content = slide_header(slide, "Intel L1 cache")
bash(content.box(),
"""$ getconf -a | grep LEVEL1_DCACHE
LEVEL1_DCACHE_SIZE 32768
LEVEL1_DCACHE_ASSOC 8
LEVEL1_DCACHE_LINESIZE 64""",
text_style=s(align="left"))
list_wrapper = content.box(p_top=20)
list_item(
list_wrapper,
show="next+").text("~bold{Cache line size} - 64 B (6 offset bits)")
list_item(list_wrapper, show="next+").text("~bold{Associativity} (N) - 8")
list_item(list_wrapper, show="next+").text("~bold{Size} - 32768 B")
list_item(list_wrapper, show="next+").text("32768 / 64 => 512 cache lines")
list_item(list_wrapper,
show="next+").text("512 / 8 => 64 buckets (6 index bits)")
slides.set_style("tag", s(color=address_colors[0]))
tag = slides.get_style("tag")
slides.set_style("index", tag.compose(s(color=address_colors[1])))
slides.set_style("offset", tag.compose(s(color=address_colors[2])))
styles = ["tag", "index", "offset"]
colors = ["#F0134D", "#FF6F5E", "#1F6650", "#40BFC1"]
def address(cols, content, next=True, use_style=True, row=0):
for i, col in enumerate(cols):
show = "1+"
if next:
show = "next+" if i == 0 else "last+"
style = "default"
if use_style:
if i == 0:
style = s(color=colors[row])
else:
style = styles[i - 1]
col.box(show=show).text(content[i], style=style)
slide = new_slide(slides)
content = slide_header(slide, "Offset = 4B")
width = 700
columns = 4
row = content.box(horizontal=True)
cols = [row.box(width=width // columns) for _ in range(columns)]
address(cols, ("Number", "Tag", "Index", "Offset"),
next=False,
use_style=False)
address(cols, ("A", "..100000", "000000", "000000"), next=False)
address(cols, ("B", "..100000", "000000", "000100"), row=1)
address(cols, ("C", "..100000", "000000", "001000"), row=2)
address(cols, ("D", "..100000", "000000", "001100"), row=3)
hash_dimension = 80
(htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension,
hash_size // 2)
for i in range(4):
hitems[0].box(show="{}+".format(i + 1),
width=hash_dimension // 4,
height=hash_dimension).rect(bg_color=colors[i])
list_wrapper = content.box(p_top=40)
list_item(
list_wrapper,
show="next+").text("Same bucket, same cache line for each number")
list_item(list_wrapper,
show="next+").text("Most efficient, no space is wasted")
slide = new_slide(slides)
content = slide_header(slide, "Offset = 64B")
row = content.box(horizontal=True)
cols = [row.box(width=width // columns) for _ in range(columns)]
address(cols, ("Number", "Tag", "Index", "Offset"),
next=False,
use_style=False)
address(cols, ("A", "..100000", "000000", "000000"), next=False)
address(cols, ("B", "..100000", "000001", "000000"), row=1)
address(cols, ("C", "..100000", "000010", "000000"), row=2)
address(cols, ("D", "..100000", "000011", "000000"), row=3)
(htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension,
hash_size // 2)
for i in range(4):
hitems[i * 2].box(show="{}+".format(i + 1),
width=hash_dimension // 4,
height=hash_dimension,
x=0).rect(bg_color=colors[i])
list_wrapper = content.box(p_top=40)
list_item(list_wrapper,
show="next+").text("Different bucket for each number")
list_item(list_wrapper, show="next+").text("Wastes 60B in each cache line")
list_item(list_wrapper,
show="next+").text("Equally distributed among buckets")
slide = new_slide(slides)
content = slide_header(slide, "Offset = 4096B")
row = content.box(horizontal=True)
cols = [row.box(width=width // columns) for _ in range(columns)]
address(cols, ("Number", "Tag", "Index", "Offset"),
next=False,
use_style=False)
address(cols, ("A", "..100000", "000000", "000000"), next=False)
address(cols, ("B", "..100001", "000000", "000000"), row=1)
address(cols, ("C", "..100010", "000000", "000000"), row=2)
address(cols, ("D", "..100011", "000000", "000000"), row=3)
(htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension,
hash_size // 2)
for i in range(4):
hitems[i % 2].box(show="{}+".format(i + 1),
width=hash_dimension // 4,
height=hash_dimension,
x=0).rect(bg_color=colors[i])
list_wrapper = content.box(p_top=40)
list_item(list_wrapper, show="next+").text(
"Same bucket, but different cache lines for each number!")
list_item(list_wrapper,
show="next+").text("Bucket full => evictions necessary")
slide = new_slide(slides)
content = slide_header(slide, "How to measure?")
content.box().text("~tt{l1d.replacement}", style=s(size=48))
content.box(
p_top=20).text("How many times was a cache line loaded into L1?")
if backup:
bash(content.box(p_top=40, show="next+"),
"""$ perf stat -e l1d.replacement ./example1
4B offset -> 149 558
4096B offset -> 426 218 383""",
text_style=s(align="left"))
def run_commands(step):
for data in step.hashes:
if 'noFail' == data['Expected'].strip():
step_assert(step).assert_true(utils.bash(data['Command']).successful())
else:
step_assert(step).assert_equals(utils.bash(data['Command']).output_text().strip(),data['Expected'].strip())
def project(slides: Slides):
slide = slides.new_slide()
content = slide_header(slide, "Project management (Cargo)")
content.box(height=400).image("imgs/cargo.png")
slide = slides.new_slide()
slide.update_style("code", s(size=30))
content = slide_header(slide, "Using libraries")
line = content.box(width="fill", horizontal=True)
cargo = line.box(width="50%", p_right=50)
cargo.box().text("Cargo.toml")
cargo_code = code(
cargo.box(), """
[package]
name = "hello_world"
version = "0.1.0"
[dependencies]
ibverbs = "0.4"
json = "1.0"
protobuf = "2.0"
""", "toml")
main = line.box(width="50%", show="2+", y=0)
main.box().text("main.rs")
main_code = code(
main.box(), """
use json::parse;
fn main() {
parse("data.json");
}""")
arrow = Arrow(20)
p1 = cargo_code.line_box(5).p("100%", "50%")
p2 = main_code.line_box(0).p(0, "50%")
slide.box(show="2+").line(
[p1.add(-40, 0), p1, p2.add(-10, 0)],
stroke_width=5,
color="orange",
end_arrow=arrow)
content.box(height=10)
content.box(show="3+").text("More than 26k libraries available")
slide = slides.new_slide()
content = slide_header(slide, "Unified documentation")
content.box(width=900).image("imgs/rust-docs.png")
slide = slides.new_slide()
content = slide_header(slide, "Integrated tooling")
def cargo_line(parent, text, code, show="1+", **text_args):
wrapper = parent.box(width="fill", horizontal=True, show=show)
textbox = wrapper.box(width="50%")
textbox = textbox.text(text, **text_args)
codebox = wrapper.box(width="40%")
bash(codebox, code, x=0, width="fill")
return textbox
wrapper = content.box(width="fill")
cargo_line(wrapper, "Build", "$ cargo build", "1+")
cargo_line(wrapper, "Run", "$ cargo run", "2+")
slide = slides.new_slide()
slide.update_style("code", s(size=40))
content = slide_header(slide, "Integrated tooling (tests)")
code(content.box(), """
#[test]
fn test_add() {
assert_eq!(add(1, 2), 3);
}
""")
content.box(height=20)
bash(content.box(show="2+"), "$ cargo test")
slide = slides.new_slide()
slide.update_style("code", s(size=40))
content = slide_header(slide, "Integrated tooling (benchmarks)")
code(
content.box(), """
#[bench]
fn bench_add_two(b: &mut Bencher) {
b.iter(|| add_two(2));
}
""")
content.box(height=20)
bash(content.box(show="2+"), "$ cargo bench")
slide = slides.new_slide()
content = slide_header(slide, "Integrated tooling")
wrapper = content.box(width="fill")
cargo_line(wrapper, "Format", "$ cargo fmt")
cargo_line(wrapper, "Lint", "$ cargo clippy", "next+")
box = cargo_line(wrapper, "Publish to SC", "$ cargo publish", "next+")
box = box.line_box(0)
box.line([box.p("53%", "55%"), box.p("73%", "55%")], stroke_width=3)
slide = slides.new_slide()
slide.update_style("code", s(size=24))
content = slide_header(slide, "Build scripts")
content.box().text("build.rs")
code_width = 940
code_step(content.box(width=code_width, height=400),
"""
fn main() {
// generate Protobuf objects
protoc_rust::run("protobuf/message.proto", "src/protos");
// generate C headers
cbindgen::Builder::new()
.generate()
.write_to_file("bindings.h");
}
""",
"1", ((0, 1, 2, 3, None, None, None, None, 8),
(0, 1, 2, 3, 4, 5, 6, 7, 8)),
width=code_width)
slide = slides.new_slide()
content = slide_header(slide, "(interlude)")
content.box(height=600).image("imgs/meme-cargo.jpg")
#!/usr/bin/env python3
import os, sys
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASE_DIR)
from utils import bash
if __name__ == '__main__':
live = 'ps -ef |grep "create_vm_start.py"|grep -v grep|wc -l'
stdout, stderr = bash(live)
if int(stdout.decode('utf-8')) == 0:
start = '/usr/bin/python3 /export/VMWare_Auto/vm_create/bin/create_vm_start.py &>/dev/null'
bash(start)
readpkglog as extra_readpkglog, \
readmainlog as extra_readmainlog
abspath = os.path.abspath(__file__)
abspath = os.path.dirname(abspath)
os.chdir(abspath)
logger = logging.getLogger('buildbot')
configure_logger(logger,
logfile=MAIN_LOGFILE,
rotate_size=1024 * 1024 * 10,
enable_notify=True,
consolelog=CONSOLE_LOGFILE)
# refuse to run in systemd-nspawn
if 'systemd-nspawn' in bash('systemd-detect-virt || true'):
logger.error('Refused to run in systemd-nspawn.')
raise AssertionError('Refused to run in systemd-nspawn.')
REPO_ROOT = Path(PKGBUILD_DIR)
class Job:
def __init__(self, buildarch, pkgconfig, version, multiarch=False):
assert buildarch in BUILD_ARCHS
self.arch = buildarch
self.pkgconfig = pkgconfig
self.version = version
self.multiarch = multiarch
self.added = time()
def branch_prediction(slides: SlideDeck, backup: bool):
if backup:
slide = new_slide(slides)
content = slide_header(slide, "Code (backup)")
code(
content.box(),
"""std::vector<float> data = /* 32K random floats in [1, 10] */;
float sum = 0;
// std::sort(data.begin(), data.end());
for (auto x : data)
{
if (x < 6.0f)
{
sum += x;
}
}""")
slide = new_slide(slides)
content = slide_header(slide, "Result (backup)")
content.box(width="90%").image("images/example0a-time.png")
slide = new_slide(slides)
content = slide_header(slide, "Most upvoted Stack Overflow question")
content.box(height=600).image("images/stack-overflow.png")
slide = new_slide(slides)
content = slide_header(slide,
"What is going on? (Intel Amplifier - VTune)")
content.box(height=600).image("images/vtune.png")
slide = new_slide(slides)
content = slide_header(slide, "What is going on? (perf)")
bash(content.box(),
"""$ perf stat ./example0a --benchmark_filter=nosort
853,672012 task-clock (msec) # 0,997 CPUs utilized
30 context-switches # 0,035 K/sec
0 cpu-migrations # 0,000 K/sec
199 page-faults # 0,233 K/sec
3 159 530 915 cycles # 3,701 GHz
1 475 799 619 instructions # 0,47 insn per cycle
419 608 357 branches # 491,533 M/sec
102 425 035 branch-misses # 24,41% of all branches""",
text_style=s(align="left", size=34))
slide = new_slide(slides)
content = slide_header(slide, "Branch predictor")
content.box(height=600).image("images/haswell-diagram.png")
content.box(width=130, height=40, x=595, y=140).rect(color=COLOR_FRONTEND,
stroke_width=3)
slide = new_slide(slides)
content = slide_header(slide, "CPU pipeline 101")
content.box(height=400).image("images/branch-miss-pipeline.svg")
slide = new_slide(slides)
content = slide_header(slide, "Branch predictor")
list_wrapper = content.box()
list_item(list_wrapper).text("CPU tries to predict results of branches")
list_item(list_wrapper,
show="next+").text("Misprediction can cost ~15-20 cycles!",
escape_char="#")
slide = new_slide(slides)
content = slide_header(slide, "Simple branch predictor - unsorted array")
code(content.box(p_bottom=40), """if (data[i] < 6) {
...
}""")
row = content.box(horizontal=True)
box_dimension = 100
def array(numbers, predictions, start=1, needle=6):
stroke_width = 2
size = 36
for i in range(len(numbers)):
box = row.box(width=box_dimension,
height=box_dimension).rect(color="black",
stroke_width=stroke_width)
number = str(numbers[i])
box.text(number, s(bold=True, size=size))
predicted_correctly = (predictions[i] and numbers[i] < needle) or (
not predictions[i] and numbers[i] >= needle)
prediction = "green" if predicted_correctly else "red"
show_overlay = "{}+".format(start + i * 2 + 1)
overlay = box.overlay(show=show_overlay).rect(
color="black", bg_color=prediction, stroke_width=stroke_width)
overlay.text(number, s(color="white", bold=True, size=size))
show_text = start + i * 2
row.box(x=i * box_dimension,
y=box_dimension,
width=box_dimension,
show="{}-{}".format(show_text,
show_text + 1)).text("{} < {}?".format(
number, needle))
values = [6, 2, 1, 7, 4, 8, 3, 9]
text_style = s(align="left", size=42)
width = 400
def predict_sequence(wrapper: Box, values, start=1):
for i in range(len(values)):
value = "Taken" if values[i] else "Not taken"
show_start = start + i * 2
wrapper.overlay(
show="{}-{}".format(show_start, show_start + 1)).rect(
bg_color="white").text("Prediction: {}".format(value),
style=text_style)
def predict_value(index):
if index == 0:
return False
return values[index - 1] < 6
predictions = [predict_value(i) for i in range(len(values))]
array(values, predictions, start=2)
prediction_wrapper = content.box(p_top=60,
width=width).text("Prediction: Not taken",
style=text_style)
predict_sequence(prediction_wrapper, predictions, start=1)
content.box(show="next+", p_top=40).text("2 hits, 6 misses (25% hit rate)")
slide = new_slide(slides)
content = slide_header(slide, "Simple branch predictor - sorted array")
code(content.box(p_bottom=40), """if (data[i] < 6) {
...
}""")
row = content.box(horizontal=True)
values = [1, 2, 3, 4, 6, 7, 8, 9]
predictions = [predict_value(i) for i in range(len(values))]
array(values, predictions, start=2)
prediction_wrapper = content.box(p_top=60,
width=width).text("Prediction: Not taken",
style=text_style)
predict_sequence(prediction_wrapper, predictions, start=1)
content.box(show="next+", p_top=40).text("6 hits, 2 misses (75% hit rate)")
if backup:
size = 40
slide = new_slide(slides)
content = slide_header(slide, "How can the compiler help?")
row = content.box(horizontal=True)
row.box(y=0, p_right=50, width=400).image("images/bm-float-code.png")
row.box(y=0, width=600).image("images/bm-float-bin.png")
content.box(p_top=20).text(
"With ~tt{float}, there are two branches per iteration",
style=s(size=size))
slide = new_slide(slides)
content = slide_header(slide, "How can the compiler help?")
row = content.box(horizontal=True)
row.box(y=0, p_right=50, width=400).image("images/bm-int-code.png")
row.box(y=0, width=600).image("images/bm-int-bin.png")
content.box(p_top=20).text(
"With ~tt{int}, one branch is removed (using ~tt{cmov})",
style=s(size=size))
slide = new_slide(slides)
content = slide_header(slide, "How to measure?")
content.box().text("~tt{branch-misses}", style=s(size=48))
content.box(p_top=20).text("How many times was a branch mispredicted?")
if backup:
bash(content.box(p_top=40, show="next+"),
"""$ perf stat -e branch-misses ./example0a
with sort -> 383 902
without sort -> 101 652 009""",
text_style=s(align="left"))
slide = new_slide(slides)
slide.update_style("code", s(size=40))
content = slide_header(slide, "How to help the branch predictor?")
list_wrapper = content.box()
list_item(list_wrapper).text("More predictable data")
list_item(list_wrapper, show="next+").text("Profile-guided optimization")
list_item(list_wrapper,
show="next+").text("Remove (unpredictable) branches")
list_item(list_wrapper,
show="next+").text("Compiler hints (use with caution)")
code(
list_wrapper.box(show="last+", p_top=20, p_bottom=20),
"""if (__builtin_expect(will_it_blend(), 0)) {
// this branch is not likely to be taken
}""")
slide = new_slide(slides)
content = slide_header(slide, "Branch target prediction")
list_wrapper = content.box()
list_item(list_wrapper).text(
"Target of a jump is not known at compile time:")
list_item(list_wrapper, show="next+", level=1).text("Function pointer")
list_item(list_wrapper, show="next+",
level=1).text("Function return address")
list_item(list_wrapper, show="next+", level=1).text("Virtual method")
if backup:
slide = new_slide(slides)
slide.update_style("code", s(size=26))
content = slide_header(slide, "Code (backup)")
code(
content,
"""struct A { virtual void handle(size_t* data) const = 0; };
struct B: public A { void handle(size_t* data) const final { *data += 1; } };
struct C: public A { void handle(size_t* data) const final { *data += 2; } };
std::vector<std::unique_ptr<A>> data = /* 4K random B/C instances */;
// std::sort(data.begin(), data.end(), /* sort by instance type */);
size_t sum = 0;
for (auto& x : data)
{
x->handle(&sum);
}""")
slide = new_slide(slides)
content = slide_header(slide, "Result (backup)")
content.box(width="90%").image("images/example0b-time.png")
slide = new_slide(slides)
content = slide_header(slide, "perf (backup)")
bash(content.box(),
"""$ perf stat -e branch-misses ./example0b
with sort -> 337 274
without sort -> 84 183 161""",
text_style=s(align="left"))
def unsafe(slides: Slides):
slide = slides.new_slide()
slide.update_style("code", s(size=32))
content = slide_header(slide, "Where's the catch?")
content.box().text("We have seen things that mutate through a shared borrow")
code_width = 800
code_step(content.box(width=code_width, height=400), """
// Arc::clone
fn clone(&self) -> Arc<T>;
// Mutex::lock
fn lock(&self) -> &mut T;
// AtomicU64::store
fn store(&self, val: u64, order: Ordering);
""", 2, [
(0, 1, None, None, None, None),
(0, 1, 2, 3, None, None),
(0, 1, 2, 3, 4, 5)
], width=code_width)
content.box(show="5+").text("This is called ~tt{interior mutability} and requires unsafe Rust",
s(size=32))
slide = slides.new_slide()
content = slide_header(slide, "Enter unsafe Rust")
content.box().text("Some scenarios are not expressible in (safe) Rust")
content.box(height=20)
content.box(show="next+").text("In some cases, something more is required to:")
list = content.box()
list_item(list, show="next+").text("Express inherently unsafe paradigms")
list_item(list, show="next+").text("Improve performance")
list_item(list, show="next+").text("Interact with I/O, OS, hardware, network")
slide = slides.new_slide()
content = slide_header(slide, "Unsafe Rust")
content.box().text("You can mark parts of code with the ~tt{unsafe} keyword")
content.box(show="next+").text("Unsafe Rust is a ~emph{superset} of Rust")
def unsafe_slide(header, code_body, content_show="1", code_size=36):
slide = slides.new_slide()
slide.update_style("code", s(size=code_size))
content = slide_header(slide, "Unsafe Rust")
content.box(y=0).text("Unsafe Rust allows:")
content.box(height=20)
content.box(show=content_show).text(header)
content.box(height=10)
code(content.box(show=content_show), code_body)
unsafe_slide("Accessing a global mutable variable", """
static mut COUNTER: u32 = 0;
fn increment_count() {
unsafe {
COUNTER += 1;
}
}""", content_show="2+")
unsafe_slide("Dereferencing a raw pointer", """
let ptr = 0xCAFECAFE as *mut u32;
unsafe {
*ptr = 5;
}""")
unsafe_slide("Calling an unsafe function", """
unsafe {
zlib_compress(&buffer, buffer.len());
}""")
unsafe_slide("Implementing an unsafe trait", """
unsafe impl Send for MySuperSafeType {
...
}""")
slide = slides.new_slide()
slide.update_style("code", s(size=18))
content = slide_header(slide, "Finding unsafe code - C++")
code_box = code(content, """
std::atomic<LifecycleId> ArenaImpl::lifecycle_id_generator_;
GOOGLE_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, NULL};
void ArenaImpl::Init() {
lifecycle_id_ =
lifecycle_id_generator_.fetch_add(1, std::memory_order_relaxed);
hint_.store(nullptr, std::memory_order_relaxed);
threads_.store(nullptr, std::memory_order_relaxed);
if (initial_block_) {
// Thread which calls Init() owns the first block. This allows the
// single-threaded case to allocate on the first block without having to
// perform atomic operations.
new (initial_block_) Block(options_.initial_block_size, NULL);
SerialArena* serial =
SerialArena::New(initial_block_, &thread_cache(), this);
serial->set_next(NULL);
threads_.store(serial, std::memory_order_relaxed);
space_allocated_.store(options_.initial_block_size,
std::memory_order_relaxed);
CacheSerialArena(serial);
} else {
space_allocated_.store(0, std::memory_order_relaxed);
}
}
""")
code_box.overlay(show="2+", z_level=99).rect(bg_color=CODE_HIGHLIGHT_COLOR)
slide = slides.new_slide()
content = slide_header(slide, "Finding unsafe code - Rust")
bash(content.box(), '$ grep "unsafe" main.rs', text_style=s(size=40))
slide = slides.new_slide()
slide.box().text("""Rust builds safe abstractions
on top of unsafe foundations""", s(size=50))
def denormals(slides: SlideDeck, backup: bool):
if backup:
slide = new_slide(slides)
content = slide_header(slide, "Code (backup)")
code(content.box(), """float F = static_cast<float>(std::stof(argv[1]));
std::vector<float> data(4 * 1024 * 1024, 1);
for (int r = 0; r < 100; r++)
{
for (auto& item: data)
{
item *= F;
}
}""")
slide = new_slide(slides)
content = slide_header(slide, "Result (backup)")
content.box(width="50%").image("images/example2-time.png")
slide = new_slide(slides)
colors = ["#B22222", "#007944", "#0018AE"]
styles = ["sign", "exponent", "significand"]
for i, style in enumerate(styles):
slide.set_style(style, s(color=colors[i], size=42))
content = slide_header(slide, "Denormal floating point numbers")
row = content.box(horizontal=True)
box_dimension = 60
def floating_point(wrapper: Box, colors, values):
for i in range(len(colors)):
box = wrapper.box(width=box_dimension, height=box_dimension)
box.rect(color="black", bg_color=colors[i], stroke_width=2)
box.text(str(values[i]), s(color="white", bold=True))
floating_point(row, [colors[0]] + [colors[1]] * 5 + [colors[2]] * 10, [
0,
0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 1
])
row_label = content.box(x=0, horizontal=True, show="next+")
row_label.box(x=250, y=0).text("Zero exponent")
row_label.box(x=670, y=0).text("Non-zero significand")
content.box(p_top=80, width=700, height=50).image("images/float.svg")
list_wrapper = content.box(p_top=20)
list_item(list_wrapper, show="next+").text("Numbers close to zero")
list_item(list_wrapper, show="last+").text("Hidden bit = 0, smaller bias")
content.box(p_top=40, show="next+").text("Operations on denormal numbers are slow!", style=s(size=46))
slide = new_slide(slides)
content = slide_header(slide, "Floating point handling")
content.box(height=600).image("images/haswell-diagram.png")
content.box(width=130, height=32, x=800, y=52).rect(color=COLOR_FRONTEND, stroke_width=3)
content.box(width=35, height=100, x=988, y=80).rect(color=COLOR_FRONTEND, stroke_width=3)
content.box(width=80, height=165, x=212, y=335).rect(color=COLOR_BACKEND, stroke_width=3)
slide = new_slide(slides)
content = slide_header(slide, "How to measure?")
content.box().text("~tt{fp_assist.any}", style=s(size=48))
content.box(p_top=20).text("How many times the CPU switched to the microcode FP handler?")
if backup:
bash(content.box(p_top=40, show="next+"), """$ perf stat -e fp_assist.any ./example2
0 -> 0
0.3 -> 15 728 640""", text_style=s(align="left"))
slide = new_slide(slides)
slide.update_style("code", s(size=40))
content = slide_header(slide, "How to fix it?")
list_wrapper = content.box()
list_item(list_wrapper).text("The nuclear option: ~tt{-ffast-math}")
list_item(list_wrapper, level=1).text("Sacrifice correctness to gain more FP performance")
list_item(list_wrapper, show="next+").text("Set CPU flags:")
list_item(list_wrapper, level=1, show="last+").text("Flush-to-zero - treat denormal outputs as 0")
list_item(list_wrapper, level=1, show="last+").text("Denormals-to-zero - treat denormal inputs as 0")
code(list_wrapper.box(p_top=40, show="next+"), """_mm_setcsr(_mm_getcsr() | 0x8040);
// or
_MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON);
_MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
""")
def qe(self, sourceLang, targetLang, sourceText, targetText):
"""
Performs translation quality estimation on sourceText to targetText using deepQuest
It's ok to raise Exceptions here. They are handled upstream.
"""
task_name = f'{sourceLang}_{targetLang}'
if not task_name in self.pairsEpochs.keys():
raise Exception(
f'{sourceLang}-{targetLang} language pair not supported')
#if not os.path.isdir(f'data/{task_name}'):
if not os.path.isdir(f'qe/deepQuest-config/saved_models/{task_name}'):
raise Exception(f'data/{task_name} does not exits')
os.makedirs('data/tmp', exist_ok=True)
aligned = hunalign(sourceText, targetText)
repeatText = lambda text, times=100: '\n'.join([text] * times)
# Sanitize input
sourceText = [tokenize(x[0], sourceLang, False) for x in aligned]
targetText = [tokenize(x[1], sourceLang, False) for x in aligned]
sourceTextPlain = '\n'.join([' '.join(x) for x in sourceText])
targetTextPlain = '\n'.join([' '.join(x) for x in targetText])
fileSource = 'qe/deepQuest-config/data_input/test.src'
with open(fileSource, 'w') as f:
f.write(repeatText(sourceTextPlain) + '\n')
fileTarget = 'qe/deepQuest-config/data_input/test.trg'
with open(fileTarget, 'w') as f:
f.write(repeatText(targetTextPlain) + '\n')
tokensTarget = [item for sublist in targetText for item in sublist]
best_epoch = self.pairsEpochs[task_name]
store_path = f'../../deepQuest-config/saved_models/{task_name}'
filename = lambda threshold: f'{store_path}/val_epoch_{best_epoch}_threshold_0.{threshold}_output_0.pred'
with DirCrawler('qe/deepQuest/quest'):
(_output, _error) = bash(f"""
bash ../../deepQuest-config/estimate-wordQEbRNN.sh {task_name} {best_epoch}
""")
#print(_output)
#print(_error)
features = []
for i in range(10):
outputFile = filename(i)
if not os.path.isfile(outputFile):
raise Exception('Server Processing Error')
with open(outputFile, 'r') as outputFile:
features.append(
[1 * (x == 'OK\n') for x in outputFile.readlines()])
# Transpose
features = [[features[j][i] for j in range(len(features))]
for i in range(len(features[0]))]
# Average lines
features = [sum(x) / len(x) for x in features]
# Take only relevant number of tokens
features = features[:len(tokensTarget)]
os.remove('log-keras.txt')
os.remove('log-keras-error.txt')
shutil.rmtree('datasets')
to_remove = \
['val.qe_metrics', f'val_epoch_{best_epoch}_output_0.pred'] + \
[f'val_epoch_{best_epoch}_threshold_0.{str(x)}_output_0.pred' for x in range(10)]
[os.remove(f'{store_path}/' + x) for x in to_remove]
os.remove(fileSource)
os.remove(fileTarget)
return {'status': 'OK', 'qe': features}
