Example #1
0
def design(slides: Slides):
    slide = slides.new_slide()
    content = slide_header(slide, "Design by community")

    list = content.box()
    list_item(list).text("Open source")
    list_item(list).text("RFC")
    list.box(width=700, height=200, p_top=40).image("imgs/rust-rfc.png")

    slide = slides.new_slide()
    content = slide_header(slide, "Backwards compatibility")

    small = s(size=28)

    list = content.box()
    list_item(list).text("Strong BC guarantees")
    list_item(list, show="2+").text("New version every 6 weeks")
    list_item(list, show="2+", level=1).text(
        "Thousands of libraries tested to spot regressions", style=small)
    list_item(list, show="3+").text("Big changes => new edition")
    list_item(list, show="3+", level=1).text("Rust 2015 vs Rust 2018",
                                             style=small)

    slide = slides.new_slide()
    slide.update_style("code", s(size=38))
    content = slide_header(slide, "Unstable features")

    code(content.box(), """
#![feature(async_await)]
async fn foo() {
    ...
}""")

    content.box(height=20)
    bash(content.box(show="2+"), "$ cargo +nightly build")
Example #2
0
 def reset_dir(self, pkgdirname=None, all=False, rmpkg=True):
     if all:
         logger.info('resetting %s', str(REPO_ROOT))
         bash(GIT_RESET_SUBDIR, cwd=REPO_ROOT)
     else:
         if not pkgdirname:
             return False
         cwd = REPO_ROOT / pkgdirname
         if cwd.exists():
             logger.info('resetting %s', str(cwd))
             try:
                 bash(GIT_RESET_SUBDIR, cwd=cwd)
             except Exception:
                 logger.error(f'Unable to reset dir {cwd}')
                 print_exc_plus()
             for fpath in [f for f in cwd.iterdir()]:
                 if fpath.is_dir() and \
                         fpath.name in ('pkg', 'src'):
                     if fpath.name == 'pkg':
                         fpath.chmod(0o0755)
                     rmtree(fpath)
                 elif rmpkg and fpath.is_file() and \
                         (fpath.name.endswith(PKG_SUFFIX) or \
                          fpath.name.endswith(PKG_SIG_SUFFIX)):
                     fpath.unlink()
         else:
             return False
     return True
Example #3
0
    def cargo_line(parent, text, code, show="1+", **text_args):
        wrapper = parent.box(width="fill", horizontal=True, show=show)
        textbox = wrapper.box(width="50%")
        textbox = textbox.text(text, **text_args)

        codebox = wrapper.box(width="40%")
        bash(codebox, code, x=0, width="fill")
        return textbox
 def download(self):
     """Download the datasource if not already downloaded"""
     if not os.path.isfile(self.filepath):
         utils.bash("wget -P "+self.download_dir+" "+self.config["download_url"])
         self.unzip()
         self.process()
     else:
         print self.filepath + " already exists"
 def shp2pgsql(self, datafile, table):
     """Load shapefiles into a postgres database"""
     
     print "Opening {shapefile}".format(shapefile=datafile)
     query = "shp2pgsql -s {srs} -W LATIN1 -g geom {shapefile} {table} > temp.sql".format(srs=self.srs, shapefile=datafile, table=table)
     utils.bash(query)
     print "Sql schema generated"
     self.sql2pgsql(open('temp.sql', 'r').read())
 def raster2pgsql(self, datafile, table):
     print "Opening {raster}".format(raster=datafile)
     query = "raster2pgsql -s {srs} -I -C {raster} {table} > temp.sql".format(srs=self.srs, raster=datafile, table=table)
     utils.bash(query)
     print "Sql schema generated"
     try:
         self.sql2pgsql(open('temp.sql', 'r').read())
     except psycopg2.ProgrammingError:
         print "Rasters not supported in PostGIS 1.5 Please upgrade to 2.0"
Example #7
0
def run_commands(step):
    for data in step.hashes:
        if 'noFail' == data['Expected'].strip():
            step_assert(step).assert_true(
                utils.bash(data['Command']).successful())
        else:
            step_assert(step).assert_equals(
                utils.bash(data['Command']).output_text().strip(),
                data['Expected'].strip())
Example #8
0
 def done(self, fnames, overwrite=False):
     '''
         return None means success
         else returns an error string
     '''
     if [
             f for f in fnames
             if not (f.endswith(PKG_SUFFIX) or f.endswith(PKG_SIG_SUFFIX))
     ]:
         return "files to upload are garbage"
     filter_sig = lambda fnames: [
         fname for fname in fnames if not fname.endswith(PKG_SIG_SUFFIX)
     ]
     if sorted(filter_sig(fnames)) == sorted(filter_sig(self.fnames)):
         try:
             update_path = Path('updates')
             for pkgfname in filter_sig(fnames):
                 pkg_found = False
                 sig_found = False
                 for fpath in update_path.iterdir():
                     if fpath.is_dir():
                         continue
                     if fpath.name == pkgfname:
                         pkg_found = fpath
                     elif fpath.name == f'{pkgfname}.sig':
                         sig_found = fpath
                 if pkg_found and sig_found:
                     try:
                         bash(f'{GPG_VERIFY_CMD} {sig_found} {pkg_found}')
                     except CalledProcessError:
                         ret = f'{pkg_found} GPG verify error'
                         logger.error(ret)
                         print_exc_plus()
                         return ret
                     else:
                         # gpg verify success
                         continue
                 else:
                     return f'file missing: pkg {pkg_found} sig {sig_found}'
                 raise RuntimeError("unexpected error")
             else:
                 # gpg verified for all
                 try:
                     if update(overwrite=overwrite):
                         return None
                     else:
                         raise RuntimeError('update return false')
                 except Exception:
                     print_exc_plus()
                     return f'{pkg_found} update error'
                 raise RuntimeError("unexpected error")
         finally:
             self.__init__()
     else:
         return "Wrong file"
Example #9
0
 def tick(self):
     '''
         check for updates,
         create new jobs
         and run them
     '''
     if not self.__buildjobs:
         # This part check for updates
         if time() - self.last_updatecheck <= UPDATE_INTERVAL * 60:
             if not self.idle:
                 logger.info('Buildbot is idling for package updates.')
                 self.idle = True
             return 60
         else:
             self.last_updatecheck = time()
             self.idle = False
             # git pull repo
             try:
                 bash(GIT_PULL, cwd=REPO_ROOT)
             except Exception:
                 print_exc_plus()
             self.pkgconfigs = load_all_yaml()
             updates = updmgr.check_update()
             for update in updates:
                 (pkgconfig, ver, buildarchs) = update
                 march = True if len(buildarchs) >= 2 else False
                 for arch in buildarchs:
                     newjob = Job(arch, pkgconfig, ver, multiarch=march)
                     self._new_buildjob(newjob)
             return 0
     else:
         # This part does the job
         self.idle = False
         job = self.__get_job()
         if not job:
             logging.error('No job got')
             return
         if job.multiarch:
             self.__clean(job, remove_pkg=True)
             self.__makepkg(job)
             self.__sign(job)
             if self.__upload(job):
                 self.__clean(job, remove_pkg=True)
         else:
             self.__makepkg(job)
             self.__sign(job)
             if self.__upload(job):
                 if job.pkgconfig.cleanbuild:
                     self.__clean(job, remove_pkg=True)
                 else:
                     self.__clean(job, rm_src=False, remove_pkg=True)
         self.__finish_job(job.pkgconfig.dirname)
         return 0
    def install(self):
        dst_dir = os.path.join(self.args.installPrefixDir, 'bin')
        os.makedirs(dst_dir, exist_ok=True)
        retval = 0

        try:
            theApp = AN_APP
            if self.args.target != '':
                theApp = '{}-{}'.format(self.args.target, theApp)

            theApp = os.path.join(dst_dir, theApp)
            output = subprocess.check_output([theApp, '--version'],
                    stderr=subprocess.STDOUT)
            if output is None:
                output = b''
        except BaseException as err:
            output = b''

        if not self.args.forceInstall and bytes(self.args.ver, 'utf-8') in output:
            print('{app} {ver} is already installed'
                    .format(app=self.args.app, ver=self.args.ver))
        else:
            print('compiling {app} {ver}'
                    .format(app=self.args.app, ver=self.args.ver))
            code_dir = self.args.codePrefixDir
            if self.args.target != '':
                code_dir = os.path.join(code_dir, self.args.target)
            code_dir = os.path.join(code_dir, self.args.app)
            if self.args.forceInstall:
                shutil.rmtree(code_dir, ignore_errors=True)
            os.makedirs(code_dir)

            utils.git('clone', [URL, code_dir])
            os.chdir(code_dir)
            version = self.args.ver.replace('.','_')
            utils.git('checkout', ['binutils-{}'.format(version)])
            os.mkdir('build')
            os.chdir('build')

            configureCmd = '../configure --prefix={} --disable-nls'.format(
                    self.args.installPrefixDir)
            if self.args.target != '':
                configureCmd += ' --target={}'.format(self.args.target)
            subprocess.run(configureCmd,
                    shell=True,
                    stdout=subprocess.DEVNULL) # Too much logging overflows 4MB travis-ci log limit
            subprocess.run('make all -j {}'.format(multiprocessing.cpu_count()),
                    shell=True,
                    stdout=subprocess.DEVNULL) # Too much logging overflows 4MB travis-ci log limit
            utils.bash('make install')

        return retval
Example #11
0
    def align(self, sourceLang, targetLang, sourceText, targetText):
        """
        Performs word alignment on sourceText to targetText using fast_align.
        The output is in well known Pharaoh format.
        It's ok to raise Exceptions here. They are handled upstream.
        """
        file1, file2 = self.findRawData(sourceLang, targetLang)
        out = formatParallel(file1, file2)

        sourceText = tokenize(sourceText.replace('\n', ' '), sourceLang)
        targetText = tokenize(targetText.replace('\n', ' '), targetLang)

        out = ["{} ||| {}".format(sourceText, targetText)] + out
        # Found relevant parallel corpus
        print("Files found: {}, {}".format(file1, file2))

        with open('tmp.parallel', 'w') as tmpFile:
            print('\n'.join(out), file=tmpFile)
        (output, _
         ) = bash("align/fast_align/build/fast_align -d -o -v -i tmp.parallel")
        os.remove('tmp.parallel')

        return {
            'status':
            'OK',
            'alignment':
            '\n'.join(output.split('\n')[:(targetText.count('\n') + 1)])
        }
Example #12
0
def false_sharing(slides: SlideDeck, backup: bool):
    if backup:
        slide = new_slide(slides)
        content = slide_header(slide, "Code (backup)")
        code(
            content.box(), """// tid - [0, NO_OF_THREADS)
void thread_fn(int tid, double* data)
{
    size_t repetitions = 1024 * 1024 * 1024UL;
    for (size_t i = 0; i < repetitions; i++)
    {
        data[tid] *= i;
    }
}""")
        slide = new_slide(slides)
        content = slide_header(slide, "Result (backup)")
        content.box(height=600).image("images/example3-time.png")

    slide = new_slide(slides)
    content = slide_header(slide, "Cache system")
    content.box(height=600).image("images/haswell-diagram.png")
    content.box(width=230, height=40, x=816, y=530).rect(color=COLOR_BACKEND,
                                                         stroke_width=3)
    content.box(width=226, height=40, x=240, y=570).rect(color=COLOR_BACKEND,
                                                         stroke_width=3)

    slide = new_slide(slides)
    content = slide_header(slide, "Cache coherency")
    content.box(height=600).image("images/false-sharing.svg")

    slide = new_slide(slides)
    content = slide_header(slide, "False sharing")
    content.box(width="90%").image("images/false-sharing-array.svg")

    slide = new_slide(slides)
    content = slide_header(slide, "How to measure?")
    content.box().text("~tt{l2_rqsts.all_rfo}", s(size=48))
    content.box(p_top=20).text(
        "How many times some core invalidated data in other cores?")

    if backup:
        bash(content.box(p_top=40, show="next+"),
             """$ perf stat -e l2_rqsts.all_rfo ./example3
1 thread   ->        59 711
2 threads  -> 1 112 258 710""",
             text_style=s(align="left"))
Example #13
0
def hunalign(sourceText, targetText):
    """
    Performs sentence alignment on sourceText to targetText using Hunalign.
    It's ok to raise Exceptions here. They are handled upstream.
    """
    sourceText = re.sub('\n', ' ', sourceText)
    sourceText = re.sub('\.', '.\n', sourceText)
    sourceText = re.sub('\?', '.\n', sourceText)
    targetText = re.sub('\n', ' ', targetText)
    targetText = re.sub('\.', '.\n', targetText)
    targetText = re.sub('\?', '?\n', targetText)

    sourceText = sourceText.split('\n')
    targetText = targetText.split('\n')

    sourceText = list(
        filter(lambda x: not re.match('.?.?(\.|\?)', x), sourceText))
    targetText = list(
        filter(lambda x: not re.match('.?.?(\.|\?)', x), targetText))

    with DirCrawler('align/hunalign'):
        with open('tmp.src', 'w') as f:
            f.write('\n'.join(sourceText) + '\n')
        with open('tmp.trg', 'w') as f:
            f.write('\n'.join(targetText) + '\n')
        (output, _error) = bash(
            "perl hunalignwrapper.pl --hunalign=./hunalign/src/hunalign/hunalign tmp.src tmp.trg"
        )
        os.remove('tmp.src')
        os.remove('tmp.trg')

    output = list(filter(lambda x: x.strip() != '', output.split('\n')))
    align = [[int(y) for y in x.split('\t')[0].split('-')] for x in output]

    # Process the ladder output of Hunalign
    output = []
    for l, r in align:
        left = []
        right = []
        while True:
            if l == 0 and r == 0:
                output.append([' '.join(left), ' '.join(right)])
                break
            elif l > r:
                left.append(sourceText.pop(0))
                l -= 1
                continue
            elif l < r:
                right.append(targetText.pop(0))
                r -= 1
                continue
            else:
                left.append(sourceText.pop(0))
                right.append(targetText.pop(0))
                l -= 1
                r -= 1
    return output
Example #14
0
def repo_add(fpaths):
    assert type(fpaths) is list
    assert not [None for fpath in fpaths if fpath.parent != fpaths[0].parent]
    for fpath in fpaths:
        assert issubclass(type(fpath), os.PathLike) and \
               fpath.name.endswith(PKG_SUFFIX)
    dbpath = fpaths[0].parent / f'{REPO_NAME}.db.tar.gz'
    return bash(
        f'{REPO_CMD} {dbpath} {" ".join([str(fpath) for fpath in fpaths])}',
        RUN_CMD_TIMEOUT=5 * 60)
Example #15
0
    def _execute(self, script, stdin, language):
        "Generate stdout by executing the script with given stdin."
        if language.lower() != 'python':
            raise Exception('Language not supported: {}'.format(language))

        if language == 'python':
            fd, filename = tempfile.mktemp(suffix=".py")
            os.write(fd, script)
            stdout = bash("python " + filename, stdin=stdin)
            os.close(fd)
            return stdout

        # TODO: Add support for more languages
        return ''
Example #16
0
    def qe(self, sourceLang, targetLang, sourceText, targetText):
        """
        Performs translation quality estimation on sourceText to targetText using OpenKiwi
        It's ok to raise Exceptions here. They are handled upstream.
        """

        if not [sourceLang, targetLang] in [['cs', 'de'], ['en', 'de']]:
            raise Exception(
                f'{sourceLang}-{targetLang} language pair not supported')

        # Sanitize input
        aligned = hunalign(sourceText, targetText)
        sourceText = [tokenize(x[0], sourceLang, False) for x in aligned]
        targetText = [tokenize(x[1], sourceLang, False) for x in aligned]
        sourceTextPlain = '\n'.join([' '.join(x) for x in sourceText])
        targetTextPlain = '\n'.join([' '.join(x) for x in targetText])

        with DirCrawler('qe/openkiwi-config'):
            fileSource = 'data/input.src'
            with open(fileSource, 'w') as fileSourceW:
                fileSourceW.write(sourceTextPlain)

            fileTarget = 'data/input.trg'
            with open(fileTarget, 'w') as fileTargetW:
                fileTargetW.write(targetTextPlain)

            (_output, _error) = bash(f"""
                kiwi predict --config experiments/predict_estimator_{sourceLang}_{targetLang}.yaml
                 """)
            #print(_output)
            #print(_error)

            fileOut = 'data/tags'
            with open(fileOut, 'r') as f:
                out = [
                    1 - float(x.rstrip('\n'))
                    for x in ' '.join(f.readlines()).split(' ')
                ]

        # map the estimation from an interval to discrete values:
        # >= 0.5 -> 1.0
        # >= 0.4 -> 0.7
        # >= 0.1 -> 0.3
        # <= 0.1 -> 0.1
        out = map(
            lambda x: 1 if x >= 0.5 else 0.7 if x >= 0.4 else 0.3
            if x >= 1 else 0.1, out)
        out = list(out)
        return {'status': 'OK', 'qe': out}
Example #17
0
def disconnect(dc,vm_name):
    logger=my_logger()
    if dc == 'two':
        vcent_user,vcent_pass,vcent_ip = settings.TVCENTER_USER, settings.TVCENTER_PASSWORD, settings.TVCENTER_IP
    elif dc == 'san':
        vcent_user,vcent_pass,vcent_ip = settings.SVCENTER_USER, settings.SVCENTER_PASSWORD, settings.SVCENTER_IP
    else:
        vcent_user,vcent_pass,vcent_ip = settings.FVCENTER_USER, settings.FVCENTER_PASSWORD, settings.FVCENTER_IP
    dis='govc device.disconnect -u "%s":"%s"@"%s" -k -dc="%s" -vm "%s" cdrom-3000' \
    %(vcent_user,vcent_pass,vcent_ip,dc,vm_name)
    stdout, stderr = bash(dis)
    if stderr:
        logger.error('机房: %s VM_Name: %s cdrom断开连接失败 %s' % (dc, vm_name, stderr.decode('utf-8')))
    else:
        logger.info('机房: %s VM_Name: %s cdrom已断开连接' % (dc, vm_name))
Example #18
0
def repo_remove(fpaths):
    assert type(fpaths) is list
    assert not [None for fpath in fpaths if fpath.parent != fpaths[0].parent]
    for fpath in fpaths:
        assert issubclass(type(fpath), os.PathLike) and \
               fpath.name.endswith(PKG_SUFFIX)
    dbpath = fpaths[0].parent / f'{REPO_NAME}.db.tar.gz'
    for fpath in fpaths:
        throw_away(fpath)
        sigpath = fpath.parent / f'{fpath.name}.sig'
        # there is a fscking problem that fscking pathlib always follow symlinks
        if sigpath.exists() or sigpath.is_symlink():
            throw_away(sigpath)
    pkgnames = [
        get_pkg_details_from_name(fpath.name).pkgname for fpath in fpaths
    ]
    return bash(f'{REPO_REMOVE_CMD} {dbpath} {" ".join(pkgnames)}',
                RUN_CMD_TIMEOUT=5 * 60)
 def unzip(self):
     """Unpack the source"""
     utils.bash("unzip {} -d {}".fromat(self.filepath, self.dir))
            output = b''
    except BaseException as err:
        output = b''

    if bytes(args.o.ver, 'utf-8') in output:
        print('{app} {ver} is already installed'.format(app=args.o.src, ver=VER))
        exit(0)
    else:
        print('compiling {app} {ver}'.format(app=args.o.src, ver=VER))
        os.makedirs(args.o.src, exist_ok=True)

        try:
            url = 'git://sourceware.org/git/binutils-gdb.git'
            utils.git('clone', [url, args.o.src])
            os.chdir(args.o.src)
            version = VER.replace('.','_')
            utils.git('checkout', ['binutils-{ver}'.format(ver=version)])
            os.mkdir('build')
            os.chdir('build')
        except BaseException as ex:
            traceback.print_exc()
            exit(1)

        print('configure')
        utils.bash('../configure --prefix={0} --target={1} --disable-nls'
                .format(args.o.prefix, TARGET))
        utils.bash('make all -j {}'.format(multiprocessing.cpu_count()))
        utils.bash('make install')

        exit(0)
Example #21
0
 def __sign(self, job):
     logger.info('signing in %s %s', job.pkgconfig.dirname, job.arch)
     cwd = REPO_ROOT / job.pkgconfig.dirname
     for fpath in cwd.iterdir():
         if fpath.name.endswith(PKG_SUFFIX):
             bash(f'{GPG_SIGN_CMD} {fpath.name}', cwd=cwd)
Example #22
0
    def qe(self, sourceLang, targetLang, sourceText, targetText):
        """
        Performs translation quality estimation on sourceText to targetText using QuEst++ and fast_align
        It's ok to raise Exceptions here. They are handled upstream.
        """
        os.makedirs('data/tmp', exist_ok=True)

        if not [sourceLang, targetLang] in self.supportedPairs:
            raise Exception("{}-{} language pair not supported".format(
                sourceLang, targetLang))

        # Sanitize input
        aligned = hunalign(sourceText, targetText)
        sourceText = [tokenize(x[0], sourceLang, False) for x in aligned]
        targetText = [tokenize(x[1], sourceLang, False) for x in aligned]
        sourceTextPlain = '\n'.join([' '.join(x) for x in sourceText])
        targetTextPlain = '\n'.join([' '.join(x) for x in targetText])

        alignments = fast_align.FastAlign().align(sourceLang, targetLang,
                                                  sourceTextPlain,
                                                  targetTextPlain)['alignment']
        with open('data/tmp/alignments', 'w') as fileAlignments:
            fileAlignments.write(alignments)

        with open('data/tmp/source', 'w') as fileSource:
            fileSource.write(sourceTextPlain)

        with open('data/tmp/target', 'w') as fileTarget:
            fileTarget.write(targetTextPlain)

        with DirCrawler('qe/questplusplus'):
            print("Extracting features")
            (_output, _error) = bash("""
                 java -cp QuEst++.jar:lib/* shef.mt.WordLevelFeatureExtractor
                 -lang english spanish
                 -input ../../data/tmp/source ../../data/tmp/target
                 -alignments ../../data/tmp/alignments
                 -config ../questplusplus-config/config.word-level.properties
                 """)

            outputFile = 'output/test/output.txt'
            if not os.path.isfile(outputFile):
                raise Exception('Server Processing Error')
            with open(outputFile, 'r') as outputFileR:
                features = outputFileR.readlines()

        os.remove('data/tmp/alignments')
        os.remove('data/tmp/source')
        os.remove('data/tmp/target')

        features = [[
            x.split('=')[1] for x in line.rstrip('\n').rstrip('\t').split('\t')
        ] for line in features]
        with open('data/tmp/features', 'w') as fileFeatures:
            fileFeatures.write('\n'.join(['\t'.join(x) for x in features]))
        with open('data/tmp/labels', 'w') as fileLabels:
            fileLabels.write('\n'.join(['1'] * len(features)))

        with DirCrawler('qe/questplusplus'):
            print("Removing output directory structure for feature extractor")
            os.remove(outputFile)
            os.rmdir('output/test')
            os.rmdir('output')

            print("Machine Learning")
            (_output, _error) = bash(f"""
                python learning/src/learn_model.py ../questplusplus-config/svr_{sourceLang}_{targetLang}.cfg
                """)

            with open('predicted.csv', 'r') as predictedFile:
                output = [
                    float(x.rstrip('\n').split('\t')[1])
                    for x in predictedFile.readlines()
                ]
            os.remove('predicted.csv')

        os.remove('data/tmp/features')
        os.remove('data/tmp/labels')
        os.rmdir('data/tmp')
        return {'status': 'OK', 'qe': output}
Example #23
0
def cache_conflicts(slides: SlideDeck, backup: bool):
    if backup:
        slide = new_slide(slides)
        content = slide_header(slide, "Code (backup)")
        code(
            content.box(),
            """// Addresses of N integers, each `offset` bytes apart
std::vector<int*> data = ...;
for (auto ptr: data)
{
    *ptr += 1;
}
// Offsets: 4, 64, 4000, 4096, 4128""")
        slide = new_slide(slides)
        content = slide_header(slide, "Result (backup)")
        content.box(height=600).image("images/example1-time.png")

    slide = new_slide(slides)
    content = slide_header(slide, "Cache memory")
    content.box(height=600).image("images/haswell-diagram.png")
    content.box(width=230, height=40, x=816, y=530).rect(color=COLOR_BACKEND,
                                                         stroke_width=3)

    slide = new_slide(slides)
    content = slide_header(slide, "How are (L1) caches implemented")
    list_wrapper = content.box()
    list_item(list_wrapper).text("N-way set associative table")
    list_item(list_wrapper, level=1, show="last+").text("Hardware hash table")
    list_item(list_wrapper, show="next+").text("Key = address (8B)")
    list_item(list_wrapper, show="next+").text("Entry = cache line (64B)")

    slide = new_slide(slides)
    content = slide_header(slide, "N-way set associative cache")
    hash_size = 8
    hash_dimension = 60

    def table(wrapper: Box,
              size,
              dimension,
              buckets=None,
              bucket_indices=True):
        htable = wrapper.box(horizontal=True)
        items = []
        for i in range(size):
            cell = htable.box(width=dimension,
                              height=dimension,
                              horizontal=True).rect("black", stroke_width=2)
            items.append(cell)

        if buckets:
            bucket_width = int((size / buckets) * dimension)
            for i in range(buckets):
                pos = i * bucket_width
                htable.box(x=pos, y=0, width=bucket_width,
                           height=dimension).rect("black", stroke_width=6)
                if bucket_indices:
                    htable.box(x=pos, y=dimension - 5,
                               width=bucket_width).text(str(i))

        return (htable, items)

    content.box().text("Size = {} cache lines".format(hash_size),
                       style="notice")
    (htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension)
    arrow_wrapper = content.box()
    arrow = Arrow(20)
    arrow_wrapper.box().line([
        hitems[0].p("50%", 0).add(0, -20),
        hitems[-1].p("50%", 0).add(0, -20),
    ],
                             start_arrow=arrow,
                             end_arrow=arrow,
                             stroke_width=5,
                             color=COLOR_NOTE)

    content.box(
        p_top=20,
        show="next+").text("Associativity (N) - # of cache lines per bucket")
    content.box(p_top=10, show="next+").text("# of buckets = Size / N")

    row = content.box(horizontal=True, p_top=20)
    lcol = row.box(y=0)
    rcol = row.box(y=0, p_left=20)

    def htable_row(text, block_count):
        padding = 20
        height = 110
        lcol.box(show="next+", p_top=padding, height=height).text(text)
        return table(rcol.box(show="last+", p_top=padding, height=height),
                     hash_size, hash_dimension, block_count)

    htable_row("N = 1 (direct mapped)", hash_size)
    htable_row("N = {} (fully associative)".format(hash_size), 1)
    htable_row("N = 2", hash_size // 2)

    slide = new_slide(slides)
    content = slide_header(slide, "How are addresses hashed?")

    content.box().text("64-bit address:")
    row = content.box(horizontal=True, width=800)
    widths = ["60%", "25%", "15%"]
    address_colors = ["#B22222", "#007944", "#0018AE"]
    labels = ["Tag", "Index", "Offset"]

    for i in range(3):
        wrapper = row.box(width=widths[i]).rect(color=address_colors[i],
                                                stroke_width=4)
        wrapper.box(padding=4).text(labels[i])
    labelrow = content.box(horizontal=True, x=220)
    labelrow.box().text("63")
    labelrow.box(p_left=770).text("0")

    list_wrapper = content.box(p_top=20)
    list_item(list_wrapper, show="next+").text("Offset", "bold")
    list_item(list_wrapper, level=1,
              show="last+").text("Selects byte within a cache line")
    list_item(list_wrapper, level=1,
              show="last+").text("log2(cache line size) bits")
    list_item(list_wrapper, show="next+").text("Index", "bold")
    list_item(list_wrapper, level=1,
              show="last+").text("Selects bucket within the cache")
    list_item(list_wrapper, level=1,
              show="last+").text("log2(bucket count) bits")
    list_item(list_wrapper, show="next+").text("Tag", "bold")
    list_item(list_wrapper, level=1, show="last+").text("Used for matching")

    slide = new_slide(slides)
    content = slide_header(slide, "N-way set associative cache")

    queue = content.box(x="55%", y=40, horizontal=True)
    queue.box(p_right=40).text("Cache lines:")
    colors = ("#F0134D", "#FF6F5E", "#F0134D")
    cacheline_labels = ("A", "B", "C")
    for i in range(3):
        queue.box(width=hash_dimension,
                  height=hash_dimension).rect(color="black",
                                              bg_color=colors[i],
                                              stroke_width=5).text(
                                                  cacheline_labels[i],
                                                  style=s(bold=True,
                                                          color="white"))

    index = content.box(x="55%", y=90, horizontal=True)
    index.box(p_right=75).text("Index bits:")
    index_bits = (0, 1, 0)
    for i in range(3):
        index.box(width=hash_dimension,
                  height=hash_dimension).text(str(index_bits[i]))

    def insert(slot, show, item):
        wrapper = slot.overlay(show=show)
        wrapper.rect(bg_color=colors[item])
        wrapper.text(cacheline_labels[item], style=s(bold=True, color="white"))

    row = content.box(horizontal=True, p_top=20)
    lcol = row.box(y=0)
    rcol = row.box(y=0, p_left=20)

    def htable_row(text, block_count):
        padding = 20
        height = 140
        lcol.box(show="next+", p_top=padding, height=height).text(text)
        return table(rcol.box(show="last+", p_top=padding, height=height),
                     hash_size, hash_dimension, block_count)

    (_, hitems) = htable_row("N = 1", hash_size)
    insert(hitems[0], "next+", 0)
    insert(hitems[1], "next+", 1)
    insert(hitems[0], "next+", 2)

    (_, hitems) = htable_row("N = {}".format(hash_size), 1)
    insert(hitems[0], "next+", 0)
    insert(hitems[1], "next+", 1)
    insert(hitems[2], "next+", 2)

    (_, hitems) = htable_row("N = 2", hash_size // 2)
    insert(hitems[0], "next+", 0)
    insert(hitems[2], "next+", 1)
    insert(hitems[1], "next+", 2)

    slide = new_slide(slides)
    slide.update_style("default", s(size=46))
    slide.update_style("bold", s(size=46))
    content = slide_header(slide, "Intel L1 cache")
    bash(content.box(),
         """$ getconf -a | grep LEVEL1_DCACHE
LEVEL1_DCACHE_SIZE      32768
LEVEL1_DCACHE_ASSOC     8
LEVEL1_DCACHE_LINESIZE  64""",
         text_style=s(align="left"))
    list_wrapper = content.box(p_top=20)
    list_item(
        list_wrapper,
        show="next+").text("~bold{Cache line size} - 64 B (6 offset bits)")
    list_item(list_wrapper, show="next+").text("~bold{Associativity} (N) - 8")
    list_item(list_wrapper, show="next+").text("~bold{Size} - 32768 B")
    list_item(list_wrapper, show="next+").text("32768 / 64 => 512 cache lines")
    list_item(list_wrapper,
              show="next+").text("512 / 8 => 64 buckets (6 index bits)")

    slides.set_style("tag", s(color=address_colors[0]))
    tag = slides.get_style("tag")
    slides.set_style("index", tag.compose(s(color=address_colors[1])))
    slides.set_style("offset", tag.compose(s(color=address_colors[2])))

    styles = ["tag", "index", "offset"]
    colors = ["#F0134D", "#FF6F5E", "#1F6650", "#40BFC1"]

    def address(cols, content, next=True, use_style=True, row=0):
        for i, col in enumerate(cols):
            show = "1+"
            if next:
                show = "next+" if i == 0 else "last+"

            style = "default"
            if use_style:
                if i == 0:
                    style = s(color=colors[row])
                else:
                    style = styles[i - 1]
            col.box(show=show).text(content[i], style=style)

    slide = new_slide(slides)
    content = slide_header(slide, "Offset = 4B")

    width = 700
    columns = 4
    row = content.box(horizontal=True)
    cols = [row.box(width=width // columns) for _ in range(columns)]

    address(cols, ("Number", "Tag", "Index", "Offset"),
            next=False,
            use_style=False)
    address(cols, ("A", "..100000", "000000", "000000"), next=False)
    address(cols, ("B", "..100000", "000000", "000100"), row=1)
    address(cols, ("C", "..100000", "000000", "001000"), row=2)
    address(cols, ("D", "..100000", "000000", "001100"), row=3)

    hash_dimension = 80
    (htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension,
                             hash_size // 2)
    for i in range(4):
        hitems[0].box(show="{}+".format(i + 1),
                      width=hash_dimension // 4,
                      height=hash_dimension).rect(bg_color=colors[i])

    list_wrapper = content.box(p_top=40)
    list_item(
        list_wrapper,
        show="next+").text("Same bucket, same cache line for each number")
    list_item(list_wrapper,
              show="next+").text("Most efficient, no space is wasted")

    slide = new_slide(slides)
    content = slide_header(slide, "Offset = 64B")

    row = content.box(horizontal=True)
    cols = [row.box(width=width // columns) for _ in range(columns)]

    address(cols, ("Number", "Tag", "Index", "Offset"),
            next=False,
            use_style=False)
    address(cols, ("A", "..100000", "000000", "000000"), next=False)
    address(cols, ("B", "..100000", "000001", "000000"), row=1)
    address(cols, ("C", "..100000", "000010", "000000"), row=2)
    address(cols, ("D", "..100000", "000011", "000000"), row=3)

    (htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension,
                             hash_size // 2)
    for i in range(4):
        hitems[i * 2].box(show="{}+".format(i + 1),
                          width=hash_dimension // 4,
                          height=hash_dimension,
                          x=0).rect(bg_color=colors[i])

    list_wrapper = content.box(p_top=40)
    list_item(list_wrapper,
              show="next+").text("Different bucket for each number")
    list_item(list_wrapper, show="next+").text("Wastes 60B in each cache line")
    list_item(list_wrapper,
              show="next+").text("Equally distributed among buckets")

    slide = new_slide(slides)
    content = slide_header(slide, "Offset = 4096B")

    row = content.box(horizontal=True)
    cols = [row.box(width=width // columns) for _ in range(columns)]

    address(cols, ("Number", "Tag", "Index", "Offset"),
            next=False,
            use_style=False)
    address(cols, ("A", "..100000", "000000", "000000"), next=False)
    address(cols, ("B", "..100001", "000000", "000000"), row=1)
    address(cols, ("C", "..100010", "000000", "000000"), row=2)
    address(cols, ("D", "..100011", "000000", "000000"), row=3)

    (htable, hitems) = table(content.box(p_top=20), hash_size, hash_dimension,
                             hash_size // 2)
    for i in range(4):
        hitems[i % 2].box(show="{}+".format(i + 1),
                          width=hash_dimension // 4,
                          height=hash_dimension,
                          x=0).rect(bg_color=colors[i])

    list_wrapper = content.box(p_top=40)
    list_item(list_wrapper, show="next+").text(
        "Same bucket, but different cache lines for each number!")
    list_item(list_wrapper,
              show="next+").text("Bucket full => evictions necessary")

    slide = new_slide(slides)
    content = slide_header(slide, "How to measure?")
    content.box().text("~tt{l1d.replacement}", style=s(size=48))
    content.box(
        p_top=20).text("How many times was a cache line loaded into L1?")

    if backup:
        bash(content.box(p_top=40, show="next+"),
             """$ perf stat -e l1d.replacement ./example1
4B    offset ->     149 558
4096B offset -> 426 218 383""",
             text_style=s(align="left"))
Example #24
0
def run_commands(step):
    for data in step.hashes:
        if 'noFail' == data['Expected'].strip():
            step_assert(step).assert_true(utils.bash(data['Command']).successful())
        else:
            step_assert(step).assert_equals(utils.bash(data['Command']).output_text().strip(),data['Expected'].strip())
Example #25
0
def project(slides: Slides):
    slide = slides.new_slide()
    content = slide_header(slide, "Project management (Cargo)")
    content.box(height=400).image("imgs/cargo.png")

    slide = slides.new_slide()
    slide.update_style("code", s(size=30))

    content = slide_header(slide, "Using libraries")

    line = content.box(width="fill", horizontal=True)

    cargo = line.box(width="50%", p_right=50)
    cargo.box().text("Cargo.toml")
    cargo_code = code(
        cargo.box(), """
[package]
name = "hello_world"
version = "0.1.0"

[dependencies]
ibverbs = "0.4"
json = "1.0"
protobuf = "2.0"
""", "toml")

    main = line.box(width="50%", show="2+", y=0)
    main.box().text("main.rs")
    main_code = code(
        main.box(), """
use json::parse;

fn main() {
    parse("data.json");
}""")

    arrow = Arrow(20)
    p1 = cargo_code.line_box(5).p("100%", "50%")
    p2 = main_code.line_box(0).p(0, "50%")
    slide.box(show="2+").line(
        [p1.add(-40, 0), p1, p2.add(-10, 0)],
        stroke_width=5,
        color="orange",
        end_arrow=arrow)

    content.box(height=10)
    content.box(show="3+").text("More than 26k libraries available")

    slide = slides.new_slide()
    content = slide_header(slide, "Unified documentation")
    content.box(width=900).image("imgs/rust-docs.png")

    slide = slides.new_slide()
    content = slide_header(slide, "Integrated tooling")

    def cargo_line(parent, text, code, show="1+", **text_args):
        wrapper = parent.box(width="fill", horizontal=True, show=show)
        textbox = wrapper.box(width="50%")
        textbox = textbox.text(text, **text_args)

        codebox = wrapper.box(width="40%")
        bash(codebox, code, x=0, width="fill")
        return textbox

    wrapper = content.box(width="fill")
    cargo_line(wrapper, "Build", "$ cargo build", "1+")
    cargo_line(wrapper, "Run", "$ cargo run", "2+")

    slide = slides.new_slide()
    slide.update_style("code", s(size=40))
    content = slide_header(slide, "Integrated tooling (tests)")

    code(content.box(), """
#[test]
fn test_add() {
    assert_eq!(add(1, 2), 3);
}
""")

    content.box(height=20)
    bash(content.box(show="2+"), "$ cargo test")

    slide = slides.new_slide()
    slide.update_style("code", s(size=40))
    content = slide_header(slide, "Integrated tooling (benchmarks)")

    code(
        content.box(), """
#[bench]
fn bench_add_two(b: &mut Bencher) {
    b.iter(|| add_two(2));
}
""")

    content.box(height=20)
    bash(content.box(show="2+"), "$ cargo bench")

    slide = slides.new_slide()
    content = slide_header(slide, "Integrated tooling")
    wrapper = content.box(width="fill")
    cargo_line(wrapper, "Format", "$ cargo fmt")
    cargo_line(wrapper, "Lint", "$ cargo clippy", "next+")
    box = cargo_line(wrapper, "Publish to SC", "$ cargo publish", "next+")
    box = box.line_box(0)
    box.line([box.p("53%", "55%"), box.p("73%", "55%")], stroke_width=3)

    slide = slides.new_slide()
    slide.update_style("code", s(size=24))
    content = slide_header(slide, "Build scripts")
    content.box().text("build.rs")

    code_width = 940
    code_step(content.box(width=code_width, height=400),
              """
fn main() {
    // generate Protobuf objects
    protoc_rust::run("protobuf/message.proto", "src/protos");

    // generate C headers
    cbindgen::Builder::new()
      .generate()
      .write_to_file("bindings.h");
}
""",
              "1", ((0, 1, 2, 3, None, None, None, None, 8),
                    (0, 1, 2, 3, 4, 5, 6, 7, 8)),
              width=code_width)

    slide = slides.new_slide()
    content = slide_header(slide, "(interlude)")
    content.box(height=600).image("imgs/meme-cargo.jpg")
Example #26
0
#!/usr/bin/env python3
import os, sys
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASE_DIR)
from utils import bash

if __name__ == '__main__':
    live = 'ps -ef |grep "create_vm_start.py"|grep -v grep|wc -l'
    stdout, stderr = bash(live)
    if int(stdout.decode('utf-8')) == 0:
        start = '/usr/bin/python3 /export/VMWare_Auto/vm_create/bin/create_vm_start.py &>/dev/null'
        bash(start)
Example #27
0
                  readpkglog as extra_readpkglog, \
                  readmainlog as extra_readmainlog

abspath = os.path.abspath(__file__)
abspath = os.path.dirname(abspath)
os.chdir(abspath)

logger = logging.getLogger('buildbot')
configure_logger(logger,
                 logfile=MAIN_LOGFILE,
                 rotate_size=1024 * 1024 * 10,
                 enable_notify=True,
                 consolelog=CONSOLE_LOGFILE)

# refuse to run in systemd-nspawn
if 'systemd-nspawn' in bash('systemd-detect-virt || true'):
    logger.error('Refused to run in systemd-nspawn.')
    raise AssertionError('Refused to run in systemd-nspawn.')

REPO_ROOT = Path(PKGBUILD_DIR)


class Job:
    def __init__(self, buildarch, pkgconfig, version, multiarch=False):
        assert buildarch in BUILD_ARCHS
        self.arch = buildarch
        self.pkgconfig = pkgconfig
        self.version = version
        self.multiarch = multiarch
        self.added = time()
Example #28
0
def branch_prediction(slides: SlideDeck, backup: bool):
    if backup:
        slide = new_slide(slides)
        content = slide_header(slide, "Code (backup)")
        code(
            content.box(),
            """std::vector<float> data = /* 32K random floats in [1, 10] */;
float sum = 0;
// std::sort(data.begin(), data.end());
for (auto x : data)
{
    if (x < 6.0f)
    {
        sum += x;
    }
}""")
        slide = new_slide(slides)
        content = slide_header(slide, "Result (backup)")
        content.box(width="90%").image("images/example0a-time.png")

    slide = new_slide(slides)
    content = slide_header(slide, "Most upvoted Stack Overflow question")
    content.box(height=600).image("images/stack-overflow.png")

    slide = new_slide(slides)
    content = slide_header(slide,
                           "What is going on? (Intel Amplifier - VTune)")
    content.box(height=600).image("images/vtune.png")

    slide = new_slide(slides)
    content = slide_header(slide, "What is going on? (perf)")
    bash(content.box(),
         """$ perf stat ./example0a --benchmark_filter=nosort
    853,672012  task-clock (msec) #   0,997 CPUs utilized
            30  context-switches  #   0,035 K/sec          
             0  cpu-migrations    #   0,000 K/sec
           199  page-faults       #   0,233 K/sec
 3 159 530 915  cycles            #   3,701 GHz
 1 475 799 619  instructions      #   0,47  insn per cycle
   419 608 357  branches          # 491,533 M/sec
   102 425 035  branch-misses     #  24,41% of all branches""",
         text_style=s(align="left", size=34))

    slide = new_slide(slides)
    content = slide_header(slide, "Branch predictor")
    content.box(height=600).image("images/haswell-diagram.png")
    content.box(width=130, height=40, x=595, y=140).rect(color=COLOR_FRONTEND,
                                                         stroke_width=3)

    slide = new_slide(slides)
    content = slide_header(slide, "CPU pipeline 101")
    content.box(height=400).image("images/branch-miss-pipeline.svg")

    slide = new_slide(slides)
    content = slide_header(slide, "Branch predictor")
    list_wrapper = content.box()
    list_item(list_wrapper).text("CPU tries to predict results of branches")
    list_item(list_wrapper,
              show="next+").text("Misprediction can cost ~15-20 cycles!",
                                 escape_char="#")

    slide = new_slide(slides)
    content = slide_header(slide, "Simple branch predictor - unsorted array")
    code(content.box(p_bottom=40), """if (data[i] < 6) {
    ...
}""")

    row = content.box(horizontal=True)
    box_dimension = 100

    def array(numbers, predictions, start=1, needle=6):
        stroke_width = 2
        size = 36
        for i in range(len(numbers)):
            box = row.box(width=box_dimension,
                          height=box_dimension).rect(color="black",
                                                     stroke_width=stroke_width)
            number = str(numbers[i])
            box.text(number, s(bold=True, size=size))

            predicted_correctly = (predictions[i] and numbers[i] < needle) or (
                not predictions[i] and numbers[i] >= needle)
            prediction = "green" if predicted_correctly else "red"
            show_overlay = "{}+".format(start + i * 2 + 1)
            overlay = box.overlay(show=show_overlay).rect(
                color="black", bg_color=prediction, stroke_width=stroke_width)
            overlay.text(number, s(color="white", bold=True, size=size))
            show_text = start + i * 2
            row.box(x=i * box_dimension,
                    y=box_dimension,
                    width=box_dimension,
                    show="{}-{}".format(show_text,
                                        show_text + 1)).text("{} < {}?".format(
                                            number, needle))

    values = [6, 2, 1, 7, 4, 8, 3, 9]
    text_style = s(align="left", size=42)
    width = 400

    def predict_sequence(wrapper: Box, values, start=1):
        for i in range(len(values)):
            value = "Taken" if values[i] else "Not taken"
            show_start = start + i * 2
            wrapper.overlay(
                show="{}-{}".format(show_start, show_start + 1)).rect(
                    bg_color="white").text("Prediction: {}".format(value),
                                           style=text_style)

    def predict_value(index):
        if index == 0:
            return False
        return values[index - 1] < 6

    predictions = [predict_value(i) for i in range(len(values))]

    array(values, predictions, start=2)
    prediction_wrapper = content.box(p_top=60,
                                     width=width).text("Prediction: Not taken",
                                                       style=text_style)
    predict_sequence(prediction_wrapper, predictions, start=1)

    content.box(show="next+", p_top=40).text("2 hits, 6 misses (25% hit rate)")

    slide = new_slide(slides)
    content = slide_header(slide, "Simple branch predictor - sorted array")
    code(content.box(p_bottom=40), """if (data[i] < 6) {
    ...
}""")

    row = content.box(horizontal=True)
    values = [1, 2, 3, 4, 6, 7, 8, 9]
    predictions = [predict_value(i) for i in range(len(values))]

    array(values, predictions, start=2)
    prediction_wrapper = content.box(p_top=60,
                                     width=width).text("Prediction: Not taken",
                                                       style=text_style)
    predict_sequence(prediction_wrapper, predictions, start=1)

    content.box(show="next+", p_top=40).text("6 hits, 2 misses (75% hit rate)")

    if backup:
        size = 40
        slide = new_slide(slides)
        content = slide_header(slide, "How can the compiler help?")
        row = content.box(horizontal=True)
        row.box(y=0, p_right=50, width=400).image("images/bm-float-code.png")
        row.box(y=0, width=600).image("images/bm-float-bin.png")
        content.box(p_top=20).text(
            "With ~tt{float}, there are two branches per iteration",
            style=s(size=size))

        slide = new_slide(slides)
        content = slide_header(slide, "How can the compiler help?")
        row = content.box(horizontal=True)
        row.box(y=0, p_right=50, width=400).image("images/bm-int-code.png")
        row.box(y=0, width=600).image("images/bm-int-bin.png")
        content.box(p_top=20).text(
            "With ~tt{int}, one branch is removed (using ~tt{cmov})",
            style=s(size=size))

    slide = new_slide(slides)
    content = slide_header(slide, "How to measure?")
    content.box().text("~tt{branch-misses}", style=s(size=48))
    content.box(p_top=20).text("How many times was a branch mispredicted?")

    if backup:
        bash(content.box(p_top=40, show="next+"),
             """$ perf stat -e branch-misses ./example0a
with    sort ->     383 902
without sort -> 101 652 009""",
             text_style=s(align="left"))

    slide = new_slide(slides)
    slide.update_style("code", s(size=40))
    content = slide_header(slide, "How to help the branch predictor?")
    list_wrapper = content.box()
    list_item(list_wrapper).text("More predictable data")
    list_item(list_wrapper, show="next+").text("Profile-guided optimization")
    list_item(list_wrapper,
              show="next+").text("Remove (unpredictable) branches")
    list_item(list_wrapper,
              show="next+").text("Compiler hints (use with caution)")
    code(
        list_wrapper.box(show="last+", p_top=20, p_bottom=20),
        """if (__builtin_expect(will_it_blend(), 0)) {
    // this branch is not likely to be taken
}""")

    slide = new_slide(slides)
    content = slide_header(slide, "Branch target prediction")
    list_wrapper = content.box()
    list_item(list_wrapper).text(
        "Target of a jump is not known at compile time:")
    list_item(list_wrapper, show="next+", level=1).text("Function pointer")
    list_item(list_wrapper, show="next+",
              level=1).text("Function return address")
    list_item(list_wrapper, show="next+", level=1).text("Virtual method")

    if backup:
        slide = new_slide(slides)
        slide.update_style("code", s(size=26))
        content = slide_header(slide, "Code (backup)")
        code(
            content,
            """struct A { virtual void handle(size_t* data) const = 0; };
struct B: public A { void handle(size_t* data) const final { *data += 1; } };
struct C: public A { void handle(size_t* data) const final { *data += 2; } };

std::vector<std::unique_ptr<A>> data = /* 4K random B/C instances */;
// std::sort(data.begin(), data.end(), /* sort by instance type */);
size_t sum = 0;
for (auto& x : data)
{
    x->handle(&sum);
}""")

        slide = new_slide(slides)
        content = slide_header(slide, "Result (backup)")
        content.box(width="90%").image("images/example0b-time.png")

        slide = new_slide(slides)
        content = slide_header(slide, "perf (backup)")
        bash(content.box(),
             """$ perf stat -e branch-misses ./example0b
with sort   ->     337 274
without sort -> 84 183 161""",
             text_style=s(align="left"))
Example #29
0
def unsafe(slides: Slides):
    slide = slides.new_slide()
    slide.update_style("code", s(size=32))
    content = slide_header(slide, "Where's the catch?")
    content.box().text("We have seen things that mutate through a shared borrow")

    code_width = 800
    code_step(content.box(width=code_width, height=400), """
// Arc::clone
fn clone(&self) -> Arc<T>;
// Mutex::lock
fn lock(&self) -> &mut T;
// AtomicU64::store
fn store(&self, val: u64, order: Ordering);
""", 2, [
        (0, 1, None, None, None, None),
        (0, 1, 2, 3, None, None),
        (0, 1, 2, 3, 4, 5)
    ], width=code_width)

    content.box(show="5+").text("This is called ~tt{interior mutability} and requires unsafe Rust",
                                s(size=32))

    slide = slides.new_slide()
    content = slide_header(slide, "Enter unsafe Rust")
    content.box().text("Some scenarios are not expressible in (safe) Rust")

    content.box(height=20)
    content.box(show="next+").text("In some cases, something more is required to:")

    list = content.box()
    list_item(list, show="next+").text("Express inherently unsafe paradigms")
    list_item(list, show="next+").text("Improve performance")
    list_item(list, show="next+").text("Interact with I/O, OS, hardware, network")

    slide = slides.new_slide()
    content = slide_header(slide, "Unsafe Rust")

    content.box().text("You can mark parts of code with the ~tt{unsafe} keyword")
    content.box(show="next+").text("Unsafe Rust is a ~emph{superset} of Rust")

    def unsafe_slide(header, code_body, content_show="1", code_size=36):
        slide = slides.new_slide()
        slide.update_style("code", s(size=code_size))
        content = slide_header(slide, "Unsafe Rust")
        content.box(y=0).text("Unsafe Rust allows:")

        content.box(height=20)
        content.box(show=content_show).text(header)
        content.box(height=10)
        code(content.box(show=content_show), code_body)

    unsafe_slide("Accessing a global mutable variable", """
static mut COUNTER: u32 = 0;

fn increment_count() {
    unsafe {
        COUNTER += 1;
    }
}""", content_show="2+")
    unsafe_slide("Dereferencing a raw pointer", """
let ptr = 0xCAFECAFE as *mut u32;
unsafe {
    *ptr = 5;
}""")
    unsafe_slide("Calling an unsafe function", """
unsafe {
    zlib_compress(&buffer, buffer.len());
}""")
    unsafe_slide("Implementing an unsafe trait", """
unsafe impl Send for MySuperSafeType {
    ...
}""")

    slide = slides.new_slide()
    slide.update_style("code", s(size=18))
    content = slide_header(slide, "Finding unsafe code - C++")
    code_box = code(content, """
std::atomic<LifecycleId> ArenaImpl::lifecycle_id_generator_;
GOOGLE_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, NULL};

void ArenaImpl::Init() {
  lifecycle_id_ =
      lifecycle_id_generator_.fetch_add(1, std::memory_order_relaxed);
  hint_.store(nullptr, std::memory_order_relaxed);
  threads_.store(nullptr, std::memory_order_relaxed);

  if (initial_block_) {
    // Thread which calls Init() owns the first block. This allows the
    // single-threaded case to allocate on the first block without having to
    // perform atomic operations.
    new (initial_block_) Block(options_.initial_block_size, NULL);
    SerialArena* serial =
        SerialArena::New(initial_block_, &thread_cache(), this);
    serial->set_next(NULL);
    threads_.store(serial, std::memory_order_relaxed);
    space_allocated_.store(options_.initial_block_size,
                           std::memory_order_relaxed);
    CacheSerialArena(serial);
  } else {
    space_allocated_.store(0, std::memory_order_relaxed);
  }
}
""")
    code_box.overlay(show="2+", z_level=99).rect(bg_color=CODE_HIGHLIGHT_COLOR)

    slide = slides.new_slide()
    content = slide_header(slide, "Finding unsafe code - Rust")
    bash(content.box(), '$ grep "unsafe" main.rs', text_style=s(size=40))

    slide = slides.new_slide()
    slide.box().text("""Rust builds safe abstractions
on top of unsafe foundations""", s(size=50))
Example #30
0
def denormals(slides: SlideDeck, backup: bool):
    if backup:
        slide = new_slide(slides)
        content = slide_header(slide, "Code (backup)")
        code(content.box(), """float F = static_cast<float>(std::stof(argv[1]));
std::vector<float> data(4 * 1024 * 1024, 1);

for (int r = 0; r < 100; r++)
{
    for (auto& item: data)
    {
        item *= F;
    }
}""")
        slide = new_slide(slides)
        content = slide_header(slide, "Result (backup)")
        content.box(width="50%").image("images/example2-time.png")

    slide = new_slide(slides)
    colors = ["#B22222", "#007944", "#0018AE"]
    styles = ["sign", "exponent", "significand"]
    for i, style in enumerate(styles):
        slide.set_style(style, s(color=colors[i], size=42))
    content = slide_header(slide, "Denormal floating point numbers")

    row = content.box(horizontal=True)
    box_dimension = 60

    def floating_point(wrapper: Box, colors, values):
        for i in range(len(colors)):
            box = wrapper.box(width=box_dimension, height=box_dimension)
            box.rect(color="black", bg_color=colors[i], stroke_width=2)
            box.text(str(values[i]), s(color="white", bold=True))

    floating_point(row, [colors[0]] + [colors[1]] * 5 + [colors[2]] * 10, [
        0,
        0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 1
    ])
    row_label = content.box(x=0, horizontal=True, show="next+")
    row_label.box(x=250, y=0).text("Zero exponent")
    row_label.box(x=670, y=0).text("Non-zero significand")

    content.box(p_top=80, width=700, height=50).image("images/float.svg")

    list_wrapper = content.box(p_top=20)
    list_item(list_wrapper, show="next+").text("Numbers close to zero")
    list_item(list_wrapper, show="last+").text("Hidden bit = 0, smaller bias")

    content.box(p_top=40, show="next+").text("Operations on denormal numbers are slow!", style=s(size=46))

    slide = new_slide(slides)
    content = slide_header(slide, "Floating point handling")
    content.box(height=600).image("images/haswell-diagram.png")
    content.box(width=130, height=32, x=800, y=52).rect(color=COLOR_FRONTEND, stroke_width=3)
    content.box(width=35, height=100, x=988, y=80).rect(color=COLOR_FRONTEND, stroke_width=3)
    content.box(width=80, height=165, x=212, y=335).rect(color=COLOR_BACKEND, stroke_width=3)

    slide = new_slide(slides)
    content = slide_header(slide, "How to measure?")
    content.box().text("~tt{fp_assist.any}", style=s(size=48))
    content.box(p_top=20).text("How many times the CPU switched to the microcode FP handler?")

    if backup:
        bash(content.box(p_top=40, show="next+"), """$ perf stat -e fp_assist.any ./example2
0   ->          0
0.3 -> 15 728 640""", text_style=s(align="left"))

    slide = new_slide(slides)
    slide.update_style("code", s(size=40))
    content = slide_header(slide, "How to fix it?")
    list_wrapper = content.box()
    list_item(list_wrapper).text("The nuclear option: ~tt{-ffast-math}")
    list_item(list_wrapper, level=1).text("Sacrifice correctness to gain more FP performance")
    list_item(list_wrapper, show="next+").text("Set CPU flags:")
    list_item(list_wrapper, level=1, show="last+").text("Flush-to-zero - treat denormal outputs as 0")
    list_item(list_wrapper, level=1, show="last+").text("Denormals-to-zero - treat denormal inputs as 0")

    code(list_wrapper.box(p_top=40, show="next+"), """_mm_setcsr(_mm_getcsr() | 0x8040);
// or
_MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON);
_MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
""")
Example #31
0
    def qe(self, sourceLang, targetLang, sourceText, targetText):
        """
        Performs translation quality estimation on sourceText to targetText using deepQuest
        It's ok to raise Exceptions here. They are handled upstream.
        """

        task_name = f'{sourceLang}_{targetLang}'
        if not task_name in self.pairsEpochs.keys():
            raise Exception(
                f'{sourceLang}-{targetLang} language pair not supported')
        #if not os.path.isdir(f'data/{task_name}'):
        if not os.path.isdir(f'qe/deepQuest-config/saved_models/{task_name}'):
            raise Exception(f'data/{task_name} does not exits')

        os.makedirs('data/tmp', exist_ok=True)

        aligned = hunalign(sourceText, targetText)

        repeatText = lambda text, times=100: '\n'.join([text] * times)

        # Sanitize input
        sourceText = [tokenize(x[0], sourceLang, False) for x in aligned]
        targetText = [tokenize(x[1], sourceLang, False) for x in aligned]
        sourceTextPlain = '\n'.join([' '.join(x) for x in sourceText])
        targetTextPlain = '\n'.join([' '.join(x) for x in targetText])

        fileSource = 'qe/deepQuest-config/data_input/test.src'
        with open(fileSource, 'w') as f:
            f.write(repeatText(sourceTextPlain) + '\n')

        fileTarget = 'qe/deepQuest-config/data_input/test.trg'
        with open(fileTarget, 'w') as f:
            f.write(repeatText(targetTextPlain) + '\n')

        tokensTarget = [item for sublist in targetText for item in sublist]

        best_epoch = self.pairsEpochs[task_name]
        store_path = f'../../deepQuest-config/saved_models/{task_name}'
        filename = lambda threshold: f'{store_path}/val_epoch_{best_epoch}_threshold_0.{threshold}_output_0.pred'

        with DirCrawler('qe/deepQuest/quest'):
            (_output, _error) = bash(f"""
                bash ../../deepQuest-config/estimate-wordQEbRNN.sh {task_name} {best_epoch}
                 """)
            #print(_output)
            #print(_error)

            features = []
            for i in range(10):
                outputFile = filename(i)
                if not os.path.isfile(outputFile):
                    raise Exception('Server Processing Error')
                with open(outputFile, 'r') as outputFile:
                    features.append(
                        [1 * (x == 'OK\n') for x in outputFile.readlines()])

            # Transpose
            features = [[features[j][i] for j in range(len(features))]
                        for i in range(len(features[0]))]
            # Average lines
            features = [sum(x) / len(x) for x in features]
            # Take only relevant number of tokens
            features = features[:len(tokensTarget)]

            os.remove('log-keras.txt')
            os.remove('log-keras-error.txt')
            shutil.rmtree('datasets')
            to_remove = \
                ['val.qe_metrics', f'val_epoch_{best_epoch}_output_0.pred'] + \
                [f'val_epoch_{best_epoch}_threshold_0.{str(x)}_output_0.pred' for x in range(10)]
            [os.remove(f'{store_path}/' + x) for x in to_remove]

        os.remove(fileSource)
        os.remove(fileTarget)

        return {'status': 'OK', 'qe': features}