Esempio n. 1
0
    def makeAllPlots(cls, figDats, overwrite, force, silent=True):
        import asyncio
        from wolframclient.evaluation import WolframEvaluatorPool
        wolfram_expressions = super().makeAllPlots(figDats, overwrite, force)
        wolfram_expressions2 = []
        for w in wolfram_expressions:
            if not silent:
                print('showing')
            # ww = wl.Show(w[0])
            ww = wl.Show(listmap(lambda x: x[0], w))
            # file = w[1]
            file = w[0][1].abspath
            wolfram_expressions2 += [
                wl.UsingFrontEnd(wl.Export(file, ww, ImageSize(1000)))
            ]

        async def runasync():
            log('running runasync')
            async with WolframEvaluatorPool() as pool:
                countr = Counter(a=1)

                async def logAfter(wlexp, c, total):
                    await pool.evaluate(wlexp)
                    log(f'Finished making {c["a"]}/{total} figures')
                    c['a'] += 1

                tasks = []
                for exp in wolfram_expressions2:
                    # for exp in wolfram_expressions:
                    tasks += [logAfter(exp, countr, len(figDats))]
                await asyncio.wait(tasks)

        asyncio.run(runasync())
Esempio n. 2
0
def manage():
    cloud_files = weval(wl.CloudObjects(wlexpr('$CloudRootDirectory')))
    for wcf in listmap(File, cloud_files):
        if wcf.abspath in _REGISTERED_SUBROOTS:
            pass
        elif wcf.abspath == _MAIN_SUBROOT:

            @log_invokation(with_args=True, stack=True)
            def recurse_cloud_file(sub_wcf):
                # f = File(f'{sub_wcf.abspath}')
                if not sub_wcf.exists:
                    recurse_cloud_file.my_stacker.done = True
                    if boolinput(
                            f'{sub_wcf} is not mirrored locally, delete cloud file?'
                    ):
                        sub_wcf.wc.delete()
                if sub_wcf.wc.isdir:
                    if Folder(sub_wcf)['.CLOUD_FILES.txt'].exists:
                        Folder(sub_wcf)['.CLOUD_FILES.txt'].write('\n'.join(
                            listmap(
                                lambda e: e.abspath.replace(
                                    f'{Folder(sub_wcf).abspath}/', ''),
                                sub_wcf.wc.files)))
                    else:
                        [recurse_cloud_file(c) for c in sub_wcf.wc.files]

            recurse_cloud_file(wcf)
            recurse_cloud_file.my_stacker.done = True
        else:
            err(f'{wcf.abspath} is not a registered Wolfram Cloud subroot')
Esempio n. 3
0
        def bar(cls, fd):
            from matplotlib import pyplot as plt  # 1 FULL SECOND IMPORT
            maxY = None if fd.maxY is None or fd.maxY == 'inf' or not isreal(
                fd.maxY) else float(fd.maxY)
            minY = None if fd.minY is None or fd.minY == 'inf' or not isreal(
                fd.minY) else float(fd.minY)
            # maxX = None if fd.maxX is None or fd.maxX == '-inf' or not isreal(fd.maxX) else float(fd.maxX)
            # minX = None if fd.minX is None or fd.minX == 'inf' or not isreal(fd.minX) else float(fd.minX)

            if maxY != None and minY != None:
                diff = maxY - minY
                pad = diff * (fd.y_pad_percent / 100)
                maxY = maxY + pad
                minY = minY - pad

            if cls.fig is None:
                if BLACK_FIGS:
                    cls.fig = plt.figure(figsize=(16, 12), facecolor='black')
                    cls.ax = cls.fig.add_subplot(111, facecolor='black')
                else:
                    cls.fig = plt.figure(figsize=(16, 12))
                    cls.ax = cls.fig.add_subplot(111)
            cls.ax.bar(fd.x,
                       fd.y,
                       color=listmap(cls.color, fd.item_colors),
                       yerr=fd.err)
            title_obj = cls.ax.set_title(fd.title, fontsize=fd.title_size)
            plt.setp(title_obj, color=text_color)
            cls.ax.axis(True)
            cls.ax.spines['left'].set_color(text_color)
            cls.ax.spines['bottom'].set_color(text_color)
            cls.ax.xaxis.label.set_color(text_color)
            cls.ax.yaxis.label.set_color(text_color)
            cls.ax.tick_params(axis='x', colors=text_color)
            cls.ax.tick_params(axis='y', colors=text_color)
Esempio n. 4
0
def line_scatter(title,
                 ts: TimeSeries,
                 *tps,
                 tp_ref=None,
                 displayModeBar=False,
                 staticPlot=True,
                 line_args=None,
                 fig_args=None,
                 layout=None,
                 annotations=None):
    import plotly.graph_objs as go
    if layout is None: layout = {}
    if fig_args is None: fig_args = {}
    if line_args is None: line_args = {}
    if annotations is None: annotations = []

    colors = (x for x in ['#0000B4', '#9FEB4C'])

    if tp_ref is None:
        tp_ref = ts
    data = [
        go.Scatter(
            x=ts.t,
            y=ts.y,
            line=dict(width=1, color='#FF0000'
                      # shape='spline'
                      )),
        *listmap(
            lambda tp: go.Scatter(
                x=tp_ref.t[tp],
                y=tp_ref.y[tp],
                mode='markers',
                marker=dict(
                    color=next(colors),
                    size=20  # default 6
                )),
            tps)
    ]
    layout = go.Layout(
        template='plotly_dark',
        title=title,
        annotations=annotations,
        # yaxis=dict(autorange=True),
        **layout)
    fig = go.Figure(data=data, layout=layout, **line_args)
    fig.update_layout(
        xaxis_title=ts.t_unit.name,
        yaxis_title=ts.y_unit,
        xaxis=dict(
            # the real reason im doing this is because in the animations, the ticks are annoyingly flickering. maybe if I hard-set this it will fix that problem.
            tickmode='linear',
            tick0=0,
            dtick=1))
    return _get_fig(fig,
                    displayModeBar=displayModeBar,
                    staticPlot=staticPlot,
                    **fig_args)
Esempio n. 5
0
 def command_list(*args):
     if len(args) == 0:
         return ['']
     elif len(args) > 1:
         return list(map(AbstractShell.com_arg, args))
     elif is_non_str_itr(args[0]):
         return list(map(AbstractShell.com_arg, args[0]))
     elif isstr(args[0]):
         return listmap(AbstractShell.com_arg, args[0].split(' '))
     else:
         return [AbstractShell.com_arg(args[0])]
Esempio n. 6
0
 def recurse_cloud_file(sub_wcf):
     # f = File(f'{sub_wcf.abspath}')
     if not sub_wcf.exists:
         recurse_cloud_file.my_stacker.done = True
         if boolinput(
                 f'{sub_wcf} is not mirrored locally, delete cloud file?'
         ):
             sub_wcf.wc.delete()
     if sub_wcf.wc.isdir:
         if Folder(sub_wcf)['.CLOUD_FILES.txt'].exists:
             Folder(sub_wcf)['.CLOUD_FILES.txt'].write('\n'.join(
                 listmap(
                     lambda e: e.abspath.replace(
                         f'{Folder(sub_wcf).abspath}/', ''),
                     sub_wcf.wc.files)))
         else:
             [recurse_cloud_file(c) for c in sub_wcf.wc.files]
Esempio n. 7
0
def caller_lines():
    # inner_fun = caller_fun(1)
    # outer_fun = caller_fun(2)
    outer_file = caller_file(2)
    start = official_caller_line(2)
    end = start + 1
    lines = listmap(lambda l: l.strip(), File(outer_file).readlines())
    while True:
        if end - start > 40: err('really???')
        e = '\n'.join(
            lines[start - 1:end -
                  1]) + '\n'  # MUST END WITH NEWLINE IN EVAL OR SINGLE MODE
        try:
            compile(e, '<string>', 'exec')
            break
        except SyntaxError:
            end += 1
    return list(range(start, end))
Esempio n. 8
0
    def confuse_analysis(self, data, lamb, identiy=True):
        @dataclass
        class IN_Result:
            backend: str
            arch: str
            pp: str
            y_pred: np.ndarray
            acts: np.ndarray
            def __str__(self):
                be = SanityAnalysis.bedict[self.backend]
                a = SanityAnalysis.adict[self.arch]
                p = SanityAnalysis.ppdict[self.pp]
                return f'{be}{a}{p}'
        in_results = []
        for bekey, bedata in listitems(data):
            if bekey in ['files', 'dest', 'y_true']: continue
            for akey, arch_data in listitems(bedata):
                for ppkey, ppdata in listitems(arch_data):
                    in_results += [IN_Result(
                        backend=bekey,
                        arch=akey,
                        pp=ppkey,
                        y_pred=arr(ppdata['y_pred']),
                        acts=arr(ppdata['acts'])
                    )]

        cmat = self.iconfuse(
            in_results,
            lamb,
            identiy=identiy
        )

        labels = listmap(
            lambda r: str(r),
            in_results
        )
        top = [None] + labels
        cmat = cmat.tolist()
        for i, li in enum(cmat):
            cmat[i] = [labels[i]] + cmat[i]
        cmat = [top] + cmat
        return cmat
Esempio n. 9
0
File: html.py Progetto: mgroth0/mlib
 def __init__(self, values, *objs, **kwargs):
     super().__init__(*listmap(HTMLOption, values), *objs, **kwargs)
Esempio n. 10
0
File: html.py Progetto: mgroth0/mlib
    def getCode(self,
                resource_root,
                resource_root_rel,
                force_fix_to_abs=False):
        ml = '<!DOCTYPE html>'

        # with Temp('temp.css') as f:
        #     f.write(self.style)
        head_objs = [
            '''<meta content="text/html;charset=utf-8" http-equiv="Content-Type">
            <meta content="utf-8" http-equiv="encoding">''',
            '<META HTTP-EQUIV="CACHE-CONTROL" CONTENT="NO-CACHE">'
        ] + ([HTMLCSSLink(href='style.css')] if self.style_dot_css else []) + [
            StyleTag(self.style)
            # StyleTag(lesscpy.compile(f.abspath, minify=True))
        ]

        if self.jQuery:
            head_objs.extend([
                ExternalScript(
                    src=
                    "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"
                ),
                ExternalScript(
                    src=
                    "https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"
                ),
                HTMLCSSLink(
                    href=
                    "https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/themes/smoothness/jquery-ui.css"
                )
            ])

        for cdn in self.CDNs:
            head_objs.append(ExternalScript(cdn))

        # head_objs.append(ExternalScript(
        # Blocked loading mixed active content
        #     src='http://cdn.jsdelivr.net/gh/bestiejs/platform.js/platform.js',
        # ))

        head_objs.extend(
            listmap(lambda x: ExternalScript(src=x.replace('.coffee', '.js')),
                    self.javascript_files) +
            ([JScript(JS(self.js, onload=self.wrap_js))] if self.
             js is not None else []))
        columnChildren = []
        for c in self.children:
            columnChildren.append(c)
            if isstr(c) or 'hidden' not in listkeys(c.attributes):
                columnChildren.append(Br)
        if columnChildren and columnChildren[-1] == Br:
            columnChildren = columnChildren[:-1]

        ml += HTMLRoot(
            HTMLHead(*head_objs),
            HTMLBody(*columnChildren,
                     style=self.bodyStyle,
                     **self.bodyAttributes,
                     identified=self.identified).getCode(
                         resource_root, resource_root_rel,
                         force_fix_to_abs)).getCode(resource_root,
                                                    resource_root_rel,
                                                    force_fix_to_abs)

        return ml
Esempio n. 11
0
def get_report(md,
               resources_root,
               exp_name,
               index_url,
               database: Database = None,
               api: SimpleAdminAPI = None,
               editable=False,
               web_resources_root=None,
               show=False):
    if exp_name not in listkeys(database):
        database[exp_name] = {}

    analysis_figdata = flat1([
        a.get_report_figdata(exp_name, resources_root, database)
        for a in ANALYSES(mode=AnalysisMode.PIPELINE)
    ])

    all_arch_figs = []
    ntrain_figs = md.ntrainims
    nepochs = md.nepochs

    for n in ntrain_figs:
        for suffix in [
                f'__val_CM{nepochs}', f'__L2-Output_CM{nepochs}',
                f'__L2-Inter_CM{nepochs}', f'__L2-Raw_CM{nepochs}'
        ]:
            arch_figs = listmap(__.label, md.archs)
            arch_figs = listmap(
                lambda a: (resources_root[f'{a}_{n}{suffix}.png'],
                           database.get_or_set_default(
                               '', exp_name, f'{a}_{n}{suffix}')), arch_figs)

            all_arch_figs.extend(arch_figs)

    mcc_name = f'Matthews_Correlation_Coefficient'

    ntrain_figs = listmap(
        lambda n:
        (resources_root[f'{mcc_name}_{n}.png'],
         database.get_or_set_default('', exp_name, f'{mcc_name}_{n}')),
        ntrain_figs)
    doc = HTMLPage(
        f'index',
        f'Symmetry Detection Report by Matt Groth',
        f'',
        f'{len(md.archs)} Architectures: ',
        *listmap(__.label, md.archs),
        f'Experiments: {md.nrepeats} per architecture',
        f'Epochs: {md.nepochs} per experiment',
        f'Batch Size: {md.batchsize}',
        f'Training Images: {md.ntrainims}',
        f'Normalized Individual Images: {md.normalized}',
        FigureTable(
            *analysis_figdata,
            (resources_root['Final_Train_MCC.png'],
             database.get_or_set_default('', exp_name, 'Final_Train_MCC')),
            *ntrain_figs,
            (pwdf()["/_figs/figs_misc/RSA_patterns/RSA_patterns.001.jpeg"],
             database.get_or_set_default('', exp_name, 'RSA_patterns'))
            if INCLUDE_RSA_SLIDE else None,
            *all_arch_figs,
            resources_root=web_resources_root,
            exp_id=exp_name,
            editable=editable),
        Hyperlink('back to index', index_url),
        api.apiElements(),
        js=api.cs(),
        style=DNN_REPORT_CSS,
        show=show)

    return doc
Esempio n. 12
0
def analyze_exp_group(eg: DNN_ExperimentGroup, cfg):
    eg.compile_folder.deleteIfExists()
    eg.metadata.copy_into(eg.compile_folder)

    ARCH_LABELS = listmap(__['label'], eg.metadata['archs'])
    NTRAINS = eg.metadata['ntrainims']
    NEPOCHS = eg.metadata['nepochs']

    [
        a.during_compile(eg) for a in ANALYSES(mode=AnalysisMode.PIPELINE)
        if a.should_run(cfg)
    ]

    experiments = experiments_from_folder(eg.folder)
    random_exp = experiments[0]

    TrainTable = FinalResult(2,
                             'test/Matthews_Correlation_Coefficient.mfig',
                             data_exists=random_exp.folder[f'test'].exists,
                             is_table=True,
                             rows=ARCH_LABELS,
                             cols=NTRAINS)

    random_exp.folder['log.pkl'].copy_into(eg.compile_folder)

    def maybe_avg_result(pre, nepochs, is_table=False, dims=2, suf=None):
        return AverageResult(
            dims,
            f'{pre}/CM{nepochs}.mfig' if suf is None else f'{pre}/{suf}',
            data_exists=random_exp.folder[pre].exists,
            is_table=is_table)

    results_to_compile = []
    for ni, ntrain in enum(NTRAINS):
        MCC = maybe_avg_result('test',
                               None,
                               dims=1,
                               suf='Matthews_Correlation_Coefficient.mfig')
        for ai, arch in enum(ARCH_LABELS):
            if cfg.salience:
                results_to_compile = [TrainTable]
            else:
                results_to_compile = [TrainTable, MCC]

            if random_exp.folder['L2-Output'].exists:
                maybe_avg_result(f'L2-Output', NEPOCHS)
                maybe_avg_result(f'L2-Inter', NEPOCHS)
                maybe_avg_result(f'L2-Raw', NEPOCHS)

            if not cfg.salience:
                results_to_compile.append(
                    maybe_avg_result(f'val', NEPOCHS, is_table=True))

            results_to_compile = [
                r for r in results_to_compile if r is not None
            ]
            for exp in experiments.filtered(
                    lambda e: e.arch == arch and e.ntrain == ntrain, ):
                for res in results_to_compile:
                    try:
                        res.append(res.exp_data(exp), (ai, ni, 0),
                                   is_GNET=arch == 'GNET')
                    except:
                        breakpoint()

            for res in results_to_compile:
                if not res.data_exists: continue
                if res.j is None:
                    log('about to breakpoint')
                    breakpoint()
                else:
                    log('res.j is not none, so no breakpoint')
                for vis in res.j.viss:
                    vis.make = True
                if res.dims == 1:
                    LINE_INDEX = -1
                    avg = np.mean(res.data, axis=0).tolist()
                    res.data = arr2d()
                    res.j.viss.append(copy.deepcopy(res.template))
                    res.j.viss[LINE_INDEX].make = True
                    res.j.viss[LINE_INDEX].y = avg
                    res.j.viss[LINE_INDEX].item_colors = CONTRAST_COLORS[ai]
                    for v in itr(res.j.viss):
                        res.j.viss[v].title = f'MCC(nTrainIms={ntrain})'
                    res.j.viss[LINE_INDEX].y_label = arch
                elif res.dims == 2:
                    if isinstance(res, AverageResult):
                        avg = np.mean(res.data, axis=2)
                    else:
                        avg = res.data[:, :, 0]
                    res.j.viss[0].confuse_max = flatmax(avg)
                    # res.j.viss[0].title_size = 30
                    if res.is_table:
                        avg = np.concatenate((res.row_headers[1:], avg),
                                             axis=1)
                        avg = np.concatenate((res.col_headers, avg), axis=0)
                    if isinstance(res, AverageResult):
                        res.j.viss[0].data = avg.tolist()
                        res.j.viss[0].title = res.j.viss[0].title.replace(
                            " ", f'{ntrain} ', 1)
                        eg.compile_exp_res_folder[
                            f'{arch}_{ntrain}__{res.suffix.replace("/", "_")}'].save(
                                res.j)
                    elif ai == len(ARCH_LABELS) - 1 and ni == len(NTRAINS) - 1:
                        res.j.viss = [
                            ConfusionMatrix(data=avg.tolist(),
                                            title="Final Training MCCs",
                                            confuse_min=0,
                                            confuse_max=1,
                                            headers_included=True,
                                            make=True,
                                            side_header_label='Architecture',
                                            top_header_label='#Train Images')
                        ]
                        eg.compile_exp_res_folder[
                            f'Final_Train_MCC.mfig'] = res.j

        for res in [
                mcc for mcc in results_to_compile
                if mcc.dims == 1 and mcc.data_exists
        ]:
            eg.compile_exp_res_folder[StringExtension(res.suffix[1:]).r({
                "test/":
                "",
                ".":
                f"_{ntrain}."
            })].save(res.j)
Esempio n. 13
0
    def compare(self, fun: Type[Correlation], GPU=False):
        special_confuse_mat = zeros(len(self.data), len(self.data))

        if (fun == PearsonCorrelation) and any([min(x) == max(x) for x in self.data]):
            raise MathFail
        #     # Pearson's Correlation Coefficient fails if
        #     # two arrays are commpared that have a zero standard deviation product (divide by zero)
        #     # Using an if statement above, I should prevent this

        data = self.data  # pleasework
        def _fun(i):  # cannot be lambda?
            return [(i, j, fun.fun(data[i, :], data[j, :])) for j in itr(data)]
        def _fun_tf(data):  # cannot be lambda?
            return fun.fun_tf(data)

        MULTIPROCESS = False

        from pathos.multiprocessing import ProcessPool


        if islinux() and MULTIPROCESS:
            #     slower than GPU
            #     BUGGY
            #     not optimized

            with ProcessPool() as p:
                # if islinux():
                # mapid = randrange(0,10000)
                # print(f'starting map {mapid}')
                r = p.map(_fun, itr(self.data))
            for results in r:
                for rr in results:
                    special_confuse_mat[rr[0], rr[1]] = rr[2]

        elif islinux() and GPU:
            import tensorflow as tf
            special_confuse_mat = tf.zeros((len(self.data), len(self.data)))







            with tf.device('/GPU:0'):
                special_confuse_mat = _fun_tf(self.data).numpy()

            # results[net] = rsa.numpy()
            # tfdata = tf.convert_to_tensor(self.data).cuda()

        else:
            r = listmap(_fun, itr(self.data))

            for results in r:
                for rr in results:
                    special_confuse_mat[rr[0], rr[1]] = rr[2]

        return ComparisonMatrix(
            data=nan_above_eye(naneye(special_confuse_mat)),
            method_used=fun.__name__,
            ground_truth=self.ground_truth,
            class_set=self.class_set
        )
Esempio n. 14
0
 def files(self):
     return arr(
         listmap(lambda wcf: File(wcf),
                 weval(CloudObjects(self.file.abspath))))
Esempio n. 15
0
    def report(self, signal, t, exps):
        if not self.remote:
            if False:  # DEBUG
                if signal is not None:
                    signal.emit('no local report yet')
                else:
                    log('no local report yet')
                return
        the_report = '\n\n\t\t\t~~JOB REPORT~~'
        if self.first_report[''] and signal is not None:
            signal.emit(the_report)
        for e in exps:
            the_report += f'\n{e.status()}'
            if self.first_report[''] and signal is not None:
                signal.emit(the_report)
        the_report += '\n\n'
        if self.first_report[''] and signal is not None:
            signal.emit(the_report)

        print(the_report)

        while True:  # clear buffer
            line = self.statusP.readline_nonblocking(1)
            if line is None: break

        log('GETTING GPU REPORT')
        gpu_report = '\n\t\t\t~~GPU REPORT~~'
        self.statusP.sendline(f'nvidia-smi; echo ${self.DONE_VAR}')
        tesla_line = False
        percents = []
        while True:
            line = self.statusP.readline_nonblocking(1)
            if line is None or self.DONE_STR in line: break
            else:
                if tesla_line:
                    percents += [int(line.split('%')[0][-2:])]
                tesla_line = 'Tesla P4' in line
        for idx, perc in enum(percents):
            gpu_report += f'\n{idx}\t{insertZeros(perc, 2)}% {Progress.prog_bar(perc, BAR_LENGTH=self.REP_BAR_LENGTH)}'
        the_report += gpu_report
        if self.first_report[''] and signal is not None:
            signal.emit(the_report)
        log('GETTING MEM REPORT')
        mem_report = '\n\n\t\t\t~~MEM REPORT~~'
        self.statusP.sendline(f'free -h; echo ${self.DONE_VAR}')
        log('send mem report request')
        while True:
            line = self.statusP.readline_nonblocking(1)
            if line is None or self.DONE_STR in line: break
            else:
                if self.MATT_STR not in line:
                    mem_report += f'\n{line}'
        the_report += mem_report
        if self.first_report[''] and signal is not None:
            signal.emit(the_report)
        log('\nGETTING CPU REPORT')
        cpu_report = '\n\n\t\t\t~~CPU REPORT~~'
        self.statusP.sendline(
            f'''echo "CPU `LC_ALL=C top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\\1/" | awk '{{print 100 - $1}}'`% RAM `free -m | awk '/Mem:/ {{ printf("%3.1f%%", $3/$2*100) }}'` HDD `df -h / | awk '/\// {{print $(NF-1)}}'`"; echo ${self.DONE_VAR}'''
        )
        log('SENT CPU LINE')
        the_report += cpu_report
        cpu_report = ''
        while True:
            line = self.statusP.readline_nonblocking(1)
            if line is None or self.DONE_STR in line: break
            else:
                if self.MATT_STR not in line:
                    cpu_report += f'\n{line}'

        cpu_stuff = tuple(
            listmap(lambda s: s.replace('%', ''),
                    cpu_report.strip().split(' ')))
        if len(cpu_stuff) == 6:
            cpu_perc = float(cpu_stuff[1])
            ram_perc = float(cpu_stuff[3])
            hdd_perc = float(cpu_stuff[5])
            cpu_report = f'\nCPU\t{insertZeros(cpu_perc, 4)}% {Progress.prog_bar(cpu_perc, BAR_LENGTH=self.REP_BAR_LENGTH)}'
            cpu_report += f'\nRAM\t{insertZeros(ram_perc, 4)}% {Progress.prog_bar(ram_perc, BAR_LENGTH=self.REP_BAR_LENGTH)}'
            cpu_report += f'\nHDD\t{insertZeros(hdd_perc, 4)}% {Progress.prog_bar(hdd_perc, BAR_LENGTH=self.REP_BAR_LENGTH)}'

            the_report += cpu_report
        else:
            the_report += f'problem getting cpu_report ({len(cpu_stuff)=})'
        log('sending signal with REPORT')
        if signal is not None:
            signal.emit(the_report)
        log('sent signal with REPORT')
        self.next_report[''] = t + 1
        self.first_report[''] = False
Esempio n. 16
0
File: dnn.py Progetto: mgroth0/dnn
class DNN(Project):
    MODES = ['JUSTRUN', 'PUSH', 'COMPILE_TEST_ALL', 'MAKEREPORT']
    HUMAN_EXPS_FOLDER = Folder('human_exps')
    if HUMAN_EXPS_FOLDER.exists:
        _human_exp_flags = listmap(__.name, HUMAN_EXPS_FOLDER.folders)
    else:
        _human_exp_flags = []
    extra_flags = _human_exp_flags + MODES + [
        'IMAGENET_COUNT', 'RSA_MAIN', 'RSA_NEW', 'ASD'
    ]

    def run(self, cfg):
        # keep modular
        assert not (cfg.REGEN_DATA and cfg.OVERWRITE_NORMS
                    )  # btw, both imply killing worker before exp

        from lib.dnn_lib import dnn
        from mlib.web.api import API
        from mlib.web.database import Database

        shadow.SHOW_INDEX = False

        if len(cfg.FLAGS) == 1 and cfg.FLAGS[0] == 'IMAGENET_COUNT':
            import count_imagenet_data
            count_imagenet_data.count()
            return None
        elif len(cfg.FLAGS) == 1 and cfg.FLAGS[0] == 'RSA_MAIN':
            # print('here1, doing Darius-RSA')
            # import rsa_for_darius
            import rsa_comp
            rsa_comp.main()
            finish_dnn_remote()
            return None
        elif len(cfg.FLAGS) == 1 and cfg.FLAGS[0] in self._human_exp_flags:
            from lib.human_exp import human_exp
            human_exp(cfg)
        else:
            flag_mode = ''.join(
                arr(cfg.FLAGS).filtered(lambda s: s in self.MODES))
            if not isblank(flag_mode): cfg.MODE = flag_mode
            if isblank(cfg.MODE):
                cfg.MODE = ''.join(self.MODES)  # unnecessary?
            if cfg.offline:
                API.offline_mode = True
                Database.offline_mode = True
                from lib import makereport
                makereport.MAKEREPORT_ONLINE = False
            from mlib.km import kmscript  # keep modular
            if ismac():
                kmscript('activate run tool window')
            dnn(cfg)
            finish_dnn_remote()

    instructions = '''Generate some images, train/test a model, run analyses, and generate plots. Tested on Mac, but not yet on linux/Windows.

- `./dnn -cfg=gen_images --INTERACT=0`
- `./dnn -cfg=test_one --INTERACT=0`

The second command will fail with a Mathematica-related error, but your results will be saved in `_figs`.

TODO: have to also consider running and developing other executables here: human_exp_1 and human_analyze

'''
    configuration = '''-MODE: (default = FULL) is a string that can contain any combination of the following (example: "CLEAN JUSTRUN")
- CLEAN
- JUSTRUN
- GETANDMAKE
- MAKEREPORT

Edit [cfg.yml]() to save configuration options. Feel free to push these.

If there is anything hardcoded that you'd like to be configurable, please submit an issue.'''
    credits = 'Darius, Xavier, Pawan\n\nheuritech, raghakot, joel'
Esempio n. 17
0
def ANALYSES(mode: AnalysisMode):
    [__import__(f.pymodname) for f in pwdf()['analyses'] if f.name != '__pycache__']
    ans = listmap(lambda a: a(mode), _ANALYSES)
    for a in ans:
        a.check_shadow()
    return ans
Esempio n. 18
0
 def all_output(self):
     return '\n'.join(listmap(utf_decode, self.readlines()))