Esempio n. 1
0
    def plot(self, pylab, axis, space, value, params={}):
        params0 = dict(color_shadow=[1.0, 0.8, 0.8], markers="k.", markers_params={})
        params0.update(params)

        color_shadow = params0.pop("color_shadow")
        markers = params0.pop("markers")
        markers_params = params0.pop("markers_params")
        if params0:
            raise ValueError(params0)

        self.axis = axis
        self.check_plot_space(space)

        minimals = [self._get_screen_coords(_, axis) for _ in value.minimals]
        print "minimals", minimals
        R2 = PosetProduct((Rcomp(), Rcomp()))
        v = R2.Us(minimals)

        from mcdp_report.generic_report_utils import extra_space_finite

        mcdp_dev_warning("incomplete PlotterUR")
        logger.error("todo: change this to plot a line")
        plot_upset_R2(
            pylab,
            v,
            axis,
            extra_space_shadow=extra_space_finite,
            color_shadow=color_shadow,
            markers=markers,
            marker_params=markers_params,
        )
Esempio n. 2
0
    def _load_hooks(self, load_arg, hooks, expected):
        errors = []
        if not hooks:
            msg = 'Could not load %r because no loading hooks provided.' % load_arg
            raise_desc(DPSemanticError, msg)
        for hook in hooks:
            try:
                try:
                    res = hook(load_arg, context=self)
                    if not isinstance(res, expected):
                        msg = 'The hook did not return the expected type.'
                        raise_desc(DPSemanticError, msg, res=res, expected=expected)
                    return res
                except TypeError:
                    msg = 'Could not use hook %r' % hook
                    logger.error(msg)
                    raise
            except DPSemanticError as e:
                if len(hooks) == 1:
                    raise
                else:
                    errors.append(e)

        s = "\n\n".join(map(str, errors))
        msg = 'Could not load %r: \n%s' % (load_arg, s)
        raise DPSemanticError(msg)
Esempio n. 3
0
    def png_error_catch2(self, request, func):
        """ func is supposed to return an image response.
            If it raises an exception, we create
            an image with the error and then we add the exception
            to the list of exceptions.
            
             """
        try:
            return func()
        except Exception as e:
            import traceback
            s = traceback.format_exc(e)

            try:
                logger.error(s)
            except UnicodeEncodeError:
                pass

            s = str(s)
            url = request.url
            referrer = request.referrer
            n = 'Error during rendering an image.'
            n+= '\n url: %s' % url
            n+= '\n referrer: %s' % referrer
            n += '\n' + indent(s, '| ')
            self.exceptions.append(n)
            from mcdp_web.utils.image_error_catch_imp import response_image
            return response_image(request, s)
Esempio n. 4
0
    def view_exception(self, exc, request):
        request.response.status = 500  # Internal Server Error

        import traceback
        compact = (DPSemanticError, DPSyntaxError)
        if isinstance(exc, compact):
            s = exc.__str__()
        else:
            s = traceback.format_exc(exc)

        url = request.url
        referrer = request.referrer
        n = 'Error during serving this URL:'
        n += '\n url: %s' % url
        n += '\n referrer: %s' % referrer
        ss = traceback.format_exc(exc)
        n += '\n' + indent(ss, '| ')
        self.exceptions.append(n)

        if 'library' in request.matchdict:
            library = self.get_current_library_name(request)
            url_refresh = '/libraries/%s/refresh_library' % library
        else:
            url_refresh = None

        u = unicode(s, 'utf-8')
        logger.error(u)
        return {'exception': u, 'url_refresh': url_refresh}
Esempio n. 5
0
def create_composite(gdc0, ndp, plotting_info):
    try:
        SKIP_INITIAL = gdc0.skip_initial
        #print('Skip initial: %s' % SKIP_INITIAL)
        return create_composite_(gdc0, ndp, plotting_info=plotting_info, SKIP_INITIAL=SKIP_INITIAL)
    except Exception as e:
        logger.error(e)
        logger.error('I will try again without the SKIP_INITIAL parameter.')
        return create_composite_(gdc0, ndp, plotting_info=plotting_info, SKIP_INITIAL=False)
Esempio n. 6
0
def check_syntax(filename, source):  # @UnusedVariable
    # print filename
    source = open(filename).read()
    try:
        _html = ast_to_html(source,
                            parse_expr=Syntax.ndpt_dp_rvalue,
                           complete_document=False, extra_css="",
                           ignore_line=lambda _lineno: False,
                           add_line_gutter=True, encapsulate_in_precode=True, 
                           add_css=False)
    except:
        logger.error('This happened to %r' %  filename)
        raise
Esempio n. 7
0
def safe_pickle_dump(value, filename, protocol=pickle.HIGHEST_PROTOCOL,
                     **safe_write_options):
    with safe_write(filename, **safe_write_options) as f:
        try:
            pickle.dump(value, f, protocol)
        except KeyboardInterrupt:
            raise
        except Exception:
            msg = 'Cannot pickle object of class %s' % describe_type(value)
            logger.error(msg)
            msg = find_pickling_error(value, protocol)
            logger.error(msg)
            raise
Esempio n. 8
0
def format_exception_for_ajax_response(e, quiet=()):
    s = e.__repr__().decode('ascii', 'ignore')
    from mocdp import logger
    try:
        logger.error(s)
    except UnicodeEncodeError:
        pass
    res = {}
    res['ok'] = False
    if isinstance(e, quiet):
        s = type(e).__name__ + ': ' + str(e)
    else:
        s = traceback.format_exc(e)
    res['error'] = cgi.escape(s)
    return res
Esempio n. 9
0
    def _parse_with_hooks(self, parse_ndp_like, string, realpath, context):

        with self._sys_path_adjust():
            context_mine = self._generate_context_with_hooks()
            try:
                result = parse_ndp_like(string, context=context_mine)
                from mcdp_lang.eval_warnings import warnings_copy_from_child_add_filename

                if context is not None:
                    warnings_copy_from_child_add_filename(context, context_mine, realpath)

                return result
            except MCDPExceptionWithWhere as e:
                logger.error("extend_with_filename(%r): seen %s" % (realpath, e))
                _type, _value, traceback = sys.exc_info()
                if e.where is None or e.where.filename is None:
                    if realpath is not None:
                        e = e.with_filename(realpath)
                    else:
                        e = e
                raise e, None, traceback
Esempio n. 10
0
def create_movie_from_png_sequence(sequence, out, fps=1.0):
    """ Creates an MP4 out of the list of png data """
    
    safe_makedirs(os.path.dirname(out))

    tmpdir = mkdtemp()
    for i, frame in enumerate(sequence):
    
        fn = os.path.join(tmpdir, '%05d.png' % i)
        with open(fn, 'w') as fi:
            fi.write(frame)

    try:
        import procgraph_mplayer  # @UnusedImport

    except ImportError as e:
        logger.error('Cannot use Procgraph to create video.')
        logger.error(e)
        logger.info('The frames are in the directory %s' % tmpdir)
    else:
        join_video_29_fixed(output=out, dirname=tmpdir,
                            pattern='.*.png', fps=fps)
Esempio n. 11
0
def graphviz_run(filename_dot, output, prog='dot'):
    suff = os.path.splitext(output)[1][1:]
    if not suff in ['png', 'pdf', 'ps', 'svg']:
        raise ValueError((output, suff))

    encoder = suff

    cmd = [prog, '-T%s' % encoder, '-o', output, filename_dot]
    try:
        # print('running graphviz')
        system_cmd_result(cwd='.', cmd=cmd,
                 display_stdout=False,
                 display_stderr=False,
                 raise_on_error=True)
        # print('done')
    except (CmdException, KeyboardInterrupt):
        emergency = 'emergency.dot'
        logger.error('saving to %r' % emergency)  # XXX
        contents = open(filename_dot).read()
        with open(emergency, 'w') as f:
            f.write(contents)
#         print(contents)
        raise
Esempio n. 12
0
def memo_disk_cache2(cache_file, data, f):
    """ 
        
        
    """
    from mocdp import logger

    dirname = os.path.dirname(cache_file)
    cachedir = os.path.join(dirname)
    if not os.path.exists(cachedir):
        try:
            os.makedirs(cachedir)
        except:
            if os.path.exists(cachedir):
                pass
            else:
                raise

    if os.path.exists(cache_file):
        # logger.info('Reading from cache %r.' % cache_file)
        try:
            res = safe_pickle_load(cache_file)
            if data != res["data"]:
                logger.info("Stale cache, recomputing.")
            else:
                return res["result"]
        except Exception as e:
            logger.error(e)

    result = f()

    if MCDPConstants.log_cache_writes:
        logger.info("Writing to cache %s." % cache_file)
    res = dict(data=data, result=result)
    safe_pickle_dump(res, cache_file)

    return result
Esempio n. 13
0
def do_plots(logger, model_name, plots, outdir, extra_params, 
             maindir, extra_dirs, use_cache):
    data = {}

    if '.mcdp' in model_name:
        model_name2 = model_name.replace('.mcdp', '')
        msg = 'Arguments should be model names, not file names.'
        msg += ' Interpreting %r as %r.' % (model_name, model_name2)
        logger.warn(msg)
        model_name = model_name2

    data['model_name'] = model_name

    if use_cache:
        cache_dir = os.path.join(outdir, '_cached/mcdp_plot_cache')
        logger.info('using cache %s' % cache_dir)
    else:
        cache_dir = None

    librarian = Librarian()
    for e in extra_dirs:
        librarian.find_libraries(e)

    library = librarian.get_library_by_dir(maindir)
    if cache_dir is not None:
        library.use_cache_dir(cache_dir)

    assert library.library_name is not None

    filename = model_name + '.mcdp'
    x = library._get_file_data(filename)
    data['s'] = x['data']
    data['filename'] = x['realpath']
    data['params'] = parse_params(extra_params)
    data['library'] = library

    d = dict(allplots)
    results = []
    for p in plots:    
        # print('plotting %r ' % p)
        try:
            if p in d:
                res = d[p](data)
            else:
                msg = 'Unknown plot.'
                raise_desc(ValueError, msg, plot=p, available=sorted(d.keys()))
        except CmdException as e:
            mcdp_dev_warning('Add better checks of error.')
            logger.error(e)
            continue
        except Exception as e:
            logger.error('While creating %r' % p)
            raise
        assert isinstance(res, list), res
        for r in res:
            assert isinstance(r, tuple), r
            mime, name, x = r
            assert isinstance(x, str), x
            ext = mime

            base = model_name + '-%s.%s' % (name, ext)

            out = os.path.join(outdir, base)
            logger.info('Writing to %s' % out)
            with open(out, 'w') as f:
                f.write(x)

            results.append(r)

    return results
Esempio n. 14
0
def refine(x, parents,
           constants, resources, functions, variables, deriv_resources,
           deriv_functions, context):
    
#     print tuple((type(x).__name__, y) for x, y in parents)
    
    is_rvalue_context = any(k == 'rvalue' for _, k in parents)
    is_fvalue_context = any(k == 'fvalue' for _, k in parents)
    
    assert not (is_rvalue_context and is_fvalue_context)
    
    if isinstance(x, CDP.VariableRef):
        if x.name in constants:
            cname = CDP.CName(x.name, where=x.where)
            res = CDP.ConstantRef(cname=cname, where=x.where)
            return res
        elif x.name in resources and x.name in functions:
            if is_fvalue_context:
                msg = 'Please use "required %s" rather than just "%s".' % (x.name, x.name)
                warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context)

                # interpret as 
                return CDP.NewResource(None, CDP.RName(x.name, where=x.where), 
                                   where=x.where)
            if is_rvalue_context:
                msg = 'Please use "provided %s" rather than just "%s".' % (x.name, x.name)
                warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context)

                return CDP.NewFunction(None, CDP.FName(x.name, where=x.where), 
                                   where=x.where)
            msg = 'I cannot say whether %r refers to the functionality or resource.' % x.name
            msg += ' Need to implement >= - aware refinement.'
            warn_language(x, MCDPWarnings.LANGUAGE_AMBIGUOS_EXPRESSION, msg, context)
            return x
        elif x.name in resources:
            if x.name in variables:
                msg = 'I cannot say whether %r refers to the variable or resource.' % x.name
                warn_language(x, MCDPWarnings.LANGUAGE_AMBIGUOS_EXPRESSION, msg, context)
                return x 
            
            msg = 'Please use "required %s" rather than just "%s".' % (x.name, x.name)
            warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context)

            return CDP.NewResource(None, CDP.RName(x.name, where=x.where), 
                                   where=x.where)

        elif x.name in functions:
            if x.name in variables:
                msg = 'I cannot say whether %r refers to the variable or functionality.' % x.name
                warn_language(x, MCDPWarnings.LANGUAGE_AMBIGUOS_EXPRESSION, msg, context) # XXX
                return x
            
            msg = 'Please use "provided %s" rather than just "%s".' % (x.name, x.name)
            warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context) # XXX

            return CDP.NewFunction(None, CDP.FName(x.name, where=x.where), 
                                   where=x.where) 

        elif x.name in deriv_resources:
            where = x.where
            drname = CDP.DerivResourceName(x.name, where=where)
            res = CDP.DerivResourceRef(drname, where=where)
            return res

        elif x.name in deriv_functions:
            # XXX: this is not correct
            where = x.where
            dfname = CDP.DerivFunctionName(x.name, where=where)
            res = CDP.DerivFunctionRef(dfname, where=where)
            return res
        
        elif x.name in variables:
            where = x.where
            dfname = CDP.VName(x.name, where=where)
            res = CDP.ActualVarRef(dfname, where=where)
            return res
        else:
            msg = 'I cannot judge this VariableRef: %r' % str(x)
            logger.error(msg)
            # raise DPInternalError(msg)
   
    if isinstance(x, CDP.SetNameGenericVar):
        if x.value in constants:
            return CDP.CName(x.value, where=x.where)

        elif x.value in deriv_resources:
            # XXX: this is not correct
            return CDP.RName(x.value, where=x.where)

        elif x.value in deriv_functions:
            # XXX: this is not correct
            return CDP.FName(x.value, where=x.where)
        else:
            
            logger.error('I cannot judge this SetNameGenericVar: %r' % str(x))
    return x
Esempio n. 15
0
    def go(selector, parse_expr, extension, use_pre=True):
        for tag in soup.select(selector):
            try:
                if tag.string is None:
                    if not tag.has_attr('id'):
                        msg = "If <pre> is empty then it needs to have an id."
                        raise_desc(ValueError, msg, tag=str(tag))
                    # load it
                    tag_id = tag['id'].encode('utf-8')
                    basename = '%s.%s' % (tag_id, extension)
                    data = library._get_file_data(basename)
                    source_code = data['data']
                    source_code = source_code.replace('\t', ' ' * 4)
                else:
                    source_code = get_source_code(tag)

                # we are not using it
                _realpath = realpath
                html = ast_to_html(source_code, parse_expr=parse_expr,
                                                complete_document=False,
                                                add_line_gutter=False,
                                                add_css=False)

                frag2 = BeautifulSoup(html, 'lxml', from_encoding='utf-8')

                if use_pre:
                    rendered = frag2.pre
                    if tag.has_attr('label'):
                        tag_label = soup.new_tag('span', **{'class': 'label'})
                        tag_label.append(tag['label'])
                        rendered.insert(0, tag_label)

                    max_len = max(map(len, source_code.split('\n')))
                    # account for the label
                    if tag.has_attr('label'):
                        max_len = max(max_len, len(tag['label']) + 6)

                    # need at least 1 to account for padding etc.
                    bonus = 1
                    style = 'width: %dch;' % (max_len + bonus)
                else:
                    # using <code>
                    rendered = frag2.pre.code
                    style = ''

                if tag.has_attr('style'):
                    style = style + tag['style'] 
                    
                if style:
                    rendered['style'] = style

                if tag.has_attr('class'):
                    rendered['class'] = tag['class']

                if tag.has_attr('id'):
                    rendered['id'] = tag['id']

                if use_pre:
                    if generate_pdf:
                        pdf = get_ast_as_pdf(source_code, parse_expr)
                        if tag.has_attr('id'):
                            basename = tag['id']
                        else:
                            hashcode = hashlib.sha224(source_code).hexdigest()[-8:]
                            basename = 'code-%s' % (hashcode)

                        docname = os.path.splitext(os.path.basename(realpath))[0]
                        download = docname + '.' + basename + '.source_code.pdf'
                        a = create_a_to_data(soup, download=download,
                                             data_format='pdf', data=pdf)
                        a['class'] = 'pdf_data'
                        a.append(NavigableString(download))
                        div = soup.new_tag('div')
                        div.append(rendered)
                        div.append(a)
                        tag.replaceWith(div)
                    else:
                        tag.replaceWith(rendered)
                else:
                    tag.replaceWith(rendered)

            except DPSyntaxError as e:
                if raise_errors:
                    raise
                logger.error(unicode(e.__str__(), 'utf-8'))
                t = soup.new_tag('pre', **{'class': 'error %s' % type(e).__name__})
                t.string = str(e)
                tag.insert_after(t)

                if tag.string is None:
                    tag.string = "`%s" % tag['id']

            except DPSemanticError as e:
                if raise_errors:
                    raise
                logger.error(unicode(e.__str__(), 'utf-8'))
                t = soup.new_tag('pre', **{'class': 'error %s' % type(e).__name__})
                t.string = str(e)
                tag.insert_after(t)
                if tag.string is None:
                    tag.string = "`%s" % tag['id']
                    
            except DPInternalError as e:
                msg = 'Error while interpreting the code:\n\n'
                msg += indent(source_code, '  | ')
                raise_wrapped(DPInternalError, e,msg, exc=sys.exc_info())
Esempio n. 16
0
def gg_figure(r, name, ggraph, do_png=True, do_pdf=True, do_svg=True,
              do_dot=True):
    """ Adds a figure to the Report r that displays this graph
        and also its source. """
    f = r.figure(name, cols=1)

    # save file in dot file
    with tmpfile(".dot") as filename_dot:
        with open(filename_dot, 'w') as fo:
            s = get_dot_string(ggraph)
            fo.write(s)

#         if False:
#             ff = '%s.dot' % id(r)
#             print('writing to %r' % ff)
#             with open(ff, 'w') as f2:
#                 f2.write(s)

        prog = 'dot'
        try:
                
            if do_png:
                with f.data_file('graph', MIME_PNG) as filename:
                    graphviz_run(filename_dot, filename, prog=prog)
    
            if do_pdf:
                with f.data_file('graph_pdf', MIME_PDF) as filename:
                    graphviz_run(filename_dot, filename, prog=prog)
    
            if do_svg:
                with f.data_file('graph_svg', MIME_SVG) as filename:
                    graphviz_run(filename_dot, filename, prog=prog)
    
                    soup = BeautifulSoup(open(filename).read(), 'lxml', from_encoding='utf-8')
                    for tag in soup.select('image'):
                        href = tag['xlink:href']
                        extensions = ['png', 'jpg']
                        for ext in extensions:
                            if ext in href:
                                with open(href) as ff:
                                    png = ff.read()
                                encoded = base64.b64encode(png)
                                from mcdp_web.images.images import get_mime_for_format
                                mime = get_mime_for_format(ext)
                                src = 'data:%s;base64,%s' % (mime, encoded)
                                tag['xlink:href'] = src
    
                    with codecs.open(filename, 'w', encoding='utf-8') as ff:
                        s = str(soup)
                        u = unicode(s, 'utf-8')
                        ff.write(u)
        except CmdException:
            if MCDPConstants.test_ignore_graphviz_errors:
                mcdp_dev_warning('suppressing errors from graphviz')
                logger.error('Graphivz failed, but I will ignore it '
                             'because of MCDPConstants.test_ignore_graphviz_errors.')
            else:
                raise

        # MIME_GRAPHVIZ
        if do_dot:
            with f.data_file('dot', MIME_PLAIN) as filename:
                with open(filename, 'w') as f:
                    f.write(s)
        
    return f