def plot(self, pylab, axis, space, value, params={}):
params0 = dict(color_shadow=[1.0, 0.8, 0.8], markers="k.", markers_params={})
params0.update(params)
color_shadow = params0.pop("color_shadow")
markers = params0.pop("markers")
markers_params = params0.pop("markers_params")
if params0:
raise ValueError(params0)
self.axis = axis
self.check_plot_space(space)
minimals = [self._get_screen_coords(_, axis) for _ in value.minimals]
print "minimals", minimals
R2 = PosetProduct((Rcomp(), Rcomp()))
v = R2.Us(minimals)
from mcdp_report.generic_report_utils import extra_space_finite
mcdp_dev_warning("incomplete PlotterUR")
logger.error("todo: change this to plot a line")
plot_upset_R2(
pylab,
v,
axis,
extra_space_shadow=extra_space_finite,
color_shadow=color_shadow,
markers=markers,
marker_params=markers_params,
)
def _load_hooks(self, load_arg, hooks, expected):
errors = []
if not hooks:
msg = 'Could not load %r because no loading hooks provided.' % load_arg
raise_desc(DPSemanticError, msg)
for hook in hooks:
try:
try:
res = hook(load_arg, context=self)
if not isinstance(res, expected):
msg = 'The hook did not return the expected type.'
raise_desc(DPSemanticError, msg, res=res, expected=expected)
return res
except TypeError:
msg = 'Could not use hook %r' % hook
logger.error(msg)
raise
except DPSemanticError as e:
if len(hooks) == 1:
raise
else:
errors.append(e)
s = "\n\n".join(map(str, errors))
msg = 'Could not load %r: \n%s' % (load_arg, s)
raise DPSemanticError(msg)
def png_error_catch2(self, request, func):
""" func is supposed to return an image response.
If it raises an exception, we create
an image with the error and then we add the exception
to the list of exceptions.
"""
try:
return func()
except Exception as e:
import traceback
s = traceback.format_exc(e)
try:
logger.error(s)
except UnicodeEncodeError:
pass
s = str(s)
url = request.url
referrer = request.referrer
n = 'Error during rendering an image.'
n+= '\n url: %s' % url
n+= '\n referrer: %s' % referrer
n += '\n' + indent(s, '| ')
self.exceptions.append(n)
from mcdp_web.utils.image_error_catch_imp import response_image
return response_image(request, s)
def view_exception(self, exc, request):
request.response.status = 500 # Internal Server Error
import traceback
compact = (DPSemanticError, DPSyntaxError)
if isinstance(exc, compact):
s = exc.__str__()
else:
s = traceback.format_exc(exc)
url = request.url
referrer = request.referrer
n = 'Error during serving this URL:'
n += '\n url: %s' % url
n += '\n referrer: %s' % referrer
ss = traceback.format_exc(exc)
n += '\n' + indent(ss, '| ')
self.exceptions.append(n)
if 'library' in request.matchdict:
library = self.get_current_library_name(request)
url_refresh = '/libraries/%s/refresh_library' % library
else:
url_refresh = None
u = unicode(s, 'utf-8')
logger.error(u)
return {'exception': u, 'url_refresh': url_refresh}
def create_composite(gdc0, ndp, plotting_info):
try:
SKIP_INITIAL = gdc0.skip_initial
#print('Skip initial: %s' % SKIP_INITIAL)
return create_composite_(gdc0, ndp, plotting_info=plotting_info, SKIP_INITIAL=SKIP_INITIAL)
except Exception as e:
logger.error(e)
logger.error('I will try again without the SKIP_INITIAL parameter.')
return create_composite_(gdc0, ndp, plotting_info=plotting_info, SKIP_INITIAL=False)
def check_syntax(filename, source): # @UnusedVariable
# print filename
source = open(filename).read()
try:
_html = ast_to_html(source,
parse_expr=Syntax.ndpt_dp_rvalue,
complete_document=False, extra_css="",
ignore_line=lambda _lineno: False,
add_line_gutter=True, encapsulate_in_precode=True,
add_css=False)
except:
logger.error('This happened to %r' % filename)
raise
def safe_pickle_dump(value, filename, protocol=pickle.HIGHEST_PROTOCOL,
**safe_write_options):
with safe_write(filename, **safe_write_options) as f:
try:
pickle.dump(value, f, protocol)
except KeyboardInterrupt:
raise
except Exception:
msg = 'Cannot pickle object of class %s' % describe_type(value)
logger.error(msg)
msg = find_pickling_error(value, protocol)
logger.error(msg)
raise
def format_exception_for_ajax_response(e, quiet=()):
s = e.__repr__().decode('ascii', 'ignore')
from mocdp import logger
try:
logger.error(s)
except UnicodeEncodeError:
pass
res = {}
res['ok'] = False
if isinstance(e, quiet):
s = type(e).__name__ + ': ' + str(e)
else:
s = traceback.format_exc(e)
res['error'] = cgi.escape(s)
return res
def _parse_with_hooks(self, parse_ndp_like, string, realpath, context):
with self._sys_path_adjust():
context_mine = self._generate_context_with_hooks()
try:
result = parse_ndp_like(string, context=context_mine)
from mcdp_lang.eval_warnings import warnings_copy_from_child_add_filename
if context is not None:
warnings_copy_from_child_add_filename(context, context_mine, realpath)
return result
except MCDPExceptionWithWhere as e:
logger.error("extend_with_filename(%r): seen %s" % (realpath, e))
_type, _value, traceback = sys.exc_info()
if e.where is None or e.where.filename is None:
if realpath is not None:
e = e.with_filename(realpath)
else:
e = e
raise e, None, traceback
def create_movie_from_png_sequence(sequence, out, fps=1.0):
""" Creates an MP4 out of the list of png data """
safe_makedirs(os.path.dirname(out))
tmpdir = mkdtemp()
for i, frame in enumerate(sequence):
fn = os.path.join(tmpdir, '%05d.png' % i)
with open(fn, 'w') as fi:
fi.write(frame)
try:
import procgraph_mplayer # @UnusedImport
except ImportError as e:
logger.error('Cannot use Procgraph to create video.')
logger.error(e)
logger.info('The frames are in the directory %s' % tmpdir)
else:
join_video_29_fixed(output=out, dirname=tmpdir,
pattern='.*.png', fps=fps)
def graphviz_run(filename_dot, output, prog='dot'):
suff = os.path.splitext(output)[1][1:]
if not suff in ['png', 'pdf', 'ps', 'svg']:
raise ValueError((output, suff))
encoder = suff
cmd = [prog, '-T%s' % encoder, '-o', output, filename_dot]
try:
# print('running graphviz')
system_cmd_result(cwd='.', cmd=cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True)
# print('done')
except (CmdException, KeyboardInterrupt):
emergency = 'emergency.dot'
logger.error('saving to %r' % emergency) # XXX
contents = open(filename_dot).read()
with open(emergency, 'w') as f:
f.write(contents)
# print(contents)
raise
def memo_disk_cache2(cache_file, data, f):
"""
"""
from mocdp import logger
dirname = os.path.dirname(cache_file)
cachedir = os.path.join(dirname)
if not os.path.exists(cachedir):
try:
os.makedirs(cachedir)
except:
if os.path.exists(cachedir):
pass
else:
raise
if os.path.exists(cache_file):
# logger.info('Reading from cache %r.' % cache_file)
try:
res = safe_pickle_load(cache_file)
if data != res["data"]:
logger.info("Stale cache, recomputing.")
else:
return res["result"]
except Exception as e:
logger.error(e)
result = f()
if MCDPConstants.log_cache_writes:
logger.info("Writing to cache %s." % cache_file)
res = dict(data=data, result=result)
safe_pickle_dump(res, cache_file)
return result
def do_plots(logger, model_name, plots, outdir, extra_params,
maindir, extra_dirs, use_cache):
data = {}
if '.mcdp' in model_name:
model_name2 = model_name.replace('.mcdp', '')
msg = 'Arguments should be model names, not file names.'
msg += ' Interpreting %r as %r.' % (model_name, model_name2)
logger.warn(msg)
model_name = model_name2
data['model_name'] = model_name
if use_cache:
cache_dir = os.path.join(outdir, '_cached/mcdp_plot_cache')
logger.info('using cache %s' % cache_dir)
else:
cache_dir = None
librarian = Librarian()
for e in extra_dirs:
librarian.find_libraries(e)
library = librarian.get_library_by_dir(maindir)
if cache_dir is not None:
library.use_cache_dir(cache_dir)
assert library.library_name is not None
filename = model_name + '.mcdp'
x = library._get_file_data(filename)
data['s'] = x['data']
data['filename'] = x['realpath']
data['params'] = parse_params(extra_params)
data['library'] = library
d = dict(allplots)
results = []
for p in plots:
# print('plotting %r ' % p)
try:
if p in d:
res = d[p](data)
else:
msg = 'Unknown plot.'
raise_desc(ValueError, msg, plot=p, available=sorted(d.keys()))
except CmdException as e:
mcdp_dev_warning('Add better checks of error.')
logger.error(e)
continue
except Exception as e:
logger.error('While creating %r' % p)
raise
assert isinstance(res, list), res
for r in res:
assert isinstance(r, tuple), r
mime, name, x = r
assert isinstance(x, str), x
ext = mime
base = model_name + '-%s.%s' % (name, ext)
out = os.path.join(outdir, base)
logger.info('Writing to %s' % out)
with open(out, 'w') as f:
f.write(x)
results.append(r)
return results
def refine(x, parents,
constants, resources, functions, variables, deriv_resources,
deriv_functions, context):
# print tuple((type(x).__name__, y) for x, y in parents)
is_rvalue_context = any(k == 'rvalue' for _, k in parents)
is_fvalue_context = any(k == 'fvalue' for _, k in parents)
assert not (is_rvalue_context and is_fvalue_context)
if isinstance(x, CDP.VariableRef):
if x.name in constants:
cname = CDP.CName(x.name, where=x.where)
res = CDP.ConstantRef(cname=cname, where=x.where)
return res
elif x.name in resources and x.name in functions:
if is_fvalue_context:
msg = 'Please use "required %s" rather than just "%s".' % (x.name, x.name)
warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context)
# interpret as
return CDP.NewResource(None, CDP.RName(x.name, where=x.where),
where=x.where)
if is_rvalue_context:
msg = 'Please use "provided %s" rather than just "%s".' % (x.name, x.name)
warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context)
return CDP.NewFunction(None, CDP.FName(x.name, where=x.where),
where=x.where)
msg = 'I cannot say whether %r refers to the functionality or resource.' % x.name
msg += ' Need to implement >= - aware refinement.'
warn_language(x, MCDPWarnings.LANGUAGE_AMBIGUOS_EXPRESSION, msg, context)
return x
elif x.name in resources:
if x.name in variables:
msg = 'I cannot say whether %r refers to the variable or resource.' % x.name
warn_language(x, MCDPWarnings.LANGUAGE_AMBIGUOS_EXPRESSION, msg, context)
return x
msg = 'Please use "required %s" rather than just "%s".' % (x.name, x.name)
warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context)
return CDP.NewResource(None, CDP.RName(x.name, where=x.where),
where=x.where)
elif x.name in functions:
if x.name in variables:
msg = 'I cannot say whether %r refers to the variable or functionality.' % x.name
warn_language(x, MCDPWarnings.LANGUAGE_AMBIGUOS_EXPRESSION, msg, context) # XXX
return x
msg = 'Please use "provided %s" rather than just "%s".' % (x.name, x.name)
warn_language(x, MCDPWarnings.LANGUAGE_REFERENCE_OK_BUT_IMPRECISE, msg, context) # XXX
return CDP.NewFunction(None, CDP.FName(x.name, where=x.where),
where=x.where)
elif x.name in deriv_resources:
where = x.where
drname = CDP.DerivResourceName(x.name, where=where)
res = CDP.DerivResourceRef(drname, where=where)
return res
elif x.name in deriv_functions:
# XXX: this is not correct
where = x.where
dfname = CDP.DerivFunctionName(x.name, where=where)
res = CDP.DerivFunctionRef(dfname, where=where)
return res
elif x.name in variables:
where = x.where
dfname = CDP.VName(x.name, where=where)
res = CDP.ActualVarRef(dfname, where=where)
return res
else:
msg = 'I cannot judge this VariableRef: %r' % str(x)
logger.error(msg)
# raise DPInternalError(msg)
if isinstance(x, CDP.SetNameGenericVar):
if x.value in constants:
return CDP.CName(x.value, where=x.where)
elif x.value in deriv_resources:
# XXX: this is not correct
return CDP.RName(x.value, where=x.where)
elif x.value in deriv_functions:
# XXX: this is not correct
return CDP.FName(x.value, where=x.where)
else:
logger.error('I cannot judge this SetNameGenericVar: %r' % str(x))
return x
def go(selector, parse_expr, extension, use_pre=True):
for tag in soup.select(selector):
try:
if tag.string is None:
if not tag.has_attr('id'):
msg = "If <pre> is empty then it needs to have an id."
raise_desc(ValueError, msg, tag=str(tag))
# load it
tag_id = tag['id'].encode('utf-8')
basename = '%s.%s' % (tag_id, extension)
data = library._get_file_data(basename)
source_code = data['data']
source_code = source_code.replace('\t', ' ' * 4)
else:
source_code = get_source_code(tag)
# we are not using it
_realpath = realpath
html = ast_to_html(source_code, parse_expr=parse_expr,
complete_document=False,
add_line_gutter=False,
add_css=False)
frag2 = BeautifulSoup(html, 'lxml', from_encoding='utf-8')
if use_pre:
rendered = frag2.pre
if tag.has_attr('label'):
tag_label = soup.new_tag('span', **{'class': 'label'})
tag_label.append(tag['label'])
rendered.insert(0, tag_label)
max_len = max(map(len, source_code.split('\n')))
# account for the label
if tag.has_attr('label'):
max_len = max(max_len, len(tag['label']) + 6)
# need at least 1 to account for padding etc.
bonus = 1
style = 'width: %dch;' % (max_len + bonus)
else:
# using <code>
rendered = frag2.pre.code
style = ''
if tag.has_attr('style'):
style = style + tag['style']
if style:
rendered['style'] = style
if tag.has_attr('class'):
rendered['class'] = tag['class']
if tag.has_attr('id'):
rendered['id'] = tag['id']
if use_pre:
if generate_pdf:
pdf = get_ast_as_pdf(source_code, parse_expr)
if tag.has_attr('id'):
basename = tag['id']
else:
hashcode = hashlib.sha224(source_code).hexdigest()[-8:]
basename = 'code-%s' % (hashcode)
docname = os.path.splitext(os.path.basename(realpath))[0]
download = docname + '.' + basename + '.source_code.pdf'
a = create_a_to_data(soup, download=download,
data_format='pdf', data=pdf)
a['class'] = 'pdf_data'
a.append(NavigableString(download))
div = soup.new_tag('div')
div.append(rendered)
div.append(a)
tag.replaceWith(div)
else:
tag.replaceWith(rendered)
else:
tag.replaceWith(rendered)
except DPSyntaxError as e:
if raise_errors:
raise
logger.error(unicode(e.__str__(), 'utf-8'))
t = soup.new_tag('pre', **{'class': 'error %s' % type(e).__name__})
t.string = str(e)
tag.insert_after(t)
if tag.string is None:
tag.string = "`%s" % tag['id']
except DPSemanticError as e:
if raise_errors:
raise
logger.error(unicode(e.__str__(), 'utf-8'))
t = soup.new_tag('pre', **{'class': 'error %s' % type(e).__name__})
t.string = str(e)
tag.insert_after(t)
if tag.string is None:
tag.string = "`%s" % tag['id']
except DPInternalError as e:
msg = 'Error while interpreting the code:\n\n'
msg += indent(source_code, ' | ')
raise_wrapped(DPInternalError, e,msg, exc=sys.exc_info())
def gg_figure(r, name, ggraph, do_png=True, do_pdf=True, do_svg=True,
do_dot=True):
""" Adds a figure to the Report r that displays this graph
and also its source. """
f = r.figure(name, cols=1)
# save file in dot file
with tmpfile(".dot") as filename_dot:
with open(filename_dot, 'w') as fo:
s = get_dot_string(ggraph)
fo.write(s)
# if False:
# ff = '%s.dot' % id(r)
# print('writing to %r' % ff)
# with open(ff, 'w') as f2:
# f2.write(s)
prog = 'dot'
try:
if do_png:
with f.data_file('graph', MIME_PNG) as filename:
graphviz_run(filename_dot, filename, prog=prog)
if do_pdf:
with f.data_file('graph_pdf', MIME_PDF) as filename:
graphviz_run(filename_dot, filename, prog=prog)
if do_svg:
with f.data_file('graph_svg', MIME_SVG) as filename:
graphviz_run(filename_dot, filename, prog=prog)
soup = BeautifulSoup(open(filename).read(), 'lxml', from_encoding='utf-8')
for tag in soup.select('image'):
href = tag['xlink:href']
extensions = ['png', 'jpg']
for ext in extensions:
if ext in href:
with open(href) as ff:
png = ff.read()
encoded = base64.b64encode(png)
from mcdp_web.images.images import get_mime_for_format
mime = get_mime_for_format(ext)
src = 'data:%s;base64,%s' % (mime, encoded)
tag['xlink:href'] = src
with codecs.open(filename, 'w', encoding='utf-8') as ff:
s = str(soup)
u = unicode(s, 'utf-8')
ff.write(u)
except CmdException:
if MCDPConstants.test_ignore_graphviz_errors:
mcdp_dev_warning('suppressing errors from graphviz')
logger.error('Graphivz failed, but I will ignore it '
'because of MCDPConstants.test_ignore_graphviz_errors.')
else:
raise
# MIME_GRAPHVIZ
if do_dot:
with f.data_file('dot', MIME_PLAIN) as filename:
with open(filename, 'w') as f:
f.write(s)
return f
