def get_user_struct(self, username=None):
'''
It is the user 'anonymous' if no login was given.
self.request.authenticated_userid == None == get_user().info.username == 'anonymous'
'''
app = self.app
user_db = app.hi.db_view.user_db
if username is None:
username = self.request.authenticated_userid
if username is not None and not username in user_db:
logger.error('User appears to have not existent id "%s".' %
username)
username = None
if username is not None:
username = username.encode('utf8')
if username in user_db:
return user_db.users[username]
else:
schema = DB.user
data = schema.generate_empty(
info=dict(username=username, name=username))
view = DB.view_manager.create_view_instance(schema, data)
view.set_root()
return view
else:
# username is None:
return user_db.users[MCDPConstants.USER_ANONYMOUS]
def gg_deepcopy(ggraph):
try:
return deepcopy(ggraph)
except Exception as e:
logger.error(traceback.format_exc(e))
mcdp_dev_warning('Deep copy of gvgen graph failed: happens when in IPython.')
return ggraph
def check_syntax(filename, source, parse_expr=Syntax.ndpt_dp_rvalue): # @UnusedVariable
# skip generated files (hack)
if filename and 'drone_unc2_' in filename:
return
# print filename
if filename is not None:
source = open(filename).read()
try:
html = ast_to_html(source,
parse_expr,
ignore_line=lambda _lineno: False,
add_line_gutter=False, encapsulate_in_precode=True,
)
# print html.__repr__()
source2 = project_html(html)
# print source
# print source2
# print indent(html, 'html |')
assert_equal_string(s2_expected=source, s2=source2)
except:
logger.error('This happened to %r' % filename)
raise
def resolve_to_realpath(href):
""" Returns the realpath of the resource"""
#print('resolving %r' % href)
if href.startswith('http'):
msg = 'I am not able to download external resources, such as:'
msg += '\n ' + href
logger.error(msg)
res.note_error(msg)
return None
if '/' in href:
href = os.path.basename(href)
try:
f = library._get_file_data(href)
except DPSemanticError as e:
msg = 'Could not find file %r.' % href
msg += '\n\n' + indent(e, ' > ')
# logger.error(msg)
# logger.error(str(e))
res.note_error(msg)
return None
realpath = f['realpath']
data = f['data']
check_not_lfs_pointer(realpath, data)
return f
def plot(self, pylab, axis, space, value, params={}):
params0 = dict(color_shadow=[1.0, 0.8, 0.8],
markers='k.',
markers_params={})
params0.update(params)
color_shadow = params0.pop('color_shadow')
markers = params0.pop('markers')
markers_params = params0.pop('markers_params')
if params0:
raise ValueError(params0)
self.axis = axis
self.check_plot_space(space)
minimals = [self._get_screen_coords(_, axis) for _ in value.minimals]
print 'minimals', minimals
R2 = PosetProduct((Rcomp(), Rcomp()))
v = R2.Us(minimals)
from mcdp_report.generic_report_utils import extra_space_finite
mcdp_dev_warning('incomplete PlotterUR')
logger.error('todo: change this to plot a line')
plot_upset_R2(pylab,
v,
axis,
extra_space_shadow=extra_space_finite,
color_shadow=color_shadow,
markers=markers,
marker_params=markers_params)
def _load_hooks(self, load_arg, hooks, expected):
errors = []
if not hooks:
msg = 'Could not load %r because no loading hooks provided.' % load_arg
raise_desc(DPSemanticError, msg)
for hook in hooks:
try:
try:
res = hook(load_arg, context=self)
if not isinstance(res, expected):
msg = 'The hook did not return the expected type.'
raise_desc(DPSemanticError,
msg,
res=res,
expected=expected)
return res
except TypeError:
msg = 'Could not use hook %r' % hook
logger.error(msg)
raise
except DPSemanticError as e:
if len(hooks) == 1:
raise
else:
errors.append(e)
s = "\n\n".join(map(str, errors))
msg = 'Could not load %r: \n%s' % (load_arg, s)
raise DPSemanticError(msg)
def do_plots_ndp(model_name, library, plots, extra_params):
possible = [p for p, _ in allplots]
plots = expand_string(plots, list(possible))
filename = model_name + '.mcdp'
x = library._get_file_data(filename)
data = {}
data['model_name'] = model_name
data['s'] = x['data']
data['filename'] = x['realpath']
data['params'] = parse_params(extra_params)
data['library'] = library
d = dict(allplots)
results = []
for p in plots:
try:
if p in d:
res = d[p](data)
else:
msg = 'Unknown plot.'
raise_desc(ValueError, msg, plot=p, available=sorted(d.keys()))
return # XXX
except CmdException as e:
mcdp_dev_warning('Add better checks of error.')
logger.error(e)
continue
except Exception as e:
logger.error('While creating %r' % p)
raise
assert isinstance(res, list), res
results.extend(res)
return results
def process_svg_files(root, out, preamble):
logger.info('root = %s' % root)
logger.info('out = %s' % out)
logger.info('preamble = %s' % preamble)
if not os.path.exists(out):
os.makedirs(out)
logger.info('Looking for *.svg files...')
svgs = locate_files(root, '*.svg')
logger.info('%d found in %s' % (len(svgs), friendly_path(root)))
errors = []
for f in svgs:
dirname = os.path.dirname(f)
basename, _ = os.path.splitext(os.path.basename(f))
target = os.path.join(out, basename + '.pdf')
target = os.path.join(dirname, basename + '.pdf')
if not needs_remake(f, target):
msg = 'The target %r is up to date.' % target
logger.info(msg)
else:
msg = 'Will build target %r.' % target
logger.info(msg)
tmpdir = create_tmpdir('svg-%s' % basename)
try:
process_svg_file(f, target, preamble, tmpdir)
except Exception as e:
logger.error(e)
errors.append(e)
def go():
image_source = image_source_from_env(e)
library = library_from_env(e)
with timeit_wall('graph_generic', 1.0):
key = (e.library_name, e.spec.url_part, e.thing_name,
text_hash)
if not key in self.last_processed2:
logger.error('Cannot find key %s' % str(key))
logger.error('keys: %s' % list(self.last_processed2))
context = e.library._generate_context_with_hooks()
thing = e.spec.load(e.library,
e.thing_name,
context=context)
else:
thing = self.last_processed2[key]
if thing is None:
return response_image(e.request, 'Could not parse.')
with timeit_wall('graph_generic - get_png_data', 1.0):
data = e.spec.get_png_data(image_source=image_source,
name=e.thing_name,
thing=thing,
data_format=data_format,
library=library)
mime = get_mime_for_format(data_format)
return response_data(e.request, data, mime)
def prerender_mathjax(s, symbols):
if symbols:
lines = symbols.split('\n')
lines = [l for l in lines if l.strip()]
m = '$$' + "\n".join(lines) +'$$\n\n'
else:
m = ''
STARTTAG = 'STARTHERE'
ENDTAG = 'ENDHERE'
s = STARTTAG + get_mathjax_preamble() + ENDTAG + m + s
try:
s = prerender_mathjax_(s)
except PrerenderError: # pragma: no cover
if 'CIRCLECI' in os.environ:
msg = 'Ignoring PrerenderError because of CircleCI.'
logger.error(msg)
return s
else:
raise
c0 = s.index(STARTTAG)
c1 = s.index(ENDTAG) + len(ENDTAG)
s = s[:c0] + s[c1:]
# s = fix_vertical_align(s)
return s
def view_exception(self, exc, request):
request.response.status = 500 # Internal Server Error
if hasattr(request, 'context'):
if isinstance(request.context, Resource):
logger.debug(context_display_in_detail(request.context))
compact = (DPSemanticError, DPSyntaxError)
if isinstance(exc, compact):
s = exc.__str__()
else:
s = traceback.format_exc(exc)
self.note_exception(exc, request=request)
u = unicode(s, 'utf-8')
logger.error(u.encode('utf8'))
root = self.get_root_relative_to_here(request)
res = {
'exception': u,
# 'url_refresh': url_refresh,
'root': root,
'static': root + '/static'
}
return res
def graphviz_run(filename_dot, output, prog='dot'):
suff = os.path.splitext(output)[1][1:]
if not suff in ['png', 'pdf', 'ps', 'svg']:
raise ValueError((output, suff))
encoder = suff
cmd = [prog, '-T%s' % encoder, '-o', output, filename_dot]
with timeit_wall('running graphviz on %s' % filename_dot, 1.0):
try:
# print('running graphviz')
system_cmd_result(
cwd='.',
cmd=cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True,
)
# print('done')
except (CmdException, KeyboardInterrupt):
emergency = 'emergency.dot'
logger.error('saving to %r' % emergency) # XXX
contents = open(filename_dot).read()
with open(emergency, 'w') as f:
f.write(contents)
raise
def embed_img_data(soup,
resolve,
img_extensions=['png', 'jpg', 'PNG', 'JPG', 'svg', 'SVG']):
"""
resolve: ref -> str or None --- how to get the data
"""
for tag in soup.select('img[src]'):
href = tag['src']
if href.startswith('data:'):
continue
for ext in img_extensions:
if not href.endswith('.' + ext):
continue
data = resolve(href)
if data is None:
logger.error('embed_img_data: Could not find file %s' % href)
continue
check_isinstance(data, str)
tag['src'] = data_encoded_for_src(data, ext)
break
def gg_figure(r, name, ggraph, do_png=True, do_pdf=True, do_svg=True,
do_dot=True):
""" Adds a figure to the Report r that displays this graph
and also its source. """
f = r.figure(name, cols=1)
# save file in dot file
with tmpfile(".dot") as filename_dot:
with open(filename_dot, 'w') as fo:
s = get_dot_string(ggraph)
fo.write(s)
# if False:
# ff = '%s.dot' % id(r)
# print('writing to %r' % ff)
# with open(ff, 'w') as f2:
# f2.write(s)
prog = 'dot'
try:
if do_png:
with f.data_file('graph', MIME_PNG) as filename:
graphviz_run(filename_dot, filename, prog=prog)
if do_pdf:
with f.data_file('graph_pdf', MIME_PDF) as filename:
graphviz_run(filename_dot, filename, prog=prog)
if do_svg:
with f.data_file('graph_svg', MIME_SVG) as filename:
graphviz_run(filename_dot, filename, prog=prog)
from mcdp_report.embedded_images import embed_svg_images
data = open(filename).read()
soup = bs(data)
embed_svg_images(soup)
# does not keep doctype: s = to_html_stripping_fragment(soup)
# this will keep the doctype
s = str(soup)
s = s.replace('<fragment>', '')
s = s.replace('</fragment>', '')
write_bytes_to_file_as_utf8(s, filename)
except CmdException:
if MCDPConstants.test_ignore_graphviz_errors:
mcdp_dev_warning('suppressing errors from graphviz')
logger.error('Graphivz failed, but I will ignore it '
'because of MCDPConstants.test_ignore_graphviz_errors.')
else:
raise
# MIME_GRAPHVIZ
if do_dot:
with f.data_file('dot', MIME_PLAIN) as filename:
with open(filename, 'w') as f:
f.write(s)
return f
def view_thing_delete(self, e):
name = e.thing_name
# basename = "%s.%s" % (name, e.spec.extension)
logger.error('Deleting %s' % name)
del e.things[name]
# filename = e.library.delete_file(basename)
# e.session.notify_deleted_file(e.shelf_name, e.library_name, filename)
raise HTTPFound(e.request.referrer)
def groupfinder(userid, request): # @UnusedVariable
from mcdp_web.main import WebApp
app = WebApp.singleton
user_db = app.hi.db_view.user_db
if not userid in user_db:
msg = 'The user is authenticated as "%s" but no such user in DB.' % userid
logger.error(msg)
userid = None # anonymous
return ['group:%s' % _ for _ in user_db[userid].groups]
def view_forbidden(self, request):
# if using as argument, context is the HTTPForbidden exception
context = request.context
e = Environment(context, request)
logger.error('forbidden url: %s' % request.url)
logger.error('forbidden referrer: %s' % request.referrer)
logger.error('forbidden exception: %s' % request.exception.message)
logger.error('forbidden result: %s' % request.exception.result)
request.response.status = 403
config = self.get_authomatic_config()
# Bug! this must be front-facing
url_internal = request.url
if self.options.url_base_internal is not None:
url_external = url_internal.replace(self.options.url_base_internal,
self.options.url_base_public)
else:
url_external = url_internal
logger.debug('next_location:\n internal: %s\n external: %s' %
(url_internal, url_external))
config['next_location'] = url_external
res = {}
res['request_exception_message'] = request.exception.message
res['request_exception_result'] = request.exception.result
# path_qs The path of the request, without host but with query string
res['came_from'] = request.path_qs
res['referrer'] = request.referrer
res['login_form'] = self.make_relative(request, URL_LOGIN)
res['url_logout'] = self.make_relative(request, URL_LOGOUT)
res['root'] = e.root
res['static'] = e.root + '/static'
# XXX DRY
providers = self.get_authomatic_config()
other_logins = {}
for x in providers:
other_logins[x] = e.root + '/authomatic/' + x
res['other_logins'] = other_logins
if context is not None:
res['context_detail'] = context_display_in_detail(context)
logger.error(res['context_detail'])
else:
res['context_detail'] = 'no context provided'
if e.username is not None:
# res['error'] = ''
res['user_struct'] = e.user
else:
res['error'] = 'You need to login to access this resource.'
res['user_struct'] = None
return res
def get_test_librarian():
package = dir_from_package_name('mcdp_data')
folder = os.path.join(package, 'bundled.mcdp_repo')
if not os.path.exists(folder):
raise_desc(ValueError, 'Test folders not found.', folder=folder)
librarian = Librarian()
librarian.find_libraries(folder)
libraries = librarian.libraries
n = len(libraries)
if n <= 1:
msg = 'Expected more libraries.'
raise_desc(ValueError, msg, folder, libraries=libraries)
orig = list(libraries)
vname = MCDPConstants.ENV_TEST_LIBRARIES
if vname in os.environ:
use = os.environ[vname].split(",")
logger.debug('environment variable %s = %s' % (vname, use))
logger.info('Because %s is set, I will use only %s instead of %s.' %
(vname, use, orig))
for _ in orig:
if not _ in use:
del libraries[_]
else:
pass
#logger.debug('environment variable %s is unset' % vname)
vname2 = MCDPConstants.ENV_TEST_LIBRARIES_EXCLUDE
if vname2 in os.environ:
exclude = os.environ[vname2].split(',')
logger.debug('environment variable %s = %s' % (vname2, exclude))
else:
exclude = []
# logger.debug('environment variable %s is unset' % vname2)
if exclude:
for a in exclude:
if not a in libraries:
msg = '%s = %s but %r is not a library.' % (vname2, exclude, a)
logger.error(msg)
else:
logger.info('Excluding %s' % vname2)
del libraries[a]
return librarian
def embed_pdf_image(tag, resolve, density):
assert tag.name == 'img'
assert tag.has_attr('src')
#print('!!embedding %s' % str(tag))
#raise Exception(str(tag))
# load pdf data
data_pdf = resolve(tag['src'])
if data_pdf is None:
add_class(tag, 'missing-image')
return
# convert PDF to PNG
# density = pixels per inch
data_png = png_from_pdf(data_pdf, density=density)
# get PNG image size in pixels
width_px, height_px = get_pixel_width_height_of_png(data_png)
# compute what was the original width of PDF in points
width_in = width_px / float(density)
height_in = height_px / float(density)
latex_options = tag.get('latex-options', '')
props = parse_includegraphics_option_string(latex_options)
if 'height' in props:
logger.warning('Cannot deal with "height" yet: latex_options = %s' %
latex_options)
if 'scale' in props:
scale = float(props['scale'])
use_width_in = width_in * scale
use_height_in = height_in * scale
elif 'width' in props:
try:
use_width_in = get_length_in_inches(props['width'])
except ValueError as e:
logger.error('Cannot interpret %s: %s' % (latex_options, e))
use_width_in = 5.0
ratio = height_in / width_in
use_height_in = use_width_in * ratio
else:
use_width_in = width_in
use_height_in = height_in
# Add it before so that we can override
add_style(tag,
after=False,
width='%sin' % use_width_in,
height='%sin' % use_height_in)
tag['size_in_pixels'] = '%s, %s' % (width_px, height_px)
# encode
tag['src'] = data_encoded_for_src(data_png, 'png')
def safe_pickle_dump(value, filename, protocol=pickle.HIGHEST_PROTOCOL,
**safe_write_options):
with safe_write(filename, **safe_write_options) as f:
try:
pickle.dump(value, f, protocol)
except KeyboardInterrupt:
raise
except Exception:
msg = 'Cannot pickle object of class %s' % describe_type(value)
logger.error(msg)
msg = find_pickling_error(value, protocol)
logger.error(msg)
raise
def resolve(href):
#print('resolving %r' % href)
try:
f = library._get_file_data(href)
except DPSemanticError:
if raise_errors:
raise
else:
msg = 'Could not find file %r.' % href
logger.error(msg)
return None
data = f['data']
# realpath = f['realpath']
return data
def format_exception_for_ajax_response(e, quiet=()):
s = e.__repr__().decode('ascii', 'ignore')
from mcdp import logger
try:
logger.error('format_exception_for_ajax_response: ' + s)
except UnicodeEncodeError:
pass
res = {}
res['ok'] = False
if isinstance(e, quiet):
s = type(e).__name__ + ': ' + str(e)
else:
s = traceback.format_exc(e)
res['error'] = cgi.escape(s)
return res
def junit_xml(compmake_db):
jobs = list(all_jobs(compmake_db))
logger.info('Loaded %d jobs' % len(jobs))
if len(jobs) < 10:
logger.error('too few jobs')
sys.exit(128)
test_cases = []
for job_id in jobs:
tc = junit_test_case_from_compmake(compmake_db, job_id)
test_cases.append(tc)
ts = TestSuite("comptests_test_suite", test_cases)
return TestSuite.to_xml_string([ts])
def create_composite(gdc0, ndp, plotting_info):
try:
SKIP_INITIAL = gdc0.skip_initial
#print('Skip initial: %s' % SKIP_INITIAL)
return create_composite_(gdc0,
ndp,
plotting_info=plotting_info,
SKIP_INITIAL=SKIP_INITIAL)
except Exception as e:
logger.error(e)
logger.error('I will try again without the SKIP_INITIAL parameter.')
return create_composite_(gdc0,
ndp,
plotting_info=plotting_info,
SKIP_INITIAL=False)
def parse_main_template(fn):
template = open(fn).read()
soup = bs_entire_document(template)
base_dir = os.path.dirname(fn)
embed_css_files(soup, base_dir)
head = soup.find('head')
if head is None:
msg = 'Could not find <head> in template'
logger.error(msg)
logger.error(str(soup))
raise Exception(msg)
template = to_html_entire_document(soup)
return template
def go(path):
db = StorageFilesystem(path, compress=True)
args = ['failed']
cq = CacheQueryDB(db)
context = Context(db)
if not list(db.keys()):
msg = 'Compmake DB is empty'
logger.error(msg)
else:
job_list = parse_job_list(args, context=context, cq=cq)
s = ""
if job_list:
job_list = job_list[:2]
s += 'Running on host: %s' % hostname
s += "\nJob failed in path %s" % path
for job_id in job_list:
if job_cache_exists(job_id, db):
cache = get_job_cache(job_id, db)
status = Cache.state2desc[cache.state]
s += "\nFailure of job %s" % job_id
if cache.state in [Cache.FAILED, Cache.BLOCKED]:
why = str(cache.exception).strip()
else:
why = 'No why for job done.'
s += '\n' + "```\n" + why + "\n```"
s += '\n\n'
else:
logger.warning('no cache for %s' % job_id)
s += '\n@censi'
s += '\n@jacopo'
s += '\n@paull'
s += '\n@walter'
s += '\n@daniele'
print(s)
slack.chat.post_message(channel, s, link_names=1)
else:
s = 'Everything is fine'
# slack.chat.post_message(channel, s)
logger.info('No jobs found')
def png_error_catch2(self, request, func):
""" func is supposed to return an image response.
If it raises an exception, we create
an image with the error and then we add the exception
to the list of exceptions.
"""
try:
return func()
except Exception as e:
s = traceback.format_exc(e)
try:
logger.error(s)
except UnicodeEncodeError:
pass
self.note_exception(e, request=request)
return response_image(request, s)
def groupfinder(userid, request): # @UnusedVariable
from mcdp_web.main import WebApp
app = WebApp.singleton
user_db = app.hi.db_view.user_db
if not userid in user_db:
msg = 'The user is authenticated as "%s" but no such user in DB.' % userid
logger.error(msg)
userid = None # anonymous
return ['group:%s' % _ for _ in user_db[userid].groups]
#
# def hash_password(pw):
# pwhash = bcrypt.hashpw(pw.encode('utf8'), bcrypt.gensalt())
# return pwhash.decode('utf8')
#
# def check_password(pw, hashed_pw):
# expected_hash = hashed_pw.encode('utf8')
# return bcrypt.checkpw(pw.encode('utf8'), expected_hash)
def go():
fn = sys.argv[1]
outdir = sys.argv[2]
warning = os.path.join(outdir, 'DO_NOT_PLACE_OTHER_FILES_HERE.txt')
write_data_to_file('It is autogenerated', warning)
f = open(fn, 'rb')
pdfReader = PyPDF2.PdfFileReader(f)
n = pdfReader.numPages
results = []
for i in range(n):
pageObj = pdfReader.getPage(i)
try:
name = get_figure_name(pageObj)
results.append((i, name))
logger.info('page %d name = %r' % (i, name))
except NoFigureName as _e:
logger.error(str(_e))
allt = ""
tmpdir = create_tmpdir('clipart')
for i, name in results:
target_pdf = os.path.join(outdir, name + '.pdf')
target_svg = os.path.join(outdir, name + '.svg')
page = i+1
process_one(fn, page, target_pdf, target_svg, tmpdir, name)
t = """
<div figure-id="fig:NAME">
<figcaption>Add caption here</figcaption>
<img src="NAME.svg" style="width: 80%; height: auto"/>
</div>
"""
t = t.replace('NAME', name)
allt += t
instructions = os.path.join(outdir, 'how_to_use.md.example')
write_data_to_file(allt, instructions)
print allt
def check_parsable(s):
from xml.etree import ElementTree as ET
# parser = ET.XMLParser()
# parser.entity["nbsp"] = unichr(160)
s = '<add-wrap-for-xml-parser>' + s + '</add-wrap-for-xml-parser>'
# print indent(s, ' for xml')
# with open('rtmp.xml', 'w') as f:
# f.write(s)
try:
_ = ET.fromstring(s)
except Exception as e:
line1, col1 = e.position
line = line1 - 1
col = col1 - 1
character = find_location(line, col, s)
msg = 'Invalid XML: %s' % e
where = Where(s, character)
logger.error('line %s col %s' % (where.line, where.col))
logger.error(where)
raise DPSyntaxError(msg, where=where)
