def extract_assets(html, basedir):
"""
Extracts all embedded assets in A links
encoded using data: and save them to file.
These are all links of the type:
<a href="data:****"/>
<a href="data:****" download='filename'/>
"""
if not os.path.exists(basedir):
os.makedirs(basedir)
soup = BeautifulSoup(html, 'lxml', from_encoding='utf-8')
for tag in soup.select('a[href]'):
href = tag['href']
if href.startswith('data:'):
_mime, data = get_mime_data_from_base64_string(href)
if tag.has_attr('download'):
basename = tag['download']
else:
logger.warn('cannot find attr "download" in tag')
# print tag
continue
print('extracting asset %s' % basename)
filename = os.path.join(basedir, basename)
write_data_to_file(data, filename)
def load_spec(self, repo_name, shelf_name, library_name, spec_name,
thing_name, context):
db_view = self.db_view
key0 = (repo_name, shelf_name, library_name, spec_name, thing_name)
if not key0 in self.evaluated:
x = get_source(db_view, repo_name, shelf_name, library_name,
spec_name, thing_name)
source = x['data']
# We can do the parsing only once. It only depends on the string
# and nothing else
key = (spec_name, source)
if not key in self.parsing_cache:
t0 = time.clock()
self.parsing_cache[key] = \
self.parse_source(spec_name, source, context)
t1 = time.clock()
dms = 1000 * (t1 - t0)
self.parsing_cache_time_ms[key] = dms
logger.warn('Parsing %s: %s ms' % (thing_name, dms))
else:
dms = self.parsing_cache_time_ms[key]
logger.debug('Parsing %s: saved %s' % (thing_name, dms))
parsed = self.parsing_cache[key]
parse_eval = specs[spec_name].parse_eval
res = parse_eval(parsed, context)
self.evaluated[key0] = res
return self.evaluated[key0]
def dp_repr_long_labeled(data):
ndp = get_ndp(data)
ndp = get_labelled_version(ndp)
try:
dp = ndp.get_dp()
except DPSemanticError:
logger.warn('Could not draw dp_graph_tree_compact_labeled_')
return []
res1 = ('txt', 'dp_repr_long_labeled', dp.repr_long())
return [res1]
def go(self):
options = self.get_options()
if options.config is not None:
logger.info('Reading configuration from %s' % options.config)
logger.warn('Other options from command line will be ignored. ')
parser = RawConfigParser()
parser.read(options.config)
sections = parser.sections()
logger.info('sections: %s' % sections)
s = 'app:main'
if not s in sections:
msg = 'Could not find section "%s": available are %s.' % (
s, format_list(sections))
msg += '\n file %s' % options.config
raise Exception(msg) # XXX
settings = dict((k, parser.get(s, k)) for k in parser.options(s))
prefix = 'mcdp_web.'
mcdp_web_settings = get_only_prefixed(settings,
prefix,
delete=True)
# mcdp_web_settings = {}
# for k,v in list(settings.items()):
# if k.startswith(prefix):
# mcdp_web_settings[k[len(prefix):]] = v
# del settings[k]
options = parse_mcdpweb_params_from_dict(mcdp_web_settings)
logger.debug('Using these options: %s' % options)
else:
logger.info('No configuration .ini specified (use --config).')
settings = {}
wa = WebApp(options, settings=settings)
# Write warning messages now
wa.get_authomatic_config()
msg = """Welcome to PyMCDP!
To access the interface, open your browser at the address
http://localhost:%s/
Use Chrome, Firefox, or Opera - Internet Explorer is not supported.
""" % options.port
logger.info(msg)
if options.delete_cache:
pass # XXX: warning deprecated
# logger.info('Deleting cache...')
# wa._refresh_library(None)
wa.serve(port=options.port)
def dp_graph_tree_compact_labeled_(data):
ndp = get_ndp(data)
ndp = get_labelled_version(ndp)
try:
dp = ndp.get_dp()
except DPSemanticError:
logger.warn('Could not draw dp_graph_tree_compact_labeled_')
return []
gg = dp_graph_tree(dp, compact=True)
return return_formats2(gg, 'dp_graph_tree_compact_labeled')
def do_plots(logger, model_name, plots, outdir, extra_params, maindir,
extra_dirs, use_cache):
if '.mcdp' in model_name:
model_name2 = model_name.replace('.mcdp', '')
msg = 'Arguments should be model names, not file names.'
msg += ' Interpreting %r as %r.' % (model_name, model_name2)
logger.warn(msg)
model_name = model_name2
if use_cache:
cache_dir = os.path.join(outdir, '_cached/mcdp_plot_cache')
logger.info('using cache %s' % cache_dir)
else:
cache_dir = None
librarian = Librarian()
for e in extra_dirs:
librarian.find_libraries(e)
library = librarian.get_library_by_dir(maindir)
if cache_dir is not None:
library.use_cache_dir(cache_dir)
assert library.library_name is not None
is_ndp = library.file_exists(model_name + '.mcdp')
is_poset = library.file_exists(model_name + '.mcdp_poset')
if is_ndp:
results = do_plots_ndp(model_name, library, plots, extra_params)
elif is_poset:
results = do_plots_poset(model_name, library, plots)
else:
msg = 'Cannot find anything corresponding to %r.' % model_name
raise_desc(ValueError, msg)
return
write_results(results, model_name, outdir)
def get_authomatic_config_(self):
CONFIG = {}
if self.options.google_consumer_key is not None:
CONFIG['google'] = {
'class_':
oauth2.Google,
'consumer_key':
self.options.google_consumer_key,
'consumer_secret':
self.options.google_consumer_secret,
'scope': [
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/userinfo.profile',
],
}
logger.info('Configured Google authentication.')
else:
logger.warn('No Google authentication configuration found.')
if self.options.facebook_consumer_key is not None:
oauth2.Facebook.user_info_url = (
'https://graph.facebook.com/v2.5/me?fields=id,first_name,last_name,picture,email,'
'gender,timezone,location,middle_name,name_format,third_party_id,website,birthday,locale'
)
CONFIG['facebook'] = {
'class_': oauth2.Facebook,
'consumer_key': self.options.facebook_consumer_key,
'consumer_secret': self.options.facebook_consumer_secret,
'scope': ['public_profile', 'email'],
}
logger.info('Configured Facebook authentication.')
else:
logger.warn('No Facebook authentication configuration found.')
if self.options.github_consumer_key is not None:
CONFIG['github'] = {
'class_': oauth2.GitHub,
'consumer_key': self.options.github_consumer_key,
'consumer_secret': self.options.github_consumer_secret,
# todo: implement
'scope': [ # 'user',
'user:email'
],
'access_headers': {
'User-Agent': 'PyMCDP'
},
}
logger.info('Configured Github authentication.')
else:
logger.warn('No Github authentication configuration found.')
if self.options.amazon_consumer_key is not None:
CONFIG['amazon'] = {
'class_': oauth2.Amazon,
'consumer_key': self.options.amazon_consumer_key,
'consumer_secret': self.options.amazon_consumer_secret,
'scope': ['profile'],
}
logger.info('Configured Amazon authentication.')
else:
logger.warn('No Amazon authentication configuration found.')
if self.options.linkedin_consumer_key is not None:
CONFIG['linkedin'] = {
'class_': oauth2.LinkedIn,
'consumer_key': self.options.linkedin_consumer_key,
'consumer_secret': self.options.linkedin_consumer_secret,
'scope': ['r_emailaddress', 'r_basicprofile'],
}
logger.info('Configured Linkedin authentication.')
else:
logger.warn('No Linkedin authentication configuration found.')
return CONFIG
def render_complete(library,
s,
raise_errors,
realpath,
generate_pdf=False,
check_refs=False,
use_mathjax=True,
filter_soup=None,
symbols=None):
"""
Transforms markdown into html and then renders the mcdp snippets inside.
s: a markdown string with embedded html snippets
Returns an HTML string; not a complete document.
filter_soup(library, soup)
"""
s0 = s
check_good_use_of_special_paragraphs(s0, realpath)
raise_missing_image_errors = raise_errors
# Imports here because of circular dependencies
from .latex.latex_preprocess import extract_maths, extract_tabular
from .latex.latex_preprocess import latex_preprocessing
from .latex.latex_preprocess import replace_equations
from .macro_col2 import col_macros, col_macros_prepare_before_markdown
from .mark.markd import render_markdown
from .preliminary_checks import do_preliminary_checks_and_fixes
from .prerender_math import prerender_mathjax
if isinstance(s, unicode):
msg = 'I expect a str encoded with utf-8, not unicode.'
raise_desc(TypeError, msg, s=s)
# need to do this before do_preliminary_checks_and_fixes
# because of & char
s, tabulars = extract_tabular(s)
s = do_preliminary_checks_and_fixes(s)
# put back tabular, because extract_maths needs to grab them
for k, v in tabulars.items():
assert k in s
s = s.replace(k, v)
# copy all math content,
# between $$ and $$
# between various limiters etc.
# returns a dict(string, substitution)
s, maths = extract_maths(s)
# print('maths = %s' % maths)
for k, v in maths.items():
if v[0] == '$' and v[1] != '$$':
if '\n\n' in v:
msg = 'Suspicious math fragment %r = %r' % (k, v)
logger.error(maths)
logger.error(msg)
raise ValueError(msg)
s = latex_preprocessing(s)
s = '<div style="display:none">Because of mathjax bug</div>\n\n\n' + s
# cannot parse html before markdown, because md will take
# invalid html, (in particular '$ ciao <ciao>' and make it work)
s = s.replace('*}', '\*}')
s, mcdpenvs = protect_my_envs(s)
# print('mcdpenvs = %s' % maths)
s = col_macros_prepare_before_markdown(s)
# print(indent(s, 'before markdown | '))
s = render_markdown(s)
# print(indent(s, 'after markdown | '))
for k, v in maths.items():
if not k in s:
msg = 'Cannot find %r (= %r)' % (k, v)
raise_desc(DPInternalError, msg, s=s)
def preprocess_equations(x):
# this gets mathjax confused
x = x.replace('>', '\\gt{}') # need brace; think a<b -> a\lt{}b
x = x.replace('<', '\\lt{}')
# print('replaced equation %r by %r ' % (x0, x))
return x
v = preprocess_equations(v)
s = s.replace(k, v)
s = replace_equations(s)
s = s.replace('\\*}', '*}')
# this parses the XML
soup = bs(s)
other_abbrevs(soup)
# need to process tabular before mathjax
escape_for_mathjax(soup)
# print(indent(s, 'before prerender_mathjax | '))
# mathjax must be after markdown because of code blocks using "$"
s = to_html_stripping_fragment(soup)
if use_mathjax:
s = prerender_mathjax(s, symbols)
soup = bs(s)
escape_for_mathjax_back(soup)
s = to_html_stripping_fragment(soup)
# print(indent(s, 'after prerender_mathjax | '))
for k, v in mcdpenvs.items():
# there is this case:
# ~~~
# <pre> </pre>
# ~~~
s = s.replace(k, v)
s = s.replace('<p>DRAFT</p>', '<div class="draft">')
s = s.replace('<p>/DRAFT</p>', '</div>')
soup = bs(s)
mark_console_pres(soup)
try:
substitute_github_refs(soup, defaults={})
except Exception as e:
msg = 'I got an error while substituting github: references.'
msg += '\nI will ignore this error because it might not be the fault of the writer.'
msg += '\n\n' + indent(str(e), '|', ' error: |')
logger.warn(msg)
# must be before make_figure_from_figureid_attr()
display_files(soup, defaults={}, raise_errors=raise_errors)
make_figure_from_figureid_attr(soup)
col_macros(soup)
fix_subfig_references(soup)
library = get_library_from_document(soup, default_library=library)
from mcdp_docs.highlight import html_interpret
html_interpret(library,
soup,
generate_pdf=generate_pdf,
raise_errors=raise_errors,
realpath=realpath)
if filter_soup is not None:
filter_soup(library=library, soup=soup)
embed_images_from_library2(soup=soup,
library=library,
raise_errors=raise_missing_image_errors)
make_videos(soup=soup)
if check_refs:
check_if_any_href_is_invalid(soup)
if getuser() == 'andrea':
if MCDPConstants.preprocess_style_using_less:
run_lessc(soup)
else:
logger.warning(
'preprocess_style_using_less=False might break the manual')
fix_validation_problems(soup)
strip_pre(soup)
if MCDPManualConstants.enable_syntax_higlighting:
syntax_highlighting(soup)
if MCDPManualConstants.enforce_status_attribute:
check_status_codes(soup, realpath)
if MCDPManualConstants.enforce_lang_attribute:
check_lang_codes(soup)
# Fixes the IDs (adding 'sec:'); add IDs to missing ones
globally_unique_id_part = 'autoid-DO-NOT-USE-THIS-VERY-UNSTABLE-LINK-' + get_md5(
s0)[:5]
fix_ids_and_add_missing(soup, globally_unique_id_part)
check_no_patently_wrong_links(soup)
s = to_html_stripping_fragment(soup)
s = replace_macros(s)
return s
def f0(self, context, request):
url_base_internal = self.options.url_base_internal
if url_base_internal is not None:
if not request.url.startswith(url_base_internal):
msg = (
'Given that url_base_internal is set, I was expecting that all urls'
' would start with it.')
raise_desc(Exception,
msg,
request_url=request.url,
url_base_internal=url_base_internal)
if '//' in urlparse.urlparse(request.url).path:
msg = 'This is an invalid URL with 2 slashes: %s' % request.url
response = Response(msg)
response.status_int = 500
return response
if redir:
url = request.url
p = urlparse.urlparse(url)
url2 = url
# only do redirection if we have url_base_internal
# The redirection is needed because of https; however
# for casual use it is likely https is not set up.
if self.options.url_base_internal:
if '127.0.0.1' in p.netloc:
url2 = url2.replace('127.0.0.1', 'localhost')
if not p.path.endswith('.html'):
if not p.path.endswith('/'):
url2 = url2.replace(p.path, p.path + '/')
if url2 != url:
logger.info('Context: %s' % context)
logger.info('Redirection:\n from: %s\n to: %s' % (url, url2))
raise HTTPFound(url2)
if request.authenticated_userid:
uid = request.authenticated_userid
from mcdp_web.main import WebApp
app = WebApp.singleton
user_db = app.hi.db_view.user_db
if not uid in user_db:
msg = 'The user is authenticated as "%s" but no such user in DB.' % uid
msg += 'We are logging out the user.'
logger.warn(msg)
headers = forget(request)
raise HTTPFound(location=request.url, headers=headers)
try:
res = f(self, context, request)
except HTTPException:
raise
except Exception as e:
msg = 'While running %s:' % (f.__name__)
msg += '\n' + indent(traceback.format_exc(e), ' >')
logger.error(msg)
raise
if isinstance(res, Response):
return res
check_isinstance(res, dict)
try:
add_other_fields(self, res, request, context=context)
except:
logger.error('Error after executing view %s' % f)
if isinstance(context, Resource):
logger.debug(context_display_in_detail(context))
raise
return res
def manual_jobs(context,
src_dirs,
resources_dirs,
out_split_dir,
output_file,
generate_pdf,
stylesheet,
stylesheet_pdf,
use_mathjax,
raise_errors,
resolve_references=True,
remove=None,
filter_soup=None,
symbols=None,
out_pdf=None,
only_refs=False,
permalink_prefix=None,
compose_config=None,
output_crossref=None,
do_last_modified=False,
wordpress_integration=False,
ignore_ref_errors=False,
likebtn=None,
extra_crossrefs=None):
"""
src_dirs: list of sources
symbols: a TeX preamble (or None)
"""
#
# if symbols is not None:
# symbols = open(symbols).read()
if stylesheet_pdf is None:
stylesheet_pdf = stylesheet
# outdir = os.path.dirname(out_split_dir) # XXX
filenames = get_markdown_files(src_dirs)
if not filenames:
msg = "Could not find any file for composing the book."
raise Exception(msg)
files_contents = []
for i, filename in enumerate(filenames):
if is_ignored_by_catkin(filename):
logger.debug('Ignoring because of CATKIN_IGNORE: %s' % filename)
continue
logger.info('adding document %s ' % friendly_path(filename))
docname, _ = os.path.splitext(os.path.basename(filename))
contents = open(filename).read()
contents_hash = get_md5(contents)[:8]
# because of hash job will be automatically erased if the source changes
out_part_basename = '%03d-%s-%s' % (i, docname, contents_hash)
job_id = '%s-%s-%s' % (docname, get_md5(filename)[:8], contents_hash)
try:
source_info = get_source_info(filename)
except NoSourceInfo as e:
logger.warn('No source info for %s:\n%s' % (filename, e))
source_info = None
for d in src_dirs:
if filename.startswith(d):
break
else:
msg = "Could not find dir for %s in %s" % (filename, src_dirs)
raise Exception(msg)
html_contents = context.comp(render_book,
generate_pdf=generate_pdf,
src_dirs=src_dirs + resources_dirs,
data=contents,
realpath=filename,
use_mathjax=use_mathjax,
symbols=symbols,
raise_errors=raise_errors,
filter_soup=filter_soup,
ignore_ref_errors=ignore_ref_errors,
job_id=job_id)
doc = DocToJoin(docname=out_part_basename,
contents=html_contents,
source_info=source_info)
files_contents.append(tuple(doc)) # compmake doesn't do namedtuples
ignore = []
if output_crossref:
ignore.append(output_crossref)
crossrefs_aug = get_cross_refs(resources_dirs,
permalink_prefix,
extra_crossrefs,
ignore=ignore)
bib_files = get_bib_files(src_dirs)
logger.debug('Found bib files:\n%s' % "\n".join(bib_files))
if bib_files:
bib_contents_aug = job_bib_contents(context, bib_files)
entry = DocToJoin(docname='bibtex',
contents=bib_contents_aug,
source_info=None)
files_contents.append(tuple(entry))
if do_last_modified:
data_aug = context.comp(make_last_modified,
files_contents=files_contents)
entry = DocToJoin(docname='last_modified',
contents=data_aug,
source_info=None)
files_contents.append(tuple(entry))
root_dir = src_dirs[0]
template = get_main_template(root_dir, resources_dirs)
references = OrderedDict()
# base_url = 'http://book.duckietown.org/master/duckiebook/pdoc'
# for extra_dir in extra_dirs:
# res = read_references(extra_dir, base_url, prefix='python:')
# references.update(res)
# extra = look_for_files(extra_dirs, "*.html")
#
# for filename in extra:
# contents = open(filename).read()
# docname = os.path.basename(filename) + '_' + get_md5(filename)[:5]
# c = (('unused', docname), contents)
# files_contents.append(c)
cs = get_md5((crossrefs_aug.get_result()))[:8]
joined_aug = context.comp(manual_join,
template=template,
files_contents=files_contents,
stylesheet=None,
remove=remove,
references=references,
resolve_references=resolve_references,
crossrefs_aug=crossrefs_aug,
permalink_prefix=permalink_prefix,
job_id='join-%s' % cs)
if compose_config is not None:
try:
data = yaml.load(open(compose_config).read()) # XXX
compose_config_interpreted = ComposeConfig.from_yaml(data)
except ValueError as e:
msg = 'Cannot read YAML config file %s' % compose_config
raise_wrapped(UserError, e, msg, compact=True)
else:
joined_aug = context.comp(make_composite,
compose_config_interpreted, joined_aug)
joined_aug = context.comp(mark_errors_and_rest, joined_aug)
if likebtn:
joined_aug = context.comp(add_likebtn, joined_aug, likebtn)
if wordpress_integration:
joined_aug = context.comp(add_related, joined_aug)
if output_file is not None:
context.comp(write, joined_aug, output_file)
if out_split_dir is not None:
joined_aug_with_html_stylesheet = context.comp(add_style, joined_aug,
stylesheet)
extra_panel_content = context.comp(get_extra_content,
joined_aug_with_html_stylesheet)
id2filename_aug = context.comp_dynamic(
create_split_jobs,
data_aug=joined_aug_with_html_stylesheet,
mathjax=True,
preamble=symbols,
extra_panel_content=extra_panel_content,
output_dir=out_split_dir,
nworkers=0,
output_crossref=output_crossref,
permalink_prefix=permalink_prefix,
only_refs=only_refs)
if not only_refs:
context.comp(write_errors_and_warnings_files, id2filename_aug,
out_split_dir)
context.comp(write_manifest_html, out_split_dir)
if out_pdf is not None:
joined_aug_with_pdf_stylesheet = context.comp(add_style, joined_aug,
stylesheet_pdf)
prerendered = context.comp(prerender,
joined_aug_with_pdf_stylesheet,
symbols=symbols)
pdf_data = context.comp(render_pdf, prerendered)
context.comp(write_data_to_file, pdf_data, out_pdf)
context.comp(write_manifest_pdf, out_pdf)
def write_errors_and_warnings_files(aug, d):
if aug.has_result():
id2filename = aug.get_result()
else:
id2filename = {}
# print('id2filename: %s' % sorted(id2filename))
assert isinstance(aug, AugmentedResult)
aug.update_refs(id2filename)
header = get_notes_panel(aug)
manifest = []
nwarnings = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_WARNING))
fn = os.path.join(d, 'warnings.html')
html = html_list_of_notes(aug,
MCDPManualConstants.NOTE_TAG_WARNING,
'warnings',
'warning',
header=header)
# update_refs_('warnings', html, id2filename)
write_data_to_file(str(html), fn, quiet=True)
if nwarnings:
manifest.append(
dict(display='%d warnings' % nwarnings, filename='warnings.html'))
msg = 'There were %d warnings: %s' % (nwarnings, fn)
logger.warn(msg)
ntasks = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_TASK))
fn = os.path.join(d, 'tasks.html')
html = html_list_of_notes(aug,
MCDPManualConstants.NOTE_TAG_TASK,
'tasks',
'task',
header=header)
# update_refs_('tasks', html, id2filename)
write_data_to_file(str(html), fn, quiet=True)
if nwarnings:
manifest.append(
dict(display='%d tasks' % ntasks, filename='tasks.html'))
msg = 'There are %d open tasks: %s' % (ntasks, fn)
logger.info(msg)
nerrors = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_ERROR))
fn = os.path.join(d, 'errors.html')
html = html_list_of_notes(aug,
MCDPManualConstants.NOTE_TAG_ERROR,
'errors',
'error',
header=header)
# update_refs_('tasks', html, id2filename)
write_data_to_file(str(html), fn, quiet=True)
if nerrors:
manifest.append(
dict(display='%d errors' % nerrors, filename='errors.html'))
msg = 'I am sorry to say that there were %d errors.\n\nPlease see: %s' % (
nerrors, fn)
logger.error('\n\n\n' + indent(msg, ' ' * 15) + '\n\n')
fn = os.path.join(d, 'errors_and_warnings.manifest.yaml')
write_data_to_file(yaml.dump(manifest), fn, quiet=False)
fn = os.path.join(d, 'errors_and_warnings.pickle')
res = AugmentedResult()
res.merge(aug)
write_data_to_file(pickle.dumps(res), fn, quiet=False)
