def extract_assets(html, basedir): """ Extracts all embedded assets in A links encoded using data: and save them to file. These are all links of the type: <a href="data:****"/> <a href="data:****" download='filename'/> """ if not os.path.exists(basedir): os.makedirs(basedir) soup = BeautifulSoup(html, 'lxml', from_encoding='utf-8') for tag in soup.select('a[href]'): href = tag['href'] if href.startswith('data:'): _mime, data = get_mime_data_from_base64_string(href) if tag.has_attr('download'): basename = tag['download'] else: logger.warn('cannot find attr "download" in tag') # print tag continue print('extracting asset %s' % basename) filename = os.path.join(basedir, basename) write_data_to_file(data, filename)
def load_spec(self, repo_name, shelf_name, library_name, spec_name, thing_name, context): db_view = self.db_view key0 = (repo_name, shelf_name, library_name, spec_name, thing_name) if not key0 in self.evaluated: x = get_source(db_view, repo_name, shelf_name, library_name, spec_name, thing_name) source = x['data'] # We can do the parsing only once. It only depends on the string # and nothing else key = (spec_name, source) if not key in self.parsing_cache: t0 = time.clock() self.parsing_cache[key] = \ self.parse_source(spec_name, source, context) t1 = time.clock() dms = 1000 * (t1 - t0) self.parsing_cache_time_ms[key] = dms logger.warn('Parsing %s: %s ms' % (thing_name, dms)) else: dms = self.parsing_cache_time_ms[key] logger.debug('Parsing %s: saved %s' % (thing_name, dms)) parsed = self.parsing_cache[key] parse_eval = specs[spec_name].parse_eval res = parse_eval(parsed, context) self.evaluated[key0] = res return self.evaluated[key0]
def dp_repr_long_labeled(data): ndp = get_ndp(data) ndp = get_labelled_version(ndp) try: dp = ndp.get_dp() except DPSemanticError: logger.warn('Could not draw dp_graph_tree_compact_labeled_') return [] res1 = ('txt', 'dp_repr_long_labeled', dp.repr_long()) return [res1]
def go(self): options = self.get_options() if options.config is not None: logger.info('Reading configuration from %s' % options.config) logger.warn('Other options from command line will be ignored. ') parser = RawConfigParser() parser.read(options.config) sections = parser.sections() logger.info('sections: %s' % sections) s = 'app:main' if not s in sections: msg = 'Could not find section "%s": available are %s.' % ( s, format_list(sections)) msg += '\n file %s' % options.config raise Exception(msg) # XXX settings = dict((k, parser.get(s, k)) for k in parser.options(s)) prefix = 'mcdp_web.' mcdp_web_settings = get_only_prefixed(settings, prefix, delete=True) # mcdp_web_settings = {} # for k,v in list(settings.items()): # if k.startswith(prefix): # mcdp_web_settings[k[len(prefix):]] = v # del settings[k] options = parse_mcdpweb_params_from_dict(mcdp_web_settings) logger.debug('Using these options: %s' % options) else: logger.info('No configuration .ini specified (use --config).') settings = {} wa = WebApp(options, settings=settings) # Write warning messages now wa.get_authomatic_config() msg = """Welcome to PyMCDP! To access the interface, open your browser at the address http://localhost:%s/ Use Chrome, Firefox, or Opera - Internet Explorer is not supported. """ % options.port logger.info(msg) if options.delete_cache: pass # XXX: warning deprecated # logger.info('Deleting cache...') # wa._refresh_library(None) wa.serve(port=options.port)
def dp_graph_tree_compact_labeled_(data): ndp = get_ndp(data) ndp = get_labelled_version(ndp) try: dp = ndp.get_dp() except DPSemanticError: logger.warn('Could not draw dp_graph_tree_compact_labeled_') return [] gg = dp_graph_tree(dp, compact=True) return return_formats2(gg, 'dp_graph_tree_compact_labeled')
def do_plots(logger, model_name, plots, outdir, extra_params, maindir, extra_dirs, use_cache): if '.mcdp' in model_name: model_name2 = model_name.replace('.mcdp', '') msg = 'Arguments should be model names, not file names.' msg += ' Interpreting %r as %r.' % (model_name, model_name2) logger.warn(msg) model_name = model_name2 if use_cache: cache_dir = os.path.join(outdir, '_cached/mcdp_plot_cache') logger.info('using cache %s' % cache_dir) else: cache_dir = None librarian = Librarian() for e in extra_dirs: librarian.find_libraries(e) library = librarian.get_library_by_dir(maindir) if cache_dir is not None: library.use_cache_dir(cache_dir) assert library.library_name is not None is_ndp = library.file_exists(model_name + '.mcdp') is_poset = library.file_exists(model_name + '.mcdp_poset') if is_ndp: results = do_plots_ndp(model_name, library, plots, extra_params) elif is_poset: results = do_plots_poset(model_name, library, plots) else: msg = 'Cannot find anything corresponding to %r.' % model_name raise_desc(ValueError, msg) return write_results(results, model_name, outdir)
def get_authomatic_config_(self): CONFIG = {} if self.options.google_consumer_key is not None: CONFIG['google'] = { 'class_': oauth2.Google, 'consumer_key': self.options.google_consumer_key, 'consumer_secret': self.options.google_consumer_secret, 'scope': [ 'https://www.googleapis.com/auth/userinfo.email', 'https://www.googleapis.com/auth/userinfo.profile', ], } logger.info('Configured Google authentication.') else: logger.warn('No Google authentication configuration found.') if self.options.facebook_consumer_key is not None: oauth2.Facebook.user_info_url = ( 'https://graph.facebook.com/v2.5/me?fields=id,first_name,last_name,picture,email,' 'gender,timezone,location,middle_name,name_format,third_party_id,website,birthday,locale' ) CONFIG['facebook'] = { 'class_': oauth2.Facebook, 'consumer_key': self.options.facebook_consumer_key, 'consumer_secret': self.options.facebook_consumer_secret, 'scope': ['public_profile', 'email'], } logger.info('Configured Facebook authentication.') else: logger.warn('No Facebook authentication configuration found.') if self.options.github_consumer_key is not None: CONFIG['github'] = { 'class_': oauth2.GitHub, 'consumer_key': self.options.github_consumer_key, 'consumer_secret': self.options.github_consumer_secret, # todo: implement 'scope': [ # 'user', 'user:email' ], 'access_headers': { 'User-Agent': 'PyMCDP' }, } logger.info('Configured Github authentication.') else: logger.warn('No Github authentication configuration found.') if self.options.amazon_consumer_key is not None: CONFIG['amazon'] = { 'class_': oauth2.Amazon, 'consumer_key': self.options.amazon_consumer_key, 'consumer_secret': self.options.amazon_consumer_secret, 'scope': ['profile'], } logger.info('Configured Amazon authentication.') else: logger.warn('No Amazon authentication configuration found.') if self.options.linkedin_consumer_key is not None: CONFIG['linkedin'] = { 'class_': oauth2.LinkedIn, 'consumer_key': self.options.linkedin_consumer_key, 'consumer_secret': self.options.linkedin_consumer_secret, 'scope': ['r_emailaddress', 'r_basicprofile'], } logger.info('Configured Linkedin authentication.') else: logger.warn('No Linkedin authentication configuration found.') return CONFIG
def render_complete(library, s, raise_errors, realpath, generate_pdf=False, check_refs=False, use_mathjax=True, filter_soup=None, symbols=None): """ Transforms markdown into html and then renders the mcdp snippets inside. s: a markdown string with embedded html snippets Returns an HTML string; not a complete document. filter_soup(library, soup) """ s0 = s check_good_use_of_special_paragraphs(s0, realpath) raise_missing_image_errors = raise_errors # Imports here because of circular dependencies from .latex.latex_preprocess import extract_maths, extract_tabular from .latex.latex_preprocess import latex_preprocessing from .latex.latex_preprocess import replace_equations from .macro_col2 import col_macros, col_macros_prepare_before_markdown from .mark.markd import render_markdown from .preliminary_checks import do_preliminary_checks_and_fixes from .prerender_math import prerender_mathjax if isinstance(s, unicode): msg = 'I expect a str encoded with utf-8, not unicode.' raise_desc(TypeError, msg, s=s) # need to do this before do_preliminary_checks_and_fixes # because of & char s, tabulars = extract_tabular(s) s = do_preliminary_checks_and_fixes(s) # put back tabular, because extract_maths needs to grab them for k, v in tabulars.items(): assert k in s s = s.replace(k, v) # copy all math content, # between $$ and $$ # between various limiters etc. # returns a dict(string, substitution) s, maths = extract_maths(s) # print('maths = %s' % maths) for k, v in maths.items(): if v[0] == '$' and v[1] != '$$': if '\n\n' in v: msg = 'Suspicious math fragment %r = %r' % (k, v) logger.error(maths) logger.error(msg) raise ValueError(msg) s = latex_preprocessing(s) s = '<div style="display:none">Because of mathjax bug</div>\n\n\n' + s # cannot parse html before markdown, because md will take # invalid html, (in particular '$ ciao <ciao>' and make it work) s = s.replace('*}', '\*}') s, mcdpenvs = protect_my_envs(s) # print('mcdpenvs = %s' % maths) s = col_macros_prepare_before_markdown(s) # print(indent(s, 'before markdown | ')) s = render_markdown(s) # print(indent(s, 'after markdown | ')) for k, v in maths.items(): if not k in s: msg = 'Cannot find %r (= %r)' % (k, v) raise_desc(DPInternalError, msg, s=s) def preprocess_equations(x): # this gets mathjax confused x = x.replace('>', '\\gt{}') # need brace; think a<b -> a\lt{}b x = x.replace('<', '\\lt{}') # print('replaced equation %r by %r ' % (x0, x)) return x v = preprocess_equations(v) s = s.replace(k, v) s = replace_equations(s) s = s.replace('\\*}', '*}') # this parses the XML soup = bs(s) other_abbrevs(soup) # need to process tabular before mathjax escape_for_mathjax(soup) # print(indent(s, 'before prerender_mathjax | ')) # mathjax must be after markdown because of code blocks using "$" s = to_html_stripping_fragment(soup) if use_mathjax: s = prerender_mathjax(s, symbols) soup = bs(s) escape_for_mathjax_back(soup) s = to_html_stripping_fragment(soup) # print(indent(s, 'after prerender_mathjax | ')) for k, v in mcdpenvs.items(): # there is this case: # ~~~ # <pre> </pre> # ~~~ s = s.replace(k, v) s = s.replace('<p>DRAFT</p>', '<div class="draft">') s = s.replace('<p>/DRAFT</p>', '</div>') soup = bs(s) mark_console_pres(soup) try: substitute_github_refs(soup, defaults={}) except Exception as e: msg = 'I got an error while substituting github: references.' msg += '\nI will ignore this error because it might not be the fault of the writer.' msg += '\n\n' + indent(str(e), '|', ' error: |') logger.warn(msg) # must be before make_figure_from_figureid_attr() display_files(soup, defaults={}, raise_errors=raise_errors) make_figure_from_figureid_attr(soup) col_macros(soup) fix_subfig_references(soup) library = get_library_from_document(soup, default_library=library) from mcdp_docs.highlight import html_interpret html_interpret(library, soup, generate_pdf=generate_pdf, raise_errors=raise_errors, realpath=realpath) if filter_soup is not None: filter_soup(library=library, soup=soup) embed_images_from_library2(soup=soup, library=library, raise_errors=raise_missing_image_errors) make_videos(soup=soup) if check_refs: check_if_any_href_is_invalid(soup) if getuser() == 'andrea': if MCDPConstants.preprocess_style_using_less: run_lessc(soup) else: logger.warning( 'preprocess_style_using_less=False might break the manual') fix_validation_problems(soup) strip_pre(soup) if MCDPManualConstants.enable_syntax_higlighting: syntax_highlighting(soup) if MCDPManualConstants.enforce_status_attribute: check_status_codes(soup, realpath) if MCDPManualConstants.enforce_lang_attribute: check_lang_codes(soup) # Fixes the IDs (adding 'sec:'); add IDs to missing ones globally_unique_id_part = 'autoid-DO-NOT-USE-THIS-VERY-UNSTABLE-LINK-' + get_md5( s0)[:5] fix_ids_and_add_missing(soup, globally_unique_id_part) check_no_patently_wrong_links(soup) s = to_html_stripping_fragment(soup) s = replace_macros(s) return s
def f0(self, context, request): url_base_internal = self.options.url_base_internal if url_base_internal is not None: if not request.url.startswith(url_base_internal): msg = ( 'Given that url_base_internal is set, I was expecting that all urls' ' would start with it.') raise_desc(Exception, msg, request_url=request.url, url_base_internal=url_base_internal) if '//' in urlparse.urlparse(request.url).path: msg = 'This is an invalid URL with 2 slashes: %s' % request.url response = Response(msg) response.status_int = 500 return response if redir: url = request.url p = urlparse.urlparse(url) url2 = url # only do redirection if we have url_base_internal # The redirection is needed because of https; however # for casual use it is likely https is not set up. if self.options.url_base_internal: if '127.0.0.1' in p.netloc: url2 = url2.replace('127.0.0.1', 'localhost') if not p.path.endswith('.html'): if not p.path.endswith('/'): url2 = url2.replace(p.path, p.path + '/') if url2 != url: logger.info('Context: %s' % context) logger.info('Redirection:\n from: %s\n to: %s' % (url, url2)) raise HTTPFound(url2) if request.authenticated_userid: uid = request.authenticated_userid from mcdp_web.main import WebApp app = WebApp.singleton user_db = app.hi.db_view.user_db if not uid in user_db: msg = 'The user is authenticated as "%s" but no such user in DB.' % uid msg += 'We are logging out the user.' logger.warn(msg) headers = forget(request) raise HTTPFound(location=request.url, headers=headers) try: res = f(self, context, request) except HTTPException: raise except Exception as e: msg = 'While running %s:' % (f.__name__) msg += '\n' + indent(traceback.format_exc(e), ' >') logger.error(msg) raise if isinstance(res, Response): return res check_isinstance(res, dict) try: add_other_fields(self, res, request, context=context) except: logger.error('Error after executing view %s' % f) if isinstance(context, Resource): logger.debug(context_display_in_detail(context)) raise return res
def manual_jobs(context, src_dirs, resources_dirs, out_split_dir, output_file, generate_pdf, stylesheet, stylesheet_pdf, use_mathjax, raise_errors, resolve_references=True, remove=None, filter_soup=None, symbols=None, out_pdf=None, only_refs=False, permalink_prefix=None, compose_config=None, output_crossref=None, do_last_modified=False, wordpress_integration=False, ignore_ref_errors=False, likebtn=None, extra_crossrefs=None): """ src_dirs: list of sources symbols: a TeX preamble (or None) """ # # if symbols is not None: # symbols = open(symbols).read() if stylesheet_pdf is None: stylesheet_pdf = stylesheet # outdir = os.path.dirname(out_split_dir) # XXX filenames = get_markdown_files(src_dirs) if not filenames: msg = "Could not find any file for composing the book." raise Exception(msg) files_contents = [] for i, filename in enumerate(filenames): if is_ignored_by_catkin(filename): logger.debug('Ignoring because of CATKIN_IGNORE: %s' % filename) continue logger.info('adding document %s ' % friendly_path(filename)) docname, _ = os.path.splitext(os.path.basename(filename)) contents = open(filename).read() contents_hash = get_md5(contents)[:8] # because of hash job will be automatically erased if the source changes out_part_basename = '%03d-%s-%s' % (i, docname, contents_hash) job_id = '%s-%s-%s' % (docname, get_md5(filename)[:8], contents_hash) try: source_info = get_source_info(filename) except NoSourceInfo as e: logger.warn('No source info for %s:\n%s' % (filename, e)) source_info = None for d in src_dirs: if filename.startswith(d): break else: msg = "Could not find dir for %s in %s" % (filename, src_dirs) raise Exception(msg) html_contents = context.comp(render_book, generate_pdf=generate_pdf, src_dirs=src_dirs + resources_dirs, data=contents, realpath=filename, use_mathjax=use_mathjax, symbols=symbols, raise_errors=raise_errors, filter_soup=filter_soup, ignore_ref_errors=ignore_ref_errors, job_id=job_id) doc = DocToJoin(docname=out_part_basename, contents=html_contents, source_info=source_info) files_contents.append(tuple(doc)) # compmake doesn't do namedtuples ignore = [] if output_crossref: ignore.append(output_crossref) crossrefs_aug = get_cross_refs(resources_dirs, permalink_prefix, extra_crossrefs, ignore=ignore) bib_files = get_bib_files(src_dirs) logger.debug('Found bib files:\n%s' % "\n".join(bib_files)) if bib_files: bib_contents_aug = job_bib_contents(context, bib_files) entry = DocToJoin(docname='bibtex', contents=bib_contents_aug, source_info=None) files_contents.append(tuple(entry)) if do_last_modified: data_aug = context.comp(make_last_modified, files_contents=files_contents) entry = DocToJoin(docname='last_modified', contents=data_aug, source_info=None) files_contents.append(tuple(entry)) root_dir = src_dirs[0] template = get_main_template(root_dir, resources_dirs) references = OrderedDict() # base_url = 'http://book.duckietown.org/master/duckiebook/pdoc' # for extra_dir in extra_dirs: # res = read_references(extra_dir, base_url, prefix='python:') # references.update(res) # extra = look_for_files(extra_dirs, "*.html") # # for filename in extra: # contents = open(filename).read() # docname = os.path.basename(filename) + '_' + get_md5(filename)[:5] # c = (('unused', docname), contents) # files_contents.append(c) cs = get_md5((crossrefs_aug.get_result()))[:8] joined_aug = context.comp(manual_join, template=template, files_contents=files_contents, stylesheet=None, remove=remove, references=references, resolve_references=resolve_references, crossrefs_aug=crossrefs_aug, permalink_prefix=permalink_prefix, job_id='join-%s' % cs) if compose_config is not None: try: data = yaml.load(open(compose_config).read()) # XXX compose_config_interpreted = ComposeConfig.from_yaml(data) except ValueError as e: msg = 'Cannot read YAML config file %s' % compose_config raise_wrapped(UserError, e, msg, compact=True) else: joined_aug = context.comp(make_composite, compose_config_interpreted, joined_aug) joined_aug = context.comp(mark_errors_and_rest, joined_aug) if likebtn: joined_aug = context.comp(add_likebtn, joined_aug, likebtn) if wordpress_integration: joined_aug = context.comp(add_related, joined_aug) if output_file is not None: context.comp(write, joined_aug, output_file) if out_split_dir is not None: joined_aug_with_html_stylesheet = context.comp(add_style, joined_aug, stylesheet) extra_panel_content = context.comp(get_extra_content, joined_aug_with_html_stylesheet) id2filename_aug = context.comp_dynamic( create_split_jobs, data_aug=joined_aug_with_html_stylesheet, mathjax=True, preamble=symbols, extra_panel_content=extra_panel_content, output_dir=out_split_dir, nworkers=0, output_crossref=output_crossref, permalink_prefix=permalink_prefix, only_refs=only_refs) if not only_refs: context.comp(write_errors_and_warnings_files, id2filename_aug, out_split_dir) context.comp(write_manifest_html, out_split_dir) if out_pdf is not None: joined_aug_with_pdf_stylesheet = context.comp(add_style, joined_aug, stylesheet_pdf) prerendered = context.comp(prerender, joined_aug_with_pdf_stylesheet, symbols=symbols) pdf_data = context.comp(render_pdf, prerendered) context.comp(write_data_to_file, pdf_data, out_pdf) context.comp(write_manifest_pdf, out_pdf)
def write_errors_and_warnings_files(aug, d): if aug.has_result(): id2filename = aug.get_result() else: id2filename = {} # print('id2filename: %s' % sorted(id2filename)) assert isinstance(aug, AugmentedResult) aug.update_refs(id2filename) header = get_notes_panel(aug) manifest = [] nwarnings = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_WARNING)) fn = os.path.join(d, 'warnings.html') html = html_list_of_notes(aug, MCDPManualConstants.NOTE_TAG_WARNING, 'warnings', 'warning', header=header) # update_refs_('warnings', html, id2filename) write_data_to_file(str(html), fn, quiet=True) if nwarnings: manifest.append( dict(display='%d warnings' % nwarnings, filename='warnings.html')) msg = 'There were %d warnings: %s' % (nwarnings, fn) logger.warn(msg) ntasks = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_TASK)) fn = os.path.join(d, 'tasks.html') html = html_list_of_notes(aug, MCDPManualConstants.NOTE_TAG_TASK, 'tasks', 'task', header=header) # update_refs_('tasks', html, id2filename) write_data_to_file(str(html), fn, quiet=True) if nwarnings: manifest.append( dict(display='%d tasks' % ntasks, filename='tasks.html')) msg = 'There are %d open tasks: %s' % (ntasks, fn) logger.info(msg) nerrors = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_ERROR)) fn = os.path.join(d, 'errors.html') html = html_list_of_notes(aug, MCDPManualConstants.NOTE_TAG_ERROR, 'errors', 'error', header=header) # update_refs_('tasks', html, id2filename) write_data_to_file(str(html), fn, quiet=True) if nerrors: manifest.append( dict(display='%d errors' % nerrors, filename='errors.html')) msg = 'I am sorry to say that there were %d errors.\n\nPlease see: %s' % ( nerrors, fn) logger.error('\n\n\n' + indent(msg, ' ' * 15) + '\n\n') fn = os.path.join(d, 'errors_and_warnings.manifest.yaml') write_data_to_file(yaml.dump(manifest), fn, quiet=False) fn = os.path.join(d, 'errors_and_warnings.pickle') res = AugmentedResult() res.merge(aug) write_data_to_file(pickle.dumps(res), fn, quiet=False)