def read_SchemaHash_SER_DIR(self, schema, fh):
check_isinstance(schema, SchemaHash)
res = {}
hint = self.get_hint(schema)
if hint.pattern == '%':
seq = fh.items()
else:
seq = recursive_list_dir(fh, hint)
seq = list(seq)
msg = 'read_SchemaHash_SER_DIR\n'
msg += indent(fh.tree(0), 'files ') + '\n'
msg += 'pattern is %r\n' % hint.pattern
msg += 'list is %s\n' % seq
for filename, data in seq:
try:
k = hint.key_from_filename(filename)
except NotKey:
logger.warning('Ignoring file "%s": not a key' % filename)
continue
assert not k in res, (k, hint.pattern, filename)
try:
res[k] = self.interpret_hierarchy_(schema.prototype, data)
except IncorrectFormat as e:
msg = 'While interpreting filename "%s":' % filename
raise_wrapped(IncorrectFormat,
e,
msg,
compact=True,
exc=sys.exc_info())
return res
def add_edit_links(soup, filename):
# is this is in a repo?
try:
repo_root = get_repo_root(filename)
except ValueError:
return
try:
repo_info = get_repo_information(repo_root)
except RepoInfoException as e:
logger.warning(str(e))
return
branch = repo_info['branch']
# commit = repo_info['commit']
org = repo_info['org']
repo = repo_info['repo']
relpath = os.path.relpath(filename, repo_root)
repo_base = 'https://github.com/%s/%s' % (org, repo)
blob_base = repo_base + '/blob/%s' % (branch)
edit_base = repo_base + '/edit/%s' % (branch)
blob_url = blob_base + "/" + relpath
edit_url = edit_base + "/" + relpath
for h in soup.findAll(['h1', 'h2', 'h3', 'h4']):
h.attrs['github-edit-url'] = edit_url
h.attrs['github-blob-url'] = blob_url
def interpret_SchemaList_SER_DIR(self, schema, fh):
if not isinstance(fh, ProxyDirectory):
msg = 'I expected a directory, not a file.'
raise_desc(ValueError, msg, schema=schema, fh=fh)
check_isinstance(schema, SchemaList)
found = []
for filename in fh:
try:
i = int(filename)
except ValueError:
msg = 'Filename "%s" does not represent a number.' % filename
logger.warning(msg + ' -- ignoring')
#raise_incorrect_format(msg, schema, fh.tree())
found.append(i)
if not found:
return []
max_found = max(found)
n = max_found + 1
res = [None] * n
if len(found) != n:
msg = 'Incomplete data. Found %s' % found
raise_incorrect_format(msg, schema, fh.tree())
for filename in fh:
try:
i = int(filename)
except ValueError:
continue
if not (0 <= i < n):
assert False
res[i] = self.interpret_hierarchy_(schema.prototype, fh[filename])
return res
def _update_file(self, f, from_search_dir=None):
""" from_search_dir: from whence we arrived at this file. """
if not os.path.exists(f):
msg = 'File does not exist:'
msg += '\n filename: %s' % f
msg += '\n from_search_dir: %s' % from_search_dir
raise ValueError(msg)
basename = os.path.basename(f)
check_isinstance(basename, str)
# This will fail because then in pyparsing everything is unicode
# import codecs
# data = codecs.open(f, encoding='utf-8').read()
data = open(f).read()
realpath = os.path.realpath(f)
res = dict(data=data,
realpath=realpath,
path=f,
from_search_dir=from_search_dir)
strict = False
if basename in self.file_to_contents:
realpath1 = self.file_to_contents[basename]['realpath']
path1 = self.file_to_contents[basename]['path']
expected = from_search_dir == 'mcdp-web'
if not expected:
if res['realpath'] == realpath1:
msg = 'File %r reached twice.' % basename
msg += '\n now from %s' % from_search_dir
msg += '\n prev from %s' % self.file_to_contents[basename][
'from_search_dir']
if not strict:
pass
# logger.warning(msg + "\n" +
# format_obs(dict(path1=path1,
# path2=res['path'])))
else:
raise_desc(DPSemanticError,
msg,
path1=path1,
path2=res['path'])
else:
msg = 'Found duplicated file %r.' % basename
if not strict:
if MCDPConstants.log_duplicates:
logger.warning(msg + "\n" + format_obs(
dict(path1=realpath1, path2=res['realpath'])))
else:
raise_desc(DPSemanticError,
msg,
path1=realpath1,
path2=res['realpath'])
assert isinstance(basename, str), basename
self.file_to_contents[basename] = res
def delete_file(self, basename):
''' Deletes a file '''
if not basename in self.file_to_contents:
msg = 'Filename %r does not exist in this library.' % basename
raise ValueError(msg)
realpath = self.file_to_contents[basename]['realpath']
if not os.path.exists(realpath):
msg = 'Filename %r -> %r already deleted.' % (basename, realpath)
logger.warning(msg)
os.unlink(realpath)
def embed_pdf_image(tag, resolve, density):
assert tag.name == 'img'
assert tag.has_attr('src')
#print('!!embedding %s' % str(tag))
#raise Exception(str(tag))
# load pdf data
data_pdf = resolve(tag['src'])
if data_pdf is None:
add_class(tag, 'missing-image')
return
# convert PDF to PNG
# density = pixels per inch
data_png = png_from_pdf(data_pdf, density=density)
# get PNG image size in pixels
width_px, height_px = get_pixel_width_height_of_png(data_png)
# compute what was the original width of PDF in points
width_in = width_px / float(density)
height_in = height_px / float(density)
latex_options = tag.get('latex-options', '')
props = parse_includegraphics_option_string(latex_options)
if 'height' in props:
logger.warning('Cannot deal with "height" yet: latex_options = %s' %
latex_options)
if 'scale' in props:
scale = float(props['scale'])
use_width_in = width_in * scale
use_height_in = height_in * scale
elif 'width' in props:
try:
use_width_in = get_length_in_inches(props['width'])
except ValueError as e:
logger.error('Cannot interpret %s: %s' % (latex_options, e))
use_width_in = 5.0
ratio = height_in / width_in
use_height_in = use_width_in * ratio
else:
use_width_in = width_in
use_height_in = height_in
# Add it before so that we can override
add_style(tag,
after=False,
width='%sin' % use_width_in,
height='%sin' % use_height_in)
tag['size_in_pixels'] = '%s, %s' % (width_px, height_px)
# encode
tag['src'] = data_encoded_for_src(data_png, 'png')
def _add_search_dir(self, d):
""" Adds the directory to the search directory list. """
pattern = ['*.' + ext for ext in MCDPLibrary.all_extensions]
files_mcdp = locate_files(directory=d,
pattern=pattern,
followlinks=True)
for f in files_mcdp:
assert isinstance(f, str)
if os.path.islink(f):
if not os.path.exists(f):
msg = 'Ignoring broken link %s' % f
logger.warning(msg)
continue
self._update_file(f, from_search_dir=d)
def remove_spurious(output_dir, filenames):
ignore = [
'link.html', 'toc.html', 'errors.html', 'warnings.html', 'tasks.html',
'crossref.html'
]
found = os.listdir(output_dir)
for f in found:
if not f.endswith('.html'):
continue
if f in ignore:
continue
if f not in filenames:
fn = os.path.join(output_dir, f)
if 'SPURIOUS' in open(fn).read():
# already marked as spurious
continue
msg = 'I found a spurious file from earlier compilations: %s' % fn
# msg += '(%s not in %s) ' % (f, filenames)
logger.warning(msg)
soup = read_html_doc_from_file(fn)
e = soup.find('section')
if e is not None and 'id' in e.attrs:
if False:
id_ = e.attrs['id'].replace(':section', '')
if 'autoid' not in id_:
id_ = remove_prefix(id_)
url = 'http://purl.org/dt/master/' + id_
OTHER = ((
'<p>Maybe try this link to find the version on master '
'(no guarantees): <a href="%s">%s</a></p>') %
(url, url))
OTHER += '\n<p>If that does not work, the section was renamed.</p>'
else:
OTHER = ''
else:
OTHER = ''
else:
OTHER = ''
data = spurious.replace('OTHER', OTHER)
write_data_to_file(data, fn, quiet=True)
def go(path):
db = StorageFilesystem(path, compress=True)
args = ['failed']
cq = CacheQueryDB(db)
context = Context(db)
if not list(db.keys()):
msg = 'Compmake DB is empty'
logger.error(msg)
else:
job_list = parse_job_list(args, context=context, cq=cq)
s = ""
if job_list:
job_list = job_list[:2]
s += 'Running on host: %s' % hostname
s += "\nJob failed in path %s" % path
for job_id in job_list:
if job_cache_exists(job_id, db):
cache = get_job_cache(job_id, db)
status = Cache.state2desc[cache.state]
s += "\nFailure of job %s" % job_id
if cache.state in [Cache.FAILED, Cache.BLOCKED]:
why = str(cache.exception).strip()
else:
why = 'No why for job done.'
s += '\n' + "```\n" + why + "\n```"
s += '\n\n'
else:
logger.warning('no cache for %s' % job_id)
s += '\n@censi'
s += '\n@jacopo'
s += '\n@paull'
s += '\n@walter'
s += '\n@daniele'
print(s)
slack.chat.post_message(channel, s, link_names=1)
else:
s = 'Everything is fine'
# slack.chat.post_message(channel, s)
logger.info('No jobs found')
def process_ns(t):
s = t + ''
# logger.debug('Handling %r' % t)
marker = MCDPConstants.placeholder_marker_start
marker2 = MCDPConstants.placeholder_marker_end
if marker not in s:
return
i = s.index(marker)
try:
n = i + s[i:].index(marker2)
except ValueError:
msg = 'I found the substring "![" and so I thought there would '
msg += 'be a closing "]"; however, I could not find one.'
logger.warning(msg)
logger.warning('In string: %r.' % s)
logger.warning('Above: %s' % t.parent)
return
before = s[:i]
inside = s[i + len(marker):n]
after = s[n + len(marker2):]
# logger.debug('before = %r inside = %r after = %r' % (before, inside, after))
p = Tag(name='span')
p.attrs['class'] = 'placeholder'
p.append(inside)
t.replace_with(p)
p.insert_before(before)
p.insert_after(after)
def get_source(db_view, repo_name, shelf_name, library_name, spec_name,
thing_name):
from mcdp_hdb_mcdp.library_view import get_soft_match
shelf = db_view.repos[repo_name].shelves[shelf_name]
library = shelf.libraries[library_name]
things = library.things.child(spec_name)
try:
match = get_soft_match(thing_name, list(things))
except KeyError:
msg = 'Soft match failed: Could not find %r in %s.' % (thing_name,
spec_name)
available = sorted(things)
if available:
msg += ("\n Available %s: %s." %
(spec_name, format_list(sorted(available))))
else:
msg += "\n None available."
raise_desc(DPSemanticError, msg)
else:
if match != thing_name:
if MCDPConstants.allow_soft_matching:
logger.warning('Soft matching %r to %r (deprecated)' %
(match, thing_name))
else:
msg = 'Found case in which the user relies on soft matching (%r to refer to %r).' % (
thing_name, match)
raise DPSemanticError(msg)
# TODO: add warning
data = things[match]
spec = specs[spec_name]
basename = match + '.' + spec.extension
realpath = ('%s in library %r in shelf %r in repo %r' %
(basename, library_name, shelf_name, repo_name))
return dict(data=data, realpath=realpath)
def run_lessc(soup):
""" Runs pre-processor on all styles nodes """
for style in soup.select('style'):
if 'Math' in style.attrs.get('id', ''):
continue
s1 = style.string
s1 = unescape_entities(s1)
s1 = s1.replace('AND', '&')
s1 = s1.encode('utf-8')
try:
s2 = lessc_string(s1)
except LesscError as e:
msg = 'Could not convert string using less (ignored)'
msg += '\n' + indent(e, '>> ')
logger.warning(msg)
continue
s2 = unicode(s2, 'utf-8')
# print indent(s2, 'less |')
s2 = '/* preprocessed with less */\n' + s2
style.string = s2
style['type'] = 'text/css'
def extract_img_to_file_(soup, savefile, tagname, attrname):
n = 0
tot = 0
for tag in soup.select(tagname):
tot += 1
if not attrname in tag.attrs:
msg = 'No attr %r found for tag %s' % (attrname, tag)
logger.warning(msg)
continue
src = tag.attrs[attrname]
if not src.startswith('data:'):
continue
mime, data = get_mime_data_from_base64_string(src)
# now we should make up the data
if tag.has_attr('id'):
basename = tag['id']
else:
md5 = get_md5(data)
basename = 'data-from-%s-%s' % (tagname, md5)
# Guess extension
ext = get_ext_for_mime(mime)
filename = basename + '.' + ext
# src = "%s" % filename
# ask what we should be using
# print('saving file %s with %d data' % (filename, len(data)))
use_src = savefile(filename, data)
check_isinstance(use_src, str)
tag[attrname] = use_src
n += 1
if False:
logger.debug(
('extract_img_to_file: extracted %d/%d images from %r tags, '
' attribute %r.') % (n, tot, tagname, attrname))
return n
def __init__(self, options, settings):
self.options = options
self.settings = settings
# display options
for k in sorted(self.options._values):
v = self.options._values[k]
logger.debug('option %20s = %r ' % (k, v))
# display settings
for k in sorted(self.settings):
v = self.settings[k]
logger.debug('setting %20s = %r ' % (k, v))
WebApp.singleton = self
dirname = options.libraries
if dirname is None:
package = dir_from_package_name('mcdp_data')
default_libraries = os.path.join(package, 'libraries')
msg = (
'Option "-d" not passed, so I will open the default libraries '
'shipped with PyMCDP. These might not be writable depending on your setup.'
)
logger.info(msg)
dirname = default_libraries
self.dirname = dirname
AppVisualization.__init__(self)
AppQR.__init__(self)
AppSolver.__init__(self)
AppInteractive.__init__(self)
AppSolver2.__init__(self)
AppEditorFancyGeneric.__init__(self)
WebAppImages.__init__(self)
# name -> dict(desc: )
self.views = {}
self.exceptions = []
self.add_model_view('syntax', 'Source code display')
self.add_model_view('edit_fancy', 'Editor')
# self.add_model_view('edit', 'Simple editor for IE')
self.add_model_view('solver2', desc='Solver interface')
self.add_model_view('ndp_graph', 'NDP Graph representation')
self.add_model_view('ndp_repr', 'NDP Text representation')
self.add_model_view('dp_graph', 'DP graph representation')
self.add_model_view('dp_tree', 'DP tree representation')
self.add_model_view('images', 'Other image representations')
self.add_model_view('solver', 'Graphical solver [experimental]')
# csfr_token -> Session
self.sessions = OrderedDict()
config_repos = yaml_load(self.options.repos_yaml)
logger.info('Config:\n' + indent(self.options.repos_yaml, '>'))
logger.info(config_repos)
instance = self.options.instance
#root= 'out/root'
root = create_tmpdir('HostInstance_root')
logger.debug('Tmp dir: %s' % root)
if not 'local' in config_repos:
config_repos['local'] = {}
if not 'remote' in config_repos:
config_repos['remote'] = {}
# add the bundled repository
bundled_repo_dir = os.path.join(dir_from_package_name('mcdp_data'),
'bundled.mcdp_repo')
config_repos['local']['bundled'] = bundled_repo_dir
self.hi = HostInstance(instance=instance,
upstream='master',
root=root,
repo_git=config_repos['remote'],
repo_local=config_repos['local'])
if self.options.allow_anonymous_write:
logger.warning(
'Note: because allow_anonymous_write anonymous users can admin the repos.'
)
self.hi.set_local_permission_mode()
from pyramid.security import Allow, Everyone, ALL_PERMISSIONS
MCDPResourceRoot.__acl__.append((Allow, Everyone, ALL_PERMISSIONS))
def render_complete(library,
s,
raise_errors,
realpath,
generate_pdf=False,
check_refs=False,
use_mathjax=True,
filter_soup=None,
symbols=None):
"""
Transforms markdown into html and then renders the mcdp snippets inside.
s: a markdown string with embedded html snippets
Returns an HTML string; not a complete document.
filter_soup(library, soup)
"""
s0 = s
check_good_use_of_special_paragraphs(s0, realpath)
raise_missing_image_errors = raise_errors
# Imports here because of circular dependencies
from .latex.latex_preprocess import extract_maths, extract_tabular
from .latex.latex_preprocess import latex_preprocessing
from .latex.latex_preprocess import replace_equations
from .macro_col2 import col_macros, col_macros_prepare_before_markdown
from .mark.markd import render_markdown
from .preliminary_checks import do_preliminary_checks_and_fixes
from .prerender_math import prerender_mathjax
if isinstance(s, unicode):
msg = 'I expect a str encoded with utf-8, not unicode.'
raise_desc(TypeError, msg, s=s)
# need to do this before do_preliminary_checks_and_fixes
# because of & char
s, tabulars = extract_tabular(s)
s = do_preliminary_checks_and_fixes(s)
# put back tabular, because extract_maths needs to grab them
for k, v in tabulars.items():
assert k in s
s = s.replace(k, v)
# copy all math content,
# between $$ and $$
# between various limiters etc.
# returns a dict(string, substitution)
s, maths = extract_maths(s)
# print('maths = %s' % maths)
for k, v in maths.items():
if v[0] == '$' and v[1] != '$$':
if '\n\n' in v:
msg = 'Suspicious math fragment %r = %r' % (k, v)
logger.error(maths)
logger.error(msg)
raise ValueError(msg)
s = latex_preprocessing(s)
s = '<div style="display:none">Because of mathjax bug</div>\n\n\n' + s
# cannot parse html before markdown, because md will take
# invalid html, (in particular '$ ciao <ciao>' and make it work)
s = s.replace('*}', '\*}')
s, mcdpenvs = protect_my_envs(s)
# print('mcdpenvs = %s' % maths)
s = col_macros_prepare_before_markdown(s)
# print(indent(s, 'before markdown | '))
s = render_markdown(s)
# print(indent(s, 'after markdown | '))
for k, v in maths.items():
if not k in s:
msg = 'Cannot find %r (= %r)' % (k, v)
raise_desc(DPInternalError, msg, s=s)
def preprocess_equations(x):
# this gets mathjax confused
x = x.replace('>', '\\gt{}') # need brace; think a<b -> a\lt{}b
x = x.replace('<', '\\lt{}')
# print('replaced equation %r by %r ' % (x0, x))
return x
v = preprocess_equations(v)
s = s.replace(k, v)
s = replace_equations(s)
s = s.replace('\\*}', '*}')
# this parses the XML
soup = bs(s)
other_abbrevs(soup)
# need to process tabular before mathjax
escape_for_mathjax(soup)
# print(indent(s, 'before prerender_mathjax | '))
# mathjax must be after markdown because of code blocks using "$"
s = to_html_stripping_fragment(soup)
if use_mathjax:
s = prerender_mathjax(s, symbols)
soup = bs(s)
escape_for_mathjax_back(soup)
s = to_html_stripping_fragment(soup)
# print(indent(s, 'after prerender_mathjax | '))
for k, v in mcdpenvs.items():
# there is this case:
# ~~~
# <pre> </pre>
# ~~~
s = s.replace(k, v)
s = s.replace('<p>DRAFT</p>', '<div class="draft">')
s = s.replace('<p>/DRAFT</p>', '</div>')
soup = bs(s)
mark_console_pres(soup)
try:
substitute_github_refs(soup, defaults={})
except Exception as e:
msg = 'I got an error while substituting github: references.'
msg += '\nI will ignore this error because it might not be the fault of the writer.'
msg += '\n\n' + indent(str(e), '|', ' error: |')
logger.warn(msg)
# must be before make_figure_from_figureid_attr()
display_files(soup, defaults={}, raise_errors=raise_errors)
make_figure_from_figureid_attr(soup)
col_macros(soup)
fix_subfig_references(soup)
library = get_library_from_document(soup, default_library=library)
from mcdp_docs.highlight import html_interpret
html_interpret(library,
soup,
generate_pdf=generate_pdf,
raise_errors=raise_errors,
realpath=realpath)
if filter_soup is not None:
filter_soup(library=library, soup=soup)
embed_images_from_library2(soup=soup,
library=library,
raise_errors=raise_missing_image_errors)
make_videos(soup=soup)
if check_refs:
check_if_any_href_is_invalid(soup)
if getuser() == 'andrea':
if MCDPConstants.preprocess_style_using_less:
run_lessc(soup)
else:
logger.warning(
'preprocess_style_using_less=False might break the manual')
fix_validation_problems(soup)
strip_pre(soup)
if MCDPManualConstants.enable_syntax_higlighting:
syntax_highlighting(soup)
if MCDPManualConstants.enforce_status_attribute:
check_status_codes(soup, realpath)
if MCDPManualConstants.enforce_lang_attribute:
check_lang_codes(soup)
# Fixes the IDs (adding 'sec:'); add IDs to missing ones
globally_unique_id_part = 'autoid-DO-NOT-USE-THIS-VERY-UNSTABLE-LINK-' + get_md5(
s0)[:5]
fix_ids_and_add_missing(soup, globally_unique_id_part)
check_no_patently_wrong_links(soup)
s = to_html_stripping_fragment(soup)
s = replace_macros(s)
return s
def note_warning(self, msg, locations=None):
logger.warning(msg)
self.add_note(NoteWarning(msg, locations, stacklevel=1))
def makefigure(inside, opt, asterisk): # @UnusedVariable
# align = opt # @UnusedVariable
# print('makefigure inside = %r' % inside)
def subfloat_replace(args, opts):
contents = args[0]
caption = opts[0]
check_isinstance(contents, str)
if caption is None:
label = None
else:
caption, label = get_s_without_label(caption, labelprefix="fig:")
if label is None:
caption, label = get_s_without_label(caption,
labelprefix="subfig:")
if label is not None and not label.startswith('subfig:'):
msg = 'Subfigure labels should start with "subfig:"; found %r.' % (
label)
label = 'sub' + label
msg += ' I will change to %r.' % label
logger.debug(msg)
# we need to make up an ID
if label is None:
label = 'subfig:' + get_md5(contents)
# print('making up label %r' % label)
# if label is not None:
idpart = ' id="%s"' % label
# else:
# idpart = ""
if caption is None:
caption = 'no subfloat caption'
res = '<figure class="subfloat"%s>%s<figcaption>%s</figcaption></figure>' % (
idpart, contents, caption)
return res
inside = substitute_command_ext(inside,
'subfloat',
subfloat_replace,
nargs=1,
nopt=1)
class Tmp:
label = None
def sub_caption(args, opts):
assert not opts and len(args) == 1
x, Tmp.label = get_s_without_label(args[0], labelprefix="fig:")
res = '<figcaption>' + x + "</figcaption>"
# print('caption args: %r, %r' % (args, opts))
return res
inside = substitute_command_ext(inside,
'caption',
sub_caption,
nargs=1,
nopt=0)
# print('makefigure inside without caption = %r' % inside)
if '\\caption' in inside:
msg = 'Whoops: %r' % inside
logger.warning(msg)
if Tmp.label is None:
Tmp.label = 'fig:' + get_md5(inside)
#print('making up label %r' % Tmp.label)
# if Tmp.label is not None:
idpart = ' id="%s"' % Tmp.label
# else:
# idpart = ""
res = '<figure%s>%s</figure>' % (idpart, inside)
return res
def locate_files(directory, pattern, followlinks=True,
include_directories=False,
include_files=True,
normalize=True,
ignore_patterns=None):
"""
pattern is either a string or a sequence of strings
ignore_patterns = ['*.bak']
normalize = uses realpath
"""
t0 = time.time()
if ignore_patterns is None:
ignore_patterns = MCDPConstants.locate_files_ignore_patterns
if isinstance(pattern, str):
patterns = [pattern]
else:
patterns = list(pattern)
for p in patterns:
check_isinstance(p, str)
# directories visited
visited = set()
# print('locate_files %r %r' % (directory, pattern))
filenames = []
def matches_pattern(x):
return any(fnmatch.fnmatch(x, p) for p in patterns)
def should_ignore_resource(x):
return any(fnmatch.fnmatch(x, ip) for ip in ignore_patterns)
def accept_dirname_to_go_inside(root, d):
if should_ignore_resource(d):
return False
dd = os.path.realpath(os.path.join(root, d))
if dd in visited:
return False
visited.add(dd)
return True
def accept_dirname_as_match(d):
return include_directories and \
not should_ignore_resource(d) and \
matches_pattern(d)
def accept_filename_as_match(fn):
return include_files and \
not should_ignore_resource(fn) and \
matches_pattern(fn)
ntraversed = 0
for root, dirnames, files in os.walk(directory, followlinks=followlinks):
ntraversed += 1
dirnames[:] = [_ for _ in dirnames if accept_dirname_to_go_inside(root, _)]
for f in files:
if accept_filename_as_match(f):
filename = os.path.join(root, f)
filenames.append(filename)
for d in dirnames:
if accept_dirname_as_match(d):
filename = os.path.join(root, d)
filenames.append(filename)
if normalize:
real2norm = defaultdict(lambda: [])
for norm in filenames:
real = os.path.realpath(norm)
real2norm[real].append(norm)
# print('%s -> %s' % (real, norm))
for k, v in real2norm.items():
if len(v) > 1:
msg = 'In directory:\n\t%s\n' % directory
msg += 'I found %d paths that refer to the same file:\n'
for n in v:
msg += '\t%s\n' % n
msg += 'refer to the same file:\n\t%s\n' % k
msg += 'I will silently eliminate redundancies.'
logger.warning(v)
filenames = list(real2norm.keys())
seconds = time.time() - t0
if seconds > 5:
n = len(filenames)
nuniques = len(set(filenames))
logger.debug('%.4f s for locate_files(%s,%s): %d traversed, found %d filenames (%d uniques)' %
(seconds, directory, pattern, ntraversed, n, nuniques))
return filenames
def embed_pdf_image(tag, resolve, density, raise_on_error, res, location):
assert tag.name == 'img'
assert tag.has_attr('src')
# print('!!embedding %s' % str(tag))
# raise Exception(str(tag))
# load pdf data
src = tag['src']
if src.startswith('http'):
msg = 'I will not embed remote files, such as %s: ' % src
logger.warning(msg)
found = resolve(src)
if found is None:
msg = 'Could not find PDF file %r.' % src
if raise_on_error:
raise Exception(msg) # xxx
else:
# note_error2(tag, 'Resource error', msg, ['missing-image'])
res.note_error(msg, HTMLIDLocation.for_element(tag, location))
return
data_pdf = found['data']
_realpath = found['realpath']
# convert PDF to PNG
# density = pixels per inch
try:
data_png = png_from_pdf(data_pdf, density=density)
except ConversionError as e:
msg = 'I was not able to convert the PDF "%s" to PNG.' % tag['src']
if raise_on_error:
raise_wrapped(ConversionError, e, msg, compact=True)
else:
# note_error2(tag, 'Conversion error', msg, [])
res.note_error(msg, HTMLIDLocation.for_element(tag, location))
return
# get PNG image size in pixels
width_px, height_px = get_pixel_width_height_of_png(data_png)
# compute what was the original width of PDF in points
width_in = width_px / float(density)
height_in = height_px / float(density)
latex_options = tag.get('latex-options', '')
props = parse_includegraphics_option_string(latex_options)
if 'height' in props:
msg = ('Cannot deal with "height" yet: latex_options = %s' %
latex_options)
res.note_warning(msg, HTMLIDLocation.for_element(tag, location))
if 'scale' in props:
scale = float(props['scale'])
use_width_in = width_in * scale
use_height_in = height_in * scale
elif 'width' in props:
try:
use_width_in = get_length_in_inches(props['width'])
except ValueError as e:
logger.error('Cannot interpret %s: %s' % (latex_options, e))
use_width_in = 5.0
ratio = height_in / width_in
use_height_in = use_width_in * ratio
else:
use_width_in = width_in
use_height_in = height_in
# Add it before so that we can override
add_style(tag,
after=False,
width='%sin' % use_width_in,
height='%sin' % use_height_in)
tag['size_in_pixels'] = '%s, %s' % (width_px, height_px)
# encode
tag['src'] = data_encoded_for_src(data_png, 'png')
