def get_manual_contents(srcdir):
root = os.getcwd()
directory = os.path.join(root, srcdir)
if not os.path.exists(directory):
msg = 'Expected directory %s' % directory
raise Exception(msg)
pattern = '*.md'
filenames = locate_files(directory,
pattern,
followlinks=True,
include_directories=False,
include_files=True,
normalize=False)
ok = []
for fn in filenames:
fn = os.path.relpath(fn, root)
if 'exclude' in fn:
logger.info('Excluding file %r because of string "exclude" in it' %
fn)
continue
ok.append(fn)
filenames = natsorted(ok)
for f in filenames:
docname, _extension = os.path.splitext(os.path.basename(f))
yield 'manual', docname
def process_svg_files(root, out, preamble):
logger.info('root = %s' % root)
logger.info('out = %s' % out)
logger.info('preamble = %s' % preamble)
if not os.path.exists(out):
os.makedirs(out)
logger.info('Looking for *.svg files...')
svgs = locate_files(root, '*.svg')
logger.info('%d found in %s' % (len(svgs), friendly_path(root)))
errors = []
for f in svgs:
dirname = os.path.dirname(f)
basename, _ = os.path.splitext(os.path.basename(f))
target = os.path.join(out, basename + '.pdf')
target = os.path.join(dirname, basename + '.pdf')
if not needs_remake(f, target):
msg = 'The target %r is up to date.' % target
logger.info(msg)
else:
msg = 'Will build target %r.' % target
logger.info(msg)
tmpdir = create_tmpdir('svg-%s' % basename)
try:
process_svg_file(f, target, preamble, tmpdir)
except Exception as e:
logger.error(e)
errors.append(e)
def get_all_files(dirname):
all_files = locate_files(dirname, '*', include_directories=True)
res = []
for x in all_files:
res.append(x)
if os.path.isdir(x):
res.append(x + '/')
return sorted(res)
def collect_by_extension(d):
fs = locate_files(d, '*')
ext2filename = defaultdict(list)
for filename in fs:
basename = os.path.basename(filename)
_, ext = os.path.splitext(basename)
ext2filename[ext] = []
ext2filename[ext].append(filename)
return ext2filename
def get_main_template(root_dir, resources_dirs):
fn = os.path.join(root_dir, MCDPManualConstants.main_template)
if os.path.exists(fn):
return parse_main_template(fn)
for d in resources_dirs:
fns = locate_files(d, MCDPManualConstants.main_template)
for fn in fns:
return parse_main_template(fn)
msg = 'Could not find template {}'.format(fn)
raise ValueError(msg)
def get_images(dirname, exts=None):
if exts is None:
exts = MCDPConstants.exts_for_icons
""" Returns a dict from lowercase basename to realpath """
pattern = ['*.%s' % ext for ext in exts]
allfiles = {}
files = locate_files(dirname, pattern, followlinks=True, normalize=False)
for f in files:
basename = os.path.basename(f)
basename = basename.lower()
allfiles[basename] = f
return allfiles
def _add_search_dir(self, d):
""" Adds the directory to the search directory list. """
pattern = ['*.' + ext for ext in MCDPLibrary.all_extensions]
files_mcdp = locate_files(directory=d,
pattern=pattern,
followlinks=True)
for f in files_mcdp:
assert isinstance(f, str)
if os.path.islink(f):
if not os.path.exists(f):
msg = 'Ignoring broken link %s' % f
logger.warning(msg)
continue
self._update_file(f, from_search_dir=d)
def find_libraries(d0):
'''
Finds <name>.mcdplib.
returns dict ID -> path
'''
dirs = locate_files(d0,
"*." + MCDPConstants.library_extension,
followlinks=False,
include_directories=True,
include_files=False)
res = {}
for dirname in dirs:
if dirname == '.':
dirname = os.path.realpath(dirname)
library_name = os.path.splitext(os.path.basename(dirname))[0]
assert_good_plain_identifier(library_name, 'libraries')
res[library_name] = dirname
return res
def define_tests(context):
""" Define other mcdp_lang_tests """
filenames = []
folder = dir_from_package_name('mcdp_lang_tests.ok')
filenames.extend(locate_files(folder, '*.mcdp'))
context = context.child('examples')
found = set()
def get_unique_basename(f):
orig = os.path.splitext(os.path.basename(f))[0]
if orig in found:
for i in range(10):
bn = '%s_%d' % (orig, i + 1)
if not bn in found:
return bn
assert False, (orig, found)
else:
found.add(orig)
return orig
# if False:
# folder = os.path.join(dir_from_package_name('mocdp'), '../../libraries')
# examples2 = list(locate_files(folder, '*.mcdp'))
# print('Other files found: %s' % examples2)
# filenames.extend(examples2)
# else:
examples2 = []
for f in filenames:
basename = get_unique_basename(f)
print('defining %s - %s' % (basename, f))
if f in examples2:
contents = open(f).read()
if '...' in contents:
tests = [assert_semantic_error_fn]
else:
tests = [assert_parsable_to_connected_ndp_fn]
else:
tests = get_marked_tests(f)
define_test_for(context, f, basename, tests)
def get_related(res):
filenames = locate_files('.', '*.related.yaml')
users = {}
posts = {}
for f in filenames:
location = LocalFile(f)
contents = open(f).read()
data = yaml.load(contents)
# print(indent(data, os.path.basename(f) + ' > '))
if not isinstance(data, dict):
msg = 'YAML is None'
res.note_error(msg, location)
continue
if not 'users' in data or not 'posts' in data:
msg = 'Could not find keys in dict: %s' % list(data)
res.note_error(msg, location)
users.update(data['users'])
posts.update(data['posts'])
return posts, users
def look_for_files(srcdirs, pattern):
"""
Excludes files with "excludes" in the name.
"""
results = []
results_absolute = set()
for d0 in srcdirs:
d = expand_all(d0)
if not os.path.exists(d):
msg = 'Could not find directory %r' % d
msg += '\nSearching from directory %r' % os.getcwd()
raise Exception(msg)
filenames = locate_files(d,
pattern,
followlinks=True,
include_directories=False,
include_files=True,
normalize=False)
ok = []
for fn in filenames:
fn0 = os.path.realpath(fn)
if 'exclude' in fn0:
logger.info(
'Excluding file %r because of string "exclude" in it' % fn)
else:
if fn0 in results_absolute:
logger.debug('Reached the file %s twice' % fn0)
pass
else:
results_absolute.add(fn0)
ok.append(fn)
results.extend(natsorted(ok))
logger.info('Found %d files with pattern %s in %s' %
(len(results), pattern, srcdirs))
return results
def find_libraries(self, dirname):
if is_python_module_name(dirname):
package = dir_from_package_name(dirname)
logger.info('%s -> %s' % (dirname, package))
dirname = package
if dirname.endswith('.' + MCDPConstants.library_extension):
libraries = [dirname]
else:
libraries = locate_files(dirname,
"*." + MCDPConstants.library_extension,
followlinks=False,
include_directories=True,
include_files=False)
if not libraries:
# use dirname as library path
libraries = [dirname]
for path in libraries:
self.add_lib_by_path(path)
# get all the images
allimages = {} # base.ext -> same struct as l.file_to_contents
for _short, data in self.libraries.items():
l = data['library']
for ext in MCDPConstants.exts_images:
basenames = l._list_with_extension(ext)
for b in basenames:
b_ext = b + '.' + ext
allimages[b_ext] = l.file_to_contents[b_ext]
for _short, data in self.libraries.items():
l = data['library']
for basename, d in allimages.items():
if not basename in l.file_to_contents:
l.file_to_contents[basename] = d
def _list_files_with_extension(dirname, extension):
''' List all files in the given directory with an extension.
The result is cached. '''
pattern = '*.%s' % extension
res = list(locate_files(dirname, pattern))
return res
def manual_jobs(context,
src_dir,
output_file,
generate_pdf,
bibfile,
stylesheet,
remove=None,
filter_soup=None,
extra_css=None):
manual_contents = list(get_manual_contents(src_dir))
if not manual_contents:
msg = 'Could not find any file for composing the book.'
raise Exception(msg)
# check that all the docnames are unique
pnames = [_[1] for _ in manual_contents]
if len(pnames) != len(set(pnames)):
msg = 'Repeated names detected: %s' % pnames
raise ValueError(msg)
local_files = list(locate_files(src_dir, '*.md'))
basename2filename = dict((os.path.basename(_), _) for _ in local_files)
files_contents = []
for i, (_, docname) in enumerate(manual_contents):
libname = 'unused'
logger.info('adding document %s - %s' % (libname, docname))
out_part_basename = '%02d%s' % (i, docname)
# read the file to get hash
basename = '%s.md' % docname
fn = basename2filename[basename]
contents = open(fn).read()
contents_hash = get_md5(contents)[:8]
# job will be automatically erased if the source changes
job_id = '%s-%s' % (docname, contents_hash)
res = context.comp(render_book,
src_dir,
docname,
generate_pdf,
main_file=output_file,
out_part_basename=out_part_basename,
filter_soup=filter_soup,
extra_css=extra_css,
job_id=job_id)
# source = '%s.md' % docname
# if source in basename2filename:
# filenames = [basename2filename[source]]
# erase_job_if_files_updated(context.cc, promise=res,
# filenames=filenames)
# else:
# logger.debug('Could not find file %r for date check' % source)
files_contents.append(res)
fn = os.path.join(src_dir, MCDPManualConstants.main_template)
if not os.path.exists(fn):
msg = 'Could not find template %s' % fn
raise ValueError(msg)
template = open(fn).read()
d = context.comp(manual_join,
template=template,
files_contents=files_contents,
bibfile=bibfile,
stylesheet=stylesheet,
remove=remove)
context.comp(write, d, output_file)
if os.path.exists(MCDPManualConstants.pdf_metadata_template):
context.comp(generate_metadata, src_dir)