def test_basic_file_flow(tmp_path):
"""testing all at once as it is hard to do in smaller chunk
write -> read -> list -> clean -> list again
"""
fname = 'a.txt'
content = 'this is a test'
path = Path(tmp_path) / fname
# write
write_file(tmp_path, fname, content)
assert path.exists()
# read
read_back = read_file(path)
assert read_back == content
# list
files_list = get_files_list(tmp_path, '*.txt')
assert path in files_list
# clean
clean_dir(tmp_path)
assert not path.exists()
# filelist again
files_list = get_files_list(path, '*.txt')
assert not files_list # must be empty
def test_filelist_filter(tmp_path):
"making sure we get all the files which have a givem suffix"
fnames = ['😁.md', 'a.m', 'a.txt', 'b.md', 'a.md.md']
content = 'this is a test'
clean_dir(tmp_path)
for fname in fnames:
write_file(tmp_path, fname, content)
files_list = get_files_list(tmp_path, '*.md')
assert len(files_list) == 3
files_list = get_files_list(tmp_path, '*.txt')
assert len(files_list) == 1
clean_dir(tmp_path)
def test_filelist_recursive(tmp_path):
clean_dir(tmp_path)
sub_path = tmp_path / 'newdir'
fname = 'test.md'
txt = 'hello'
write_file(sub_path, 'test.md', txt)
# ensure we have a sub dir and the file in it
assert sub_path.exists()
assert sub_path.is_dir()
read_text = read_file(sub_path / fname)
assert read_text == txt
# test we get a recursive listing
assert len(get_files_list(tmp_path)) == 1
# test that non recursive returns nothing
assert get_files_list(tmp_path, recursive=False) == []
def make_config(config):
""" Initialize a parser config with all the needed variables
Args:
config (obj_dict): the uninitialized configuration with basic
variables
Returns:
obj_dict: The initialized configuration
"""
if not config:
utils.detailed_error("Parser", 'make_config',
'supplied config is empty')
if 'template_dir' not in config:
utils.detailed_error("Parser", 'make_config',
'template_dir not found')
config.templates = {}
for fname in files.get_files_list(config.templates_path, "*.html"):
template = files.read_file(fname)
config.templates[fname.stem] = template
return config
def test_filelist_filter_empty(tmp_path):
"ensure no matches return an empty array"
clean_dir(tmp_path)
assert get_files_list(tmp_path) == []
assert get_files_list('thisisnotthedirectoryyoulookfor') == []
def __init__(self, config_filename, version='1.0'):
# Timers
self.cnts = PerfCounters()
self.cnts.start('Overall')
self.cnts.start('Init')
# [configuration]
self.current_dir = Path.cwd()
# make the config file path absolute to avoid weird cases
self.config_filename = Path(config_filename).resolve()
if not config_filename:
raise Exception("Supply a configuration filename")
# exist?
if self.config_filename.is_file(): # absolute path
self.config = files.load_config(self.config_filename)
else:
utils.error("Config file %s not found" % self.config_filename)
# site root dir is -1 from where the config is
self.config.root_dir = self.config_filename.parents[1]
self.config.build = utils.create_objdict()
# expose sitefab version to the templates
self.config.build.sitefab_version = version
# [parser] #
# initialize the parser config
parser_tpl_path = Path(self.config.parser.template_dir)
self.config.parser.templates_path = (self.config.root_dir /
parser_tpl_path)
self.config.parser = Parser.make_config(self.config.parser)
# [plugins]
# loading configuration
for d in self.config.plugins.config_dir:
config_dir = self.config.root_dir / d
# load the various config files from disk
plugins_config = defaultdict(dict)
for config_fname in files.get_files_list(config_dir, '*.yaml'):
plugin_name = config_fname.stem
category = "%s%s" % (str(config_fname.parts[-3]).capitalize(),
str(config_fname.parts[-2]).capitalize())
config = files.load_config(config_fname)
plugins_config[category][plugin_name] = config
# where to redirect the standard python log
debug_log_fname = self.get_logs_dir() / "debug.log"
self.plugins = Plugins(self.get_plugins_dirs(), debug_log_fname,
plugins_config)
# Store data generated by plugins that can be used later.
self.plugin_data = {}
self.plugin_results = defaultdict(int)
# [template rendering engine] #
self.jinja2 = Environment(loader=FileSystemLoader(
str(self.get_template_dir())),
extensions=['jinja2.ext.do'])
# loading templates custom functions
custom_filters = self.plugins.get_template_filters()
for flt_name, flt_fct in custom_filters.items():
self.jinja2.filters[flt_name] = flt_fct
# [logger] #
cfg = utils.create_objdict()
cfg.output_dir = self.get_logs_dir()
# log template not the one from the users.
cfg.template_dir = (self.config.root_dir /
self.config.logger.template_dir)
tpl_dir = self.config.root_dir / Path(self.config.logger.template_dir)
self.config.logger.template_dir = tpl_dir
cfg.log_template = "log.html"
cfg.log_index_template = "log_index.html" # noqa
cfg.stats_template = "stats.html"
self.logger = Logger(cfg, self)
# [linter] #
linter_config_filename = (self.config.root_dir /
self.config.linter.configuration_file)
linter_config = files.load_config(linter_config_filename)
linter_config.report_template_file = (
self.config.root_dir / self.config.linter.report_template_file)
linter_config.output_dir = self.get_logs_dir()
linter_config.site_output_dir = self.get_output_dir()
self.linter = Linter(linter_config)
# Finding content and assets.
self.filenames = utils.create_objdict()
self.filenames.posts = files.get_files_list(self.get_content_dir(),
"*.md")
# Cleanup the output directories.
files.clean_dir(self.get_output_dir())
self.cnts.stop('Init')
def process(self, unused, site, config):
log = ""
errors = False
plugin_name = "image_info"
input_dir = site.config.root_dir / config.input_dir
cache_file = site.config.root_dir / site.config.dir.cache / plugin_name
site_output_dir = site.config.root_dir / site.config.dir.output
# reading images list
if not input_dir:
return (SiteFab.ERROR, plugin_name, "no input_dir specified")
images = files.get_files_list(input_dir,
["*.jpg", "*.jpeg", "*.png", "*.gif"])
num_images = len(images)
if num_images == 0:
return (SiteFab.ERROR, plugin_name, "no images found")
# processing images
image_info = {}
progress_bar = tqdm(total=num_images,
unit='img',
leave=False,
desc="Generating images stats")
log_table = []
# compute info
# pack info for multiprocess
bundles = [[i, cache_file, site_output_dir] for i in images]
results = []
# allows non-multithread by setting threads to 1.
if site.config.threads > 1:
log += "Using multithreading: %s threads<br>" % (
site.config.threads)
tpool = Pool()
for data in tpool.imap_unordered(extract_image_info, bundles):
results.append(data)
progress_bar.update(1)
tpool.close()
tpool.join()
else:
for bundle in bundles:
results.append(extract_image_info(bundle))
progress_bar.update(1)
progress_bar.close()
# store data
for data in results:
info, row = data
# !convert the disk_dir and disk path to Path()
info['disk_path'] = Path(info['disk_path'])
info['disk_dir'] = Path(info['disk_dir'])
image_info[info['web_path']] = info
progress_bar.update(1)
log_table.append(row)
log += tabulate(log_table,
headers=['filename', 'size', 'hash', 'process time'],
tablefmt='html')
# make image info available to subsequent plugins
site.plugin_data['image_info'] = image_info # expose images info
if errors:
return (SiteFab.ERROR, plugin_name, log)
else:
return (SiteFab.OK, plugin_name, log)
def test_parser_templates_loaded(sitefab):
print(sitefab.config.parser.templates_path)
assert len(get_files_list(sitefab.config.parser.templates_path, "*.html"))