def setup_md_to_html_converter(self):
"""Create Markdown converter.
The converter is created with custom processors, html templates,
and extensions.
"""
templates = self.load_template_files()
extensions = [
"markdown.extensions.fenced_code",
"markdown.extensions.codehilite", "markdown.extensions.sane_lists",
"markdown.extensions.tables",
mdx_math.MathExtension()
]
self.converter = Verto(html_templates=templates, extensions=extensions)
def test_compile_files_custom(self):
'''Tests that some example files are converted with custom
html-templates.
'''
custom_templates = {
'image': '<img />',
'boxed-text': '<div class="box"></div>'
}
verto = Verto(html_templates=custom_templates)
for chapter in ['algorithms.md', 'introduction.md']:
text = pkg_resources.resource_string('verto', self.assets_template.format(chapter)).decode('utf-8')
result = verto.convert(text)
self.assertIsNot(result, None)
self.assertIsNot(result.title, None)
self.assertIsNot(result.html_string, None)
self.assertTrue(len(result.html_string) > 0)
def setup_md_to_html_converter(self):
"""Create Markdown converter.
The converter is created with custom processors, html templates,
and extensions.
"""
templates = self.load_template_files()
extensions = [
"markdown.extensions.fenced_code",
"markdown.extensions.codehilite", "markdown.extensions.sane_lists",
"markdown.extensions.tables",
mdx_math.MathExtension()
]
settings = {
"add_default_interactive_thumbnails_to_required_files": False,
}
self.converter = Verto(html_templates=templates,
extensions=extensions,
custom_settings=settings)
def test_compile_files_custom(self):
'''Tests that some example files are converted with custom
html-templates.
'''
custom_templates = {
'image': '<img />',
'boxed-text': '<div class="box"></div>'
}
verto = Verto(html_templates=custom_templates)
for chapter in ['algorithms.md', 'introduction.md']:
text = pkg_resources.resource_string(
'verto', self.assets_template.format(chapter)).decode('utf-8')
result = verto.convert(text)
self.assertIsNot(result, None)
self.assertIsNot(result.title, None)
self.assertIsNot(result.html_string, None)
self.assertTrue(len(result.html_string) > 0)
class SmokeFileTest(unittest.TestCase):
'''Tests opening of files and that verto generates some output.'''
def __init__(self, *args, **kwargs):
unittest.TestCase.__init__(self, *args, **kwargs)
self.maxDiff = None
self.verto = None
self.assets_template = 'tests/assets/smoke/{}'
def setUp(self):
'''Run before any testcases.
'''
self.verto = Verto()
def tearDown(self):
'''Run after any testcases.
'''
self.verto = None
def test_compile_files(self):
'''Tests that some example files are converted.
'''
for chapter in ['algorithms.md', 'introduction.md']:
text = pkg_resources.resource_string(
'verto', self.assets_template.format(chapter)).decode('utf-8')
result = self.verto.convert(text)
self.assertIsNot(result, None)
self.assertIsNot(result.title, None)
self.assertIsNot(result.html_string, None)
self.assertTrue(len(result.html_string) > 0)
def test_compile_files_custom(self):
'''Tests that some example files are converted with custom
html-templates.
'''
custom_templates = {
'image': '<img />',
'boxed-text': '<div class="box"></div>'
}
verto = Verto(html_templates=custom_templates)
for chapter in ['algorithms.md', 'introduction.md']:
text = pkg_resources.resource_string(
'verto', self.assets_template.format(chapter)).decode('utf-8')
result = verto.convert(text)
self.assertIsNot(result, None)
self.assertIsNot(result.title, None)
self.assertIsNot(result.html_string, None)
self.assertTrue(len(result.html_string) > 0)
def get_verto_block_patterns():
"""Get list of compiled regex patterns for verto block tags.
Returns:
(list) list of verto block regexes compiled with re.compile"""
convertor = Verto()
ext = convertor.verto_extension
block_pattern = ext.processor_info['style']['block_pattern']
block_strings = ext.processor_info['style']['strings']['block']
patterns = []
for block_string in block_strings:
c = re.compile(block_pattern.format(**{'block': block_string}))
patterns.append(c)
return patterns
class SmokeFileTest(unittest.TestCase):
'''Tests opening of files and that verto generates some output.'''
def __init__(self, *args, **kwargs):
unittest.TestCase.__init__(self, *args, **kwargs)
self.maxDiff = None
self.verto = None
self.assets_template = 'tests/assets/smoke/{}'
def setUp(self):
'''Run before any testcases.
'''
self.verto = Verto()
def tearDown(self):
'''Run after any testcases.
'''
self.verto = None
def test_compile_files(self):
'''Tests that some example files are converted.
'''
for chapter in ['algorithms.md', 'introduction.md']:
text = pkg_resources.resource_string('verto', self.assets_template.format(chapter)).decode('utf-8')
result = self.verto.convert(text)
self.assertIsNot(result, None)
self.assertIsNot(result.title, None)
self.assertIsNot(result.html_string, None)
self.assertTrue(len(result.html_string) > 0)
def test_compile_files_custom(self):
'''Tests that some example files are converted with custom
html-templates.
'''
custom_templates = {
'image': '<img />',
'boxed-text': '<div class="box"></div>'
}
verto = Verto(html_templates=custom_templates)
for chapter in ['algorithms.md', 'introduction.md']:
text = pkg_resources.resource_string('verto', self.assets_template.format(chapter)).decode('utf-8')
result = verto.convert(text)
self.assertIsNot(result, None)
self.assertIsNot(result.title, None)
self.assertIsNot(result.html_string, None)
self.assertTrue(len(result.html_string) > 0)
class BaseLoader():
"""Base loader class for individual loaders."""
def __init__(self, base_path):
"""Create a BaseLoader object."""
self.base_path = base_path
self.setup_md_to_html_converter()
def setup_md_to_html_converter(self):
"""Create Markdown converter.
The converter is created with custom processors, html templates,
and extensions.
"""
templates = self.load_template_files()
extensions = [
"markdown.extensions.fenced_code",
"markdown.extensions.codehilite",
"markdown.extensions.sane_lists",
"markdown.extensions.tables",
mdx_math.MathExtension()
]
self.converter = Verto(html_templates=templates, extensions=extensions)
def convert_md_file(self, md_file_path, config_file_path, heading_required=True, remove_title=True):
"""Return the Verto object for a given Markdown file.
Args:
md_file_path: Location of Markdown file to convert (str).
config_file_path: Path to related the config file (str).
heading_required: Boolean if the file requires a heading (bool).
remove_title: Boolean if the file's first heading should be removed (bool).
Returns:
VertoResult object
Raises:
CouldNotFindMarkdownFileError: when a given Markdown file cannot be found.
NoHeadingFoundInMarkdownFileError: when no heading can be found in a given
Markdown file.
EmptyMarkdownFileError: when no content can be found in a given Markdown
file.
MarkdownStyleError: when a verto StyleError is thrown.
"""
md_file_path = os.path.join(self.base_path, md_file_path)
config_file_path = os.path.join(self.base_path, config_file_path)
try:
# Check file exists
content = open(md_file_path, encoding="UTF-8").read()
except FileNotFoundError:
raise CouldNotFindMarkdownFileError(md_file_path, config_file_path)
custom_processors = self.converter.processor_defaults()
custom_processors.remove("scratch")
if remove_title:
custom_processors.add("remove-title")
self.converter.update_processors(custom_processors)
result = None
try:
result = self.converter.convert(content)
except StyleError as e:
raise MarkdownStyleError(md_file_path, e) from e
if heading_required:
if result.title is None:
raise NoHeadingFoundInMarkdownFileError(md_file_path)
if len(result.html_string) == 0:
raise EmptyMarkdownFileError(md_file_path)
check_converter_required_files(result.required_files, md_file_path)
check_converter_glossary_links(result.required_glossary_terms, md_file_path)
return result
def log(self, message, indent_amount=0):
"""Output the log message to the load log.
Args:
message: Text to display (str).
indent_amount: Amount of indentation required (int).
"""
indent = " " * indent_amount
text = "{indent}{text}\n".format(indent=indent, text=message)
sys.stdout.write(text)
def log_object_creation(self, created, obj, indent_amount=0):
"""Output the log message to the load log.
Args:
message: Text to display (str).
indent_amount: Amount of indentation required (int).
"""
if created:
log_prefix = "Added"
else:
log_prefix = "Updated"
self.log("{} {}: {}".format(log_prefix, obj._meta.verbose_name, obj.__str__()))
def load_yaml_file(self, yaml_file_path):
"""Load and read given YAML file.
Args:
file_path: location of yaml file to read (str).
Returns:
Either list or string, depending on structure of given yaml file
Raises:
CouldNotFindYAMLFileError: when a given config file cannot be found.
InvalidYAMLFileError: when a given config file is incorrectly formatted.
EmptyYAMLFileError: when a give config file is empty.
"""
yaml_file_path = os.path.join(self.base_path, yaml_file_path)
try:
yaml_file = open(yaml_file_path, encoding="UTF-8").read()
except FileNotFoundError:
raise CouldNotFindYAMLFileError(yaml_file_path)
try:
yaml_contents = yaml.load(yaml_file)
except yaml.YAMLError:
raise InvalidYAMLFileError(yaml_file_path)
if yaml_contents is None:
raise EmptyYAMLFileError(yaml_file_path)
return yaml_contents
def load_template_files(self):
"""Load custom HTML templates for converter.
Returns:
templates: dictionary of html templates
"""
templates = dict()
template_path = settings.CUSTOM_VERTO_TEMPLATES
for file in listdir(template_path):
template_file = re.search(r"(.*?).html$", file)
if template_file:
template_name = template_file.groups()[0]
templates[template_name] = open(template_path + file).read()
return templates
@abc.abstractmethod
def load(self):
"""Abstract method to be implemented by subclasses.
Raise:
NotImplementedError: when a user attempts to run the load() method of the
BaseLoader class.
"""
raise NotImplementedError("Subclass does not implement this method") # pragma: no cover
class BaseLoader():
"""Base loader class for individual loaders."""
def __init__(self,
base_path="",
structure_dir="structure",
content_path="",
structure_filename=""):
"""Create a BaseLoader object.
Args:
base_path: path to content_root, eg. "topics/content/" (str).
structure_dir: name of directory under base_path storing structure files (str).
content_path: path within locale/structure dir to content directory, eg. "binary-numbers/unit-plan" (str).
structure_filename: name of yaml file, eg. "unit-plan.yaml" (str).
"""
self.base_path = base_path
self.structure_dir = structure_dir
self.content_path = content_path
self.structure_filename = structure_filename
self.setup_md_to_html_converter()
def get_localised_file(self, language, filename):
"""Get full path to localised version of given file.
Args:
language: language code, matching a directory in self.base_path (str).
filename: path to file from the content directory of the loader (str).
Returns:
full path to localised version of given file (str).
"""
return os.path.join(self.get_localised_dir(language), filename)
def get_localised_dir(self, language):
"""Return full path to the localised content directory of the loader.
Args:
language: language code, matching a directory in self.base_path (str).
Returns:
full path to the localised content directory (str).
"""
return os.path.join(self.base_path, to_locale(language),
self.content_path)
@property
def structure_file_path(self):
"""Return full path to structure yaml file of the loader.
This assumes that the structure file is located in the same directory
as self.content_path, but inside the special 'structure' directory
instead of a language directory.
"""
return os.path.join(self.base_path, self.structure_dir,
self.content_path, self.structure_filename)
def setup_md_to_html_converter(self):
"""Create Markdown converter.
The converter is created with custom processors, html templates,
and extensions.
"""
templates = self.load_template_files()
extensions = [
"markdown.extensions.fenced_code",
"markdown.extensions.codehilite", "markdown.extensions.sane_lists",
"markdown.extensions.tables",
mdx_math.MathExtension()
]
self.converter = Verto(html_templates=templates, extensions=extensions)
def convert_md_file(self,
md_file_path,
config_file_path,
heading_required=True,
remove_title=True):
"""Return the Verto object for a given Markdown file.
Args:
md_file_path: Location of Markdown file to convert (str).
config_file_path: Path to related the config file (str).
heading_required: Boolean if the file requires a heading (bool).
remove_title: Boolean if the file's first heading should be removed (bool).
Returns:
VertoResult object
Raises:
CouldNotFindMarkdownFileError: when a given Markdown file cannot be found.
NoHeadingFoundInMarkdownFileError: when no heading can be found in a given
Markdown file.
EmptyMarkdownFileError: when no content can be found in a given Markdown
file.
MarkdownStyleError: when a verto StyleError is thrown.
"""
try:
# Check file exists
content = open(md_file_path, encoding="UTF-8").read()
except FileNotFoundError:
raise CouldNotFindMarkdownFileError(md_file_path, config_file_path)
custom_processors = self.converter.processor_defaults()
if remove_title:
custom_processors.add("remove-title")
self.converter.update_processors(custom_processors)
result = None
try:
result = self.converter.convert(content)
except StyleError as e:
raise MarkdownStyleError(md_file_path, e) from e
if heading_required:
if result.title is None:
raise NoHeadingFoundInMarkdownFileError(md_file_path)
if len(result.html_string) == 0:
raise EmptyMarkdownFileError(md_file_path)
check_converter_required_files(result.required_files, md_file_path)
check_converter_glossary_links(result.required_glossary_terms,
md_file_path)
return result
def log(self, message, indent_amount=0):
"""Output the log message to the load log.
Args:
message: Text to display (str).
indent_amount: Amount of indentation required (int).
"""
indent = " " * indent_amount
text = "{indent}{text}\n".format(indent=indent, text=message)
sys.stdout.write(text)
def load_yaml_file(self, yaml_file_path):
"""Load and read given YAML file.
Args:
file_path: location of yaml file to read (str).
Returns:
Either list or string, depending on structure of given yaml file
Raises:
CouldNotFindYAMLFileError: when a given config file cannot be found.
InvalidYAMLFileError: when a given config file is incorrectly formatted.
EmptyYAMLFileError: when a give config file is empty.
"""
try:
yaml_file = open(yaml_file_path, encoding="UTF-8").read()
except FileNotFoundError:
raise CouldNotFindYAMLFileError(yaml_file_path)
try:
yaml_contents = yaml.load(yaml_file)
except yaml.YAMLError:
raise InvalidYAMLFileError(yaml_file_path)
if yaml_contents is None:
raise EmptyYAMLFileError(yaml_file_path)
if isinstance(yaml_contents, dict) is False:
raise InvalidYAMLFileError(yaml_file_path)
return yaml_contents
def load_template_files(self):
"""Load custom HTML templates for converter.
Returns:
templates: dictionary of html templates
"""
templates = dict()
template_path = settings.CUSTOM_VERTO_TEMPLATES
for file in listdir(template_path):
template_file = re.search(r"(.*?).html$", file)
if template_file:
template_name = template_file.groups()[0]
templates[template_name] = open(template_path + file).read()
return templates
@abc.abstractmethod
def load(self):
"""Abstract method to be implemented by subclasses.
Raise:
NotImplementedError: when a user attempts to run the load() method of the
BaseLoader class.
"""
raise NotImplementedError(
"Subclass does not implement this method") # pragma: no cover
"""Module for the custom Django load_style_errors command."""
import importlib
from django.core import management
from verto import Verto
from verto.errors.Error import Error as VertoError
from style.utils import get_language_slugs
from style.models import Error
from utils.errors.VertoConversionError import VertoConversionError
BASE_DATA_MODULE_PATH = 'style.style_checkers.{}_data'
MARKDOWN_CONVERTER = Verto(extensions=[
"markdown.extensions.fenced_code",
], )
class Command(management.base.BaseCommand):
"""Required command class for the custom Django load_style_errors command."""
help = "Load progress outcomes to database."
def convert_markdown(self, module_path, code, field, markdown):
"""Render Markdown string to HTML.
Args:
module_path (str): Path to Python 3 module.
code (str): Code of style error that text belongs too.
field (str): Field of style error that text belongs too.
markdown (str): Markdown text to convert.
Returns:
def convert_md_file(self,
md_file_path,
config_file_path,
heading_required=True,
remove_title=True):
"""Return the Verto object for a given Markdown file.
Args:
md_file_path: Location of Markdown file to convert (str).
config_file_path: Path to related the config file (str).
heading_required: Boolean if the file requires a heading (bool).
remove_title: Boolean if the file's first heading should be removed (bool).
Returns:
VertoResult object
Raises:
CouldNotFindMarkdownFileError: when a given Markdown file cannot be found.
NoHeadingFoundInMarkdownFileError: when no heading can be found in a given
Markdown file.
EmptyMarkdownFileError: when no content can be found in a given Markdown
file.
VertoConversionError: when a verto StyleError is thrown.
"""
try:
# Check file exists
content = open(md_file_path, encoding="UTF-8").read()
except FileNotFoundError:
raise CouldNotFindMarkdownFileError(md_file_path, config_file_path)
""" Below is a hack to make the image-inline tag not require alt text to be
given when the language is not in English.
TODO: Remove this hack once translations are complete.
"""
directories = md_file_path.split('/')
if 'en' not in directories:
custom_argument_rules = {
"image-container": {
"alt": False
},
"image-inline": {
"alt": False
},
"image-tag": {
"alt": False
}
}
else:
custom_argument_rules = {
"image-container": {
"alt": True
},
"image-inline": {
"alt": True
},
"image-tag": {
"alt": True
}
}
custom_processors = self.converter.processor_defaults()
if remove_title:
custom_processors.add("remove-title")
templates = self.load_template_files()
extensions = [
"markdown.extensions.fenced_code",
"markdown.extensions.codehilite", "markdown.extensions.sane_lists",
"markdown.extensions.tables",
mdx_math.MathExtension()
]
self.converter = Verto(html_templates=templates,
extensions=extensions,
custom_argument_rules=custom_argument_rules,
processors=custom_processors)
""" End of hack. """
result = None
try:
result = self.converter.convert(content)
except VertoError as e:
raise VertoConversionError(md_file_path, e) from e
if heading_required:
if result.title is None:
raise NoHeadingFoundInMarkdownFileError(md_file_path)
if len(result.html_string) == 0:
raise EmptyMarkdownFileError(md_file_path)
if not self.lite_loader:
check_converter_required_files(result.required_files, md_file_path)
check_converter_glossary_links(result.required_glossary_terms,
md_file_path)
return result
def setUp(self):
'''Run before any testcases.
'''
self.verto = Verto()
def setUp(self):
'''Run before any testcases.
'''
self.verto = Verto()
