def text_resource_complete(self, resource, text):
"""
Save the file to a temporary place and run less compiler.
Read the generated file and return the text as output.
Set the target path to have a css extension.
"""
if not self._should_parse_resource(resource):
return
supported = [
"verbose",
("silent", "s"),
("compress", "x"),
"O0",
"O1",
"O2",
"include-path="
]
less = self.app
source = File.make_temp(text)
target = File.make_temp('')
args = [unicode(less)]
args.extend(self.process_args(supported))
args.extend([unicode(source), unicode(target)])
try:
self.call_app(args)
except subprocess.CalledProcessError:
HydeException.reraise(
"Cannot process %s. Error occurred when "
"processing [%s]" % (self.app.name, resource.source_file),
sys.exc_info())
return target.read_all()
def init(self, args):
"""
The init command. Initializes hyde-gopher with an exiting hyde site
from a bundled template at the given sitepath.
"""
sitepath = self.main(args)
if not (sitepath / 'site.yaml').exists():
raise HydeException(f"Site {sitepath} is not yet initialized.")
site = self.make_site(sitepath, args.config, None)
dest_path = sitepath / site.config.layout_root
if dest_path.exists() and not args.overwrite:
raise HydeException(
"Site {} already has a layout at {}. Use -f to overwrite."
.format(sitepath, dest_path)
)
self.logger.info("Copying default layout to site at %s", sitepath)
copy_kwargs = dict()
if dest_path.exists() and args.overwrite:
if sys.version_info < (3, 8):
raise HydeException("Can't overwrite layout on Python < 3.8.")
self.logger.warn("Overwriting %s", dest_path)
copy_kwargs["dirs_exist_ok"] = True
copytree(
Path(__file__).with_name("layout_gopher"),
dest_path,
**copy_kwargs,
)
self.logger.info("Layout copied")
if not hasattr(site.config, "gopher_base_url"):
self.logger.warn(
"Site at %s has gopher_base_url not set", sitepath
)
def get_source(self, environment, template):
"""
Calls the plugins to preprocess prior to returning the source.
"""
template = template.strip()
# Fixed so that jinja2 loader does not have issues with
# seprator in windows
#
template = template.replace(os.sep, '/')
logger.debug("Loading template [%s] and preprocessing" % template)
try:
(contents,
filename,
date) = super(HydeLoader, self).get_source(
environment, template)
except UnicodeDecodeError:
HydeException.reraise(
"Unicode error when processing %s" % template, sys.exc_info())
except TemplateError as exc:
HydeException.reraise('Error when processing %s: %s' % (
template,
str(exc)
), sys.exc_info())
if self.preprocessor:
resource = self.site.content.resource_from_relative_path(template)
if resource:
contents = self.preprocessor(resource, contents) or contents
return (contents, filename, date)
def text_resource_complete(self, resource, text):
"""
Save the file to a temporary place and run less compiler.
Read the generated file and return the text as output.
Set the target path to have a css extension.
"""
if not self._should_parse_resource(resource):
return
supported = [
"verbose", ("silent", "s"), ("compress", "x"), "O0", "O1", "O2",
"include-path="
]
less = self.app
source = File.make_temp(text)
target = File.make_temp('')
args = [unicode(less)]
args.extend(self.process_args(supported))
args.extend([unicode(source), unicode(target)])
try:
self.call_app(args)
except subprocess.CalledProcessError:
HydeException.reraise(
"Cannot process %s. Error occurred when "
"processing [%s]" % (self.app.name, resource.source_file),
sys.exc_info())
return target.read_all()
def get_source(self, environment, template):
"""
Calls the plugins to preprocess prior to returning the source.
"""
template = template.strip()
# Fixed so that jinja2 loader does not have issues with
# seprator in windows
#
template = template.replace(os.sep, '/')
logger.debug("Loading template [%s] and preprocessing" % template)
try:
(contents,
filename,
date) = super(HydeLoader, self).get_source(
environment, template)
except UnicodeDecodeError:
HydeException.reraise(
"Unicode error when processing %s" % template, sys.exc_info())
except TemplateError as exc:
HydeException.reraise('Error when processing %s: %s' % (
template,
str(exc)
), sys.exc_info())
if self.preprocessor:
resource = self.site.content.resource_from_relative_path(template)
if resource:
contents = self.preprocessor(resource, contents) or contents
return (contents, filename, date)
def app(self):
"""
Gets the application path from the site configuration.
If the path is not configured, attempts to guess the path
from the sytem path environment variable.
"""
try:
app_path = getattr(self.settings, 'app')
except AttributeError:
app_path = self.executable_name
# Honour the PATH environment variable.
if app_path is not None and not os.path.isabs(app_path):
app_path = discover_executable(app_path, self.site.sitepath)
if app_path is None:
raise HydeException(self.executable_not_found_message)
app = File(app_path)
if not app.exists:
raise HydeException(self.executable_not_found_message)
return app
def load_python_object(name):
"""
Loads a python module from string
"""
(module_name, _, object_name) = name.rpartition(".")
if module_name == '':
(module_name, object_name) = (object_name, module_name)
try:
logger.debug('Loading module [%s]' % module_name)
module = __import__(module_name)
except ImportError:
raise HydeException("The given module name [%s] is invalid." %
module_name)
if object_name == '':
return module
try:
module = sys.modules[module_name]
except KeyError:
raise HydeException("Error occured when loading module [%s]" %
module_name)
try:
logger.debug('Getting object [%s] from module [%s]' %
(object_name, module_name))
return getattr(module, object_name)
except AttributeError:
raise HydeException("Cannot load the specified plugin [%s]. "
"The given module [%s] does not contain the "
"desired object [%s]. Please fix the "
"configuration or ensure that the module is "
"installed properly" %
(name, module_name, object_name))
def __init__(self, source_file, node):
super(Resource, self).__init__(source_file)
self.source_file = source_file
if not node:
raise HydeException("Resource cannot exist without a node")
if not source_file:
raise HydeException("Source file is required"
" to instantiate a resource")
self.node = node
self.site = node.site
self._relative_deploy_path = None
def get_dependencies(self, path):
"""
Finds dependencies hierarchically based on the included
files.
"""
text = self.env.loader.get_source(self.env, path)[0]
from jinja2.meta import find_referenced_templates
try:
ast = self.env.parse(text)
except Exception, e:
HydeException.reraise(
"Error processing %s: \n%s" % (path, unicode(e)),
sys.exc_info())
def get_dependencies(self, path):
"""
Finds dependencies hierarchically based on the included
files.
"""
text = self.env.loader.get_source(self.env, path)[0]
from jinja2.meta import find_referenced_templates
try:
ast = self.env.parse(text)
except Exception, e:
HydeException.reraise(
"Error processing %s: \n%s" % (path, unicode(e)),
sys.exc_info())
def begin_site(self):
jobs = self.site.content.node_from_relative_path('jobs/')
with Log("Checking jobs metadata") as l:
for resource in jobs.walk_resources():
if not resource.is_processable:
l.output("Skipping %s" % (resource.name, ))
continue
with Log(resource.name):
# Ensure that all tags are lowercase
resource.meta.tags = [
self.fix_tag(a) for a in resource.meta.tags
]
for tester in self._get_testers():
docstring = tester.__doc__.strip()
assert docstring
with Log("Test %s" % (docstring, )):
tester(resource)
if self.errors:
proc = subprocess.Popen(
[sys.executable,
os.path.join(ROOT, "comment.py")],
stdin=subprocess.PIPE)
proc.communicate(self.get_pr_comment())
with Log("Site Processing Errors"):
for filename, errors in self.errors.items():
with Log(filename, ok_msg="x") as log:
for error in errors:
log.output(error)
raise HydeException("Some job listings failed validation")
self.site.locations = json.dumps(self.location_finder.known_locations)
def __getattr__(self, method_name):
if hasattr(Plugin, method_name):
def __call_plugins__(*args):
# logger.debug("Calling plugin method [%s]", method_name)
res = None
if self.site.plugins:
for plugin in self.site.plugins:
if hasattr(plugin, method_name):
# logger.debug(
# "\tCalling plugin [%s]",
# plugin.__class__.__name__)
function = getattr(plugin, method_name)
res = function(*args)
if res:
targs = list(args)
last = None
if len(targs):
last = targs.pop()
targs.append(res if res else last)
args = tuple(targs)
return res
return __call_plugins__
raise HydeException(
"Unknown plugin method [%s] called." % method_name)
def import_to_include(match):
"""
Converts a css import statement to include statement.
"""
if not match.lastindex:
return ''
path = match.groups(1)[0]
afile = File(
File(resource.source_file.parent.child(
path)).fully_expanded_path)
if len(afile.kind.strip()) == 0:
afile = File(afile.path + '.styl')
ref = self.site.content.resource_from_path(afile.path)
if not ref:
try:
include = self.settings.args.include
except AttributeError:
include = False
if not include:
raise HydeException("Cannot import from path [%s]" %
afile.path)
else:
ref.is_processable = False
return "\n" + \
self.template.get_include_statement(ref.relative_path) + \
"\n"
return '@import "' + path + '"\n'
def __getattr__(self, method_name):
if hasattr(Plugin, method_name):
def __call_plugins__(*args):
res = None
if self.site.plugins:
for plugin in self.site.plugins:
if hasattr(plugin, method_name):
checker = getattr(plugin,
'should_call__' + method_name)
if checker(*args):
function = getattr(plugin, method_name)
try:
res = function(*args)
except:
HydeException.reraise(
'Error occured when calling %s' %
plugin.plugin_name, sys.exc_info())
targs = list(args)
if len(targs):
last = targs.pop()
res = res if res else last
targs.append(res)
args = tuple(targs)
return res
return __call_plugins__
raise HydeException("Unknown plugin method [%s] called." % method_name)
def add_node(self, a_folder):
"""
Adds a new node to this folder's hierarchy.
Also adds to to the hashtable of path to node associations
for quick lookup.
"""
folder = Folder(a_folder)
node = self.node_from_path(folder)
if node:
logger.debug("Node exists at [%s]" % node.relative_path)
return node
if not folder.is_descendant_of(self.source_folder):
raise HydeException("The given folder [%s] does not"
" belong to this hierarchy [%s]" %
(folder, self.source_folder))
p_folder = folder
parent = None
hierarchy = []
while not parent:
hierarchy.append(p_folder)
p_folder = p_folder.parent
parent = self.node_from_path(p_folder)
hierarchy.reverse()
node = parent if parent else self
for h_folder in hierarchy:
node = node.add_child_node(h_folder)
self.node_map[str(h_folder)] = node
logger.debug("Added node [%s] to [%s]" %
(node.relative_path, self.source_folder))
return node
def add_resource(self, a_file):
"""
Adds a file to the parent node. Also adds to to the
hashtable of path to resource associations for quick lookup.
"""
afile = File(a_file)
resource = self.resource_from_path(afile)
if resource:
logger.debug("Resource exists at [%s]" % resource.relative_path)
return resource
if not afile.is_descendant_of(self.source_folder):
raise HydeException("The given file [%s] does not reside"
" in this hierarchy [%s]" %
(afile, self.source_folder))
node = self.node_from_path(afile.parent)
if not node:
node = self.add_node(afile.parent)
resource = node.add_child_resource(afile)
self.resource_map[str(afile)] = resource
logger.debug("Added resource [%s] to [%s]" %
(resource.relative_path, self.source_folder))
return resource
def load(self):
"""
Walks the `source_folder` and loads the sitemap.
Creates nodes and resources, reads metadata and injects attributes.
This is the model for hyde.
"""
if not self.source_folder.exists:
raise HydeException("The given source folder [%s]"
" does not exist" % self.source_folder)
with self.source_folder.walker as walker:
def dont_ignore(name):
for pattern in self.site.config.ignore:
if fnmatch.fnmatch(name, pattern):
return False
return True
@walker.folder_visitor
def visit_folder(folder):
if dont_ignore(folder.name):
self.add_node(folder)
else:
logger.debug("Ignoring node: %s" % folder.name)
return False
@walker.file_visitor
def visit_file(afile):
if dont_ignore(afile.name):
self.add_resource(afile)
def __init__(self, site):
super(CleverCSSPlugin, self).__init__(site)
try:
import clevercss
except ImportError as e:
raise HydeException('Unable to import CleverCSS: ' + e.message)
else:
self.clevercss = clevercss
def get_tagger_sort_method(site):
config = site.config
content = site.content
walker = 'walk_resources'
sorter = None
try:
sorter = attrgetter('tagger.sorter')(config)
walker = walker + '_sorted_by_%s' % sorter
except AttributeError:
pass
try:
walker = getattr(content, walker)
except AttributeError:
HydeException.reraise("Cannot find the sorter: %s" % sorter,
sys.exc_info())
return walker
def __init__(self, site):
super(SassyCSSPlugin, self).__init__(site)
try:
import scss
except ImportError as e:
raise HydeException('Unable to import pyScss: ' + e.message)
else:
self.scss = scss
def text_resource_complete(self, resource, text):
if not resource.source_file.name == 'rjs.conf':
return
rjs = self.app
target = File.make_temp('')
args = [unicode(rjs)]
args.extend(['-o', unicode(resource), ("out=" + target.fully_expanded_path)])
try:
self.call_app(args)
except subprocess.CalledProcessError:
HydeException.reraise(
"Cannot process %s. Error occurred when "
"processing [%s]" % (self.app.name, resource.source_file),
sys.exc_info())
return target.read_all()
def get_tagger_sort_method(site):
config = site.config
content = site.content
walker = 'walk_resources'
sorter = None
try:
sorter = attrgetter('tagger.sorter')(config)
walker = walker + '_sorted_by_%s' % sorter
except AttributeError:
pass
try:
walker = getattr(content, walker)
except AttributeError:
HydeException.reraise(
"Cannot find the sorter: %s" % sorter,
sys.exc_info())
return walker
def __init__(self, site):
super(SassPlugin, self).__init__(site)
try:
import sass
except ImportError as e:
raise HydeException('Unable to import libsass: ' + e.message)
else:
self.sass = sass
self.resources = []
def text_resource_complete(self, resource, text):
if not resource.source_file.name == 'rjs.conf':
return
rjs = self.app
target = File.make_temp('')
args = [str(rjs)]
args.extend(
['-o', str(resource), ("out=" + target.fully_expanded_path)])
try:
self.call_app(args)
except subprocess.CalledProcessError:
HydeException.reraise(
"Cannot process %s. Error occurred when "
"processing [%s]" % (self.app.name, resource.source_file),
sys.exc_info())
return target.read_all()
def __init__(self, site):
super(PILPlugin, self).__init__(site)
try:
from PIL import Image
except ImportError:
# No pillow
try:
import Image
except ImportError, e:
raise HydeException('Unable to load PIL: ' + e.message)
def __generate_resource__(self, resource, incremental=False):
self.refresh_config()
if not resource.is_processable:
logger.debug("Skipping [%s]", resource)
return
if incremental and not self.has_resource_changed(resource):
logger.debug("No changes found. Skipping resource [%s]", resource)
return
logger.debug("Processing [%s]", resource)
with self.context_for_resource(resource) as context:
target = File(self.site.config.deploy_root_path.child(
resource.relative_deploy_path))
target.parent.make()
if resource.simple_copy:
logger.debug("Simply Copying [%s]", resource)
resource.source_file.copy_to(target)
elif resource.source_file.is_text:
self.update_deps(resource)
if resource.uses_template:
logger.debug("Rendering [%s]", resource)
try:
text = self.template.render_resource(resource,
context)
except Exception as e:
HydeException.reraise("Error occurred when processing"
"template: [%s]: %s" %
(resource, repr(e)),
sys.exc_info())
else:
text = resource.source_file.read_all()
text = self.events.begin_text_resource(
resource, text) or text
text = self.events.text_resource_complete(
resource, text) or text
target.write(text)
copymode(resource.source_file.path, target.path)
else:
logger.debug("Copying binary file [%s]", resource)
self.events.begin_binary_resource(resource)
resource.source_file.copy_to(target)
self.events.binary_resource_complete(resource)
def __generate_resource__(self, resource, incremental=False):
self.refresh_config()
if not resource.is_processable:
logger.debug("Skipping [%s]", resource)
return
if incremental and not self.has_resource_changed(resource):
logger.debug("No changes found. Skipping resource [%s]", resource)
return
logger.debug("Processing [%s]", resource)
with self.context_for_resource(resource) as context:
target = File(
self.site.config.deploy_root_path.child(
resource.relative_deploy_path))
target.parent.make()
if resource.simple_copy:
logger.debug("Simply Copying [%s]", resource)
resource.source_file.copy_to(target)
elif resource.source_file.is_text:
self.update_deps(resource)
if resource.uses_template:
logger.debug("Rendering [%s]", resource)
try:
text = self.template.render_resource(resource, context)
except Exception as e:
HydeException.reraise(
"Error occurred when processing"
"template: [%s]: %s" % (resource, repr(e)),
sys.exc_info())
else:
text = resource.source_file.read_all()
text = self.events.begin_text_resource(resource,
text) or text
text = self.events.text_resource_complete(resource,
text) or text
target.write(text)
copymode(resource.source_file.path, target.path)
else:
logger.debug("Copying binary file [%s]", resource)
self.events.begin_binary_resource(resource)
resource.source_file.copy_to(target)
self.events.binary_resource_complete(resource)
def get_dependencies(self, path):
"""
Finds dependencies hierarchically based on the included
files.
"""
text = self.env.loader.get_source(self.env, path)[0]
from jinja2.meta import find_referenced_templates
try:
ast = self.env.parse(text)
except Exception as e:
HydeException.reraise("Error processing %s: \n%s" % (path, str(e)), sys.exc_info())
tpls = find_referenced_templates(ast)
deps = list(self.env.globals["deps"].get("path", []))
for dep in tpls:
deps.append(dep)
if dep:
deps.extend(self.get_dependencies(dep))
return list(set(deps))
def get_dependencies(self, path):
"""
Finds dependencies hierarchically based on the included
files.
"""
text = self.env.loader.get_source(self.env, path)[0]
from jinja2.meta import find_referenced_templates
try:
ast = self.env.parse(text)
except Exception as e:
HydeException.reraise("Error processing %s: \n%s" % (path, str(e)),
sys.exc_info())
tpls = find_referenced_templates(ast)
deps = list(self.env.globals['deps'].get('path', []))
for dep in tpls:
deps.append(dep)
if dep:
deps.extend(self.get_dependencies(dep))
return list(set(deps))
def add_child_node(self, folder):
"""
Creates a new child node and adds it to the list of child nodes.
"""
if folder.parent != self.source_folder:
raise HydeException("The given folder [%s] is not a"
" direct descendant of [%s]" %
(folder, self.source_folder))
node = Node(folder, self)
self.child_nodes.append(node)
return node
def create(self, args):
"""
The create command. Creates a new site from the template at the given
sitepath.
"""
self.main(args)
sitepath = Folder(Folder(args.sitepath).fully_expanded_path)
if sitepath.exists and not args.overwrite:
raise HydeException("The given site path [%s] already exists."
" Use -f to overwrite." % sitepath)
layout = Layout.find_layout(args.layout)
logger.info("Creating site at [%s] with layout [%s]" %
(sitepath, layout))
if not layout or not layout.exists:
raise HydeException(
"The given layout is invalid. Please check if you have the"
" `layout` in the right place and the environment variable(%s)"
" has been setup properly if you are using custom path for"
" layouts" % HYDE_DATA)
layout.copy_contents_to(args.sitepath)
logger.info("Site creation complete")
def add_child_resource(self, afile):
"""
Creates a new resource and adds it to the list of child resources.
"""
if afile.parent != self.source_folder:
raise HydeException("The given file [%s] is not"
" a direct descendant of [%s]" %
(afile, self.source_folder))
resource = Resource(afile, self)
self.resources.append(resource)
return resource
def __call_plugins__(*args):
res = None
if self.site.plugins:
for plugin in self.site.plugins:
if hasattr(plugin, method_name):
checker = getattr(plugin, 'should_call__' + method_name)
if checker(*args):
function = getattr(plugin, method_name)
try:
res = function(*args)
except:
HydeException.reraise(
'Error occured when calling %s' %
plugin.plugin_name, sys.exc_info())
targs = list(args)
if len(targs):
last = targs.pop()
res = res if res else last
targs.append(res)
args = tuple(targs)
return res
def import_to_include(match):
if not match.lastindex:
return ''
path = match.groups(1)[0]
afile = File(resource.source_file.parent.child(path))
if len(afile.kind.strip()) == 0:
afile = File(afile.path + '.less')
ref = self.site.content.resource_from_path(afile.path)
if not ref:
raise HydeException("Cannot import from path [%s]" %
afile.path)
ref.is_processable = False
return self.template.get_include_statement(ref.relative_path)
def text_resource_complete(self, resource, text):
"""
Save the file to a temporary place and run stylus compiler.
Read the generated file and return the text as output.
Set the target path to have a css extension.
"""
if not resource.source_file.kind == 'styl':
return
stylus = self.app
source = File.make_temp(text.strip())
target = source
supported = [("compress", "c"), ("include", "I")]
args = [unicode(stylus)]
args.extend(self.process_args(supported))
args.append(unicode(source))
try:
self.call_app(args)
except subprocess.CalledProcessError:
HydeException.reraise(
"Cannot process %s. Error occurred when "
"processing [%s]" % (stylus.name, resource.source_file),
sys.exc_info())
return target.read_all()
def text_resource_complete(self, resource, text):
"""
Save the file to a temporary place and run stylus compiler.
Read the generated file and return the text as output.
Set the target path to have a css extension.
"""
if not resource.source_file.kind == 'styl':
return
stylus = self.app
source = File.make_temp(text.strip())
supported = [("compress", "c"), ("include", "I")]
args = [unicode(stylus)]
args.extend(self.process_args(supported))
args.append(unicode(source))
try:
self.call_app(args)
except subprocess.CalledProcessError:
HydeException.reraise(
"Cannot process %s. Error occurred when "
"processing [%s]" % (stylus.name, resource.source_file),
sys.exc_info())
target = File(source.path + '.css')
return target.read_all()
def create(self, args):
"""
The create command. Creates a new site from the template at the given
sitepath.
"""
sitepath = self.main(args)
markers = ['content', 'layout', 'site.yaml']
exists = any((FS(sitepath.child(item)).exists for item in markers))
if exists and not args.overwrite:
raise HydeException(
"The given site path [%s] already contains a hyde site."
" Use -f to overwrite." % sitepath)
layout = Layout.find_layout(args.layout)
self.logger.info("Creating site at [%s] with layout [%s]" %
(sitepath, layout))
if not layout or not layout.exists:
raise HydeException(
"The given layout is invalid. Please check if you have the"
" `layout` in the right place and the environment variable(%s)"
" has been setup properly if you are using custom path for"
" layouts" % HYDE_DATA)
layout.copy_contents_to(args.sitepath)
self.logger.info("Site creation complete")
def __init__(self, source_folder, parent=None):
super(Node, self).__init__(source_folder)
if not source_folder:
raise HydeException("Source folder is required"
" to instantiate a node.")
self.root = self
self.module = None
self.site = None
self.source_folder = Folder(str(source_folder))
self.parent = parent
if parent:
self.root = self.parent.root
self.module = self.parent.module if self.parent.module else self
self.site = parent.site
self.child_nodes = []
self.resources = []
def _create_tag_archive(self, config):
"""
Generates archives for each tag based on the given configuration.
"""
if not 'template' in config:
raise HydeException(
"No Template specified in tagger configuration.")
content = self.site.content.source_folder
source = Folder(config.get('source', ''))
target = content.child_folder(config.get('target', 'tags'))
if not target.exists:
target.make()
# Write meta data for the configuration
meta = config.get('meta', {})
meta_text = u''
if meta:
import yaml
meta_text = yaml.dump(meta, default_flow_style=False)
extension = config.get('extension', 'html')
template = config['template']
archive_text = u"""
---
extends: false
%(meta)s
---
{%% set tag = site.tagger.tags['%(tag)s'] %%}
{%% set source = site.content.node_from_relative_path('%(node)s') %%}
{%% set walker = source['walk_resources_tagged_with_%(tag)s'] %%}
{%% extends "%(template)s" %%}
"""
for tagname, tag in self.site.tagger.tags.to_dict().iteritems():
tag_data = {
"tag": tagname,
"node": source.name,
"template": template,
"meta": meta_text
}
text = archive_text % tag_data
archive_file = File(target.child("%s.%s" % (tagname, extension)))
archive_file.delete()
archive_file.write(text.strip())
self.site.content.add_resource(archive_file)
