def text_resource_complete(self, resource, text): """ Save the file to a temporary place and run less compiler. Read the generated file and return the text as output. Set the target path to have a css extension. """ if not self._should_parse_resource(resource): return supported = [ "verbose", ("silent", "s"), ("compress", "x"), "O0", "O1", "O2", "include-path=" ] less = self.app source = File.make_temp(text) target = File.make_temp('') args = [unicode(less)] args.extend(self.process_args(supported)) args.extend([unicode(source), unicode(target)]) try: self.call_app(args) except subprocess.CalledProcessError: HydeException.reraise( "Cannot process %s. Error occurred when " "processing [%s]" % (self.app.name, resource.source_file), sys.exc_info()) return target.read_all()
def init(self, args): """ The init command. Initializes hyde-gopher with an exiting hyde site from a bundled template at the given sitepath. """ sitepath = self.main(args) if not (sitepath / 'site.yaml').exists(): raise HydeException(f"Site {sitepath} is not yet initialized.") site = self.make_site(sitepath, args.config, None) dest_path = sitepath / site.config.layout_root if dest_path.exists() and not args.overwrite: raise HydeException( "Site {} already has a layout at {}. Use -f to overwrite." .format(sitepath, dest_path) ) self.logger.info("Copying default layout to site at %s", sitepath) copy_kwargs = dict() if dest_path.exists() and args.overwrite: if sys.version_info < (3, 8): raise HydeException("Can't overwrite layout on Python < 3.8.") self.logger.warn("Overwriting %s", dest_path) copy_kwargs["dirs_exist_ok"] = True copytree( Path(__file__).with_name("layout_gopher"), dest_path, **copy_kwargs, ) self.logger.info("Layout copied") if not hasattr(site.config, "gopher_base_url"): self.logger.warn( "Site at %s has gopher_base_url not set", sitepath )
def get_source(self, environment, template): """ Calls the plugins to preprocess prior to returning the source. """ template = template.strip() # Fixed so that jinja2 loader does not have issues with # seprator in windows # template = template.replace(os.sep, '/') logger.debug("Loading template [%s] and preprocessing" % template) try: (contents, filename, date) = super(HydeLoader, self).get_source( environment, template) except UnicodeDecodeError: HydeException.reraise( "Unicode error when processing %s" % template, sys.exc_info()) except TemplateError as exc: HydeException.reraise('Error when processing %s: %s' % ( template, str(exc) ), sys.exc_info()) if self.preprocessor: resource = self.site.content.resource_from_relative_path(template) if resource: contents = self.preprocessor(resource, contents) or contents return (contents, filename, date)
def app(self): """ Gets the application path from the site configuration. If the path is not configured, attempts to guess the path from the sytem path environment variable. """ try: app_path = getattr(self.settings, 'app') except AttributeError: app_path = self.executable_name # Honour the PATH environment variable. if app_path is not None and not os.path.isabs(app_path): app_path = discover_executable(app_path, self.site.sitepath) if app_path is None: raise HydeException(self.executable_not_found_message) app = File(app_path) if not app.exists: raise HydeException(self.executable_not_found_message) return app
def load_python_object(name): """ Loads a python module from string """ (module_name, _, object_name) = name.rpartition(".") if module_name == '': (module_name, object_name) = (object_name, module_name) try: logger.debug('Loading module [%s]' % module_name) module = __import__(module_name) except ImportError: raise HydeException("The given module name [%s] is invalid." % module_name) if object_name == '': return module try: module = sys.modules[module_name] except KeyError: raise HydeException("Error occured when loading module [%s]" % module_name) try: logger.debug('Getting object [%s] from module [%s]' % (object_name, module_name)) return getattr(module, object_name) except AttributeError: raise HydeException("Cannot load the specified plugin [%s]. " "The given module [%s] does not contain the " "desired object [%s]. Please fix the " "configuration or ensure that the module is " "installed properly" % (name, module_name, object_name))
def __init__(self, source_file, node): super(Resource, self).__init__(source_file) self.source_file = source_file if not node: raise HydeException("Resource cannot exist without a node") if not source_file: raise HydeException("Source file is required" " to instantiate a resource") self.node = node self.site = node.site self._relative_deploy_path = None
def get_dependencies(self, path): """ Finds dependencies hierarchically based on the included files. """ text = self.env.loader.get_source(self.env, path)[0] from jinja2.meta import find_referenced_templates try: ast = self.env.parse(text) except Exception, e: HydeException.reraise( "Error processing %s: \n%s" % (path, unicode(e)), sys.exc_info())
def begin_site(self): jobs = self.site.content.node_from_relative_path('jobs/') with Log("Checking jobs metadata") as l: for resource in jobs.walk_resources(): if not resource.is_processable: l.output("Skipping %s" % (resource.name, )) continue with Log(resource.name): # Ensure that all tags are lowercase resource.meta.tags = [ self.fix_tag(a) for a in resource.meta.tags ] for tester in self._get_testers(): docstring = tester.__doc__.strip() assert docstring with Log("Test %s" % (docstring, )): tester(resource) if self.errors: proc = subprocess.Popen( [sys.executable, os.path.join(ROOT, "comment.py")], stdin=subprocess.PIPE) proc.communicate(self.get_pr_comment()) with Log("Site Processing Errors"): for filename, errors in self.errors.items(): with Log(filename, ok_msg="x") as log: for error in errors: log.output(error) raise HydeException("Some job listings failed validation") self.site.locations = json.dumps(self.location_finder.known_locations)
def __getattr__(self, method_name): if hasattr(Plugin, method_name): def __call_plugins__(*args): # logger.debug("Calling plugin method [%s]", method_name) res = None if self.site.plugins: for plugin in self.site.plugins: if hasattr(plugin, method_name): # logger.debug( # "\tCalling plugin [%s]", # plugin.__class__.__name__) function = getattr(plugin, method_name) res = function(*args) if res: targs = list(args) last = None if len(targs): last = targs.pop() targs.append(res if res else last) args = tuple(targs) return res return __call_plugins__ raise HydeException( "Unknown plugin method [%s] called." % method_name)
def import_to_include(match): """ Converts a css import statement to include statement. """ if not match.lastindex: return '' path = match.groups(1)[0] afile = File( File(resource.source_file.parent.child( path)).fully_expanded_path) if len(afile.kind.strip()) == 0: afile = File(afile.path + '.styl') ref = self.site.content.resource_from_path(afile.path) if not ref: try: include = self.settings.args.include except AttributeError: include = False if not include: raise HydeException("Cannot import from path [%s]" % afile.path) else: ref.is_processable = False return "\n" + \ self.template.get_include_statement(ref.relative_path) + \ "\n" return '@import "' + path + '"\n'
def __getattr__(self, method_name): if hasattr(Plugin, method_name): def __call_plugins__(*args): res = None if self.site.plugins: for plugin in self.site.plugins: if hasattr(plugin, method_name): checker = getattr(plugin, 'should_call__' + method_name) if checker(*args): function = getattr(plugin, method_name) try: res = function(*args) except: HydeException.reraise( 'Error occured when calling %s' % plugin.plugin_name, sys.exc_info()) targs = list(args) if len(targs): last = targs.pop() res = res if res else last targs.append(res) args = tuple(targs) return res return __call_plugins__ raise HydeException("Unknown plugin method [%s] called." % method_name)
def add_node(self, a_folder): """ Adds a new node to this folder's hierarchy. Also adds to to the hashtable of path to node associations for quick lookup. """ folder = Folder(a_folder) node = self.node_from_path(folder) if node: logger.debug("Node exists at [%s]" % node.relative_path) return node if not folder.is_descendant_of(self.source_folder): raise HydeException("The given folder [%s] does not" " belong to this hierarchy [%s]" % (folder, self.source_folder)) p_folder = folder parent = None hierarchy = [] while not parent: hierarchy.append(p_folder) p_folder = p_folder.parent parent = self.node_from_path(p_folder) hierarchy.reverse() node = parent if parent else self for h_folder in hierarchy: node = node.add_child_node(h_folder) self.node_map[str(h_folder)] = node logger.debug("Added node [%s] to [%s]" % (node.relative_path, self.source_folder)) return node
def add_resource(self, a_file): """ Adds a file to the parent node. Also adds to to the hashtable of path to resource associations for quick lookup. """ afile = File(a_file) resource = self.resource_from_path(afile) if resource: logger.debug("Resource exists at [%s]" % resource.relative_path) return resource if not afile.is_descendant_of(self.source_folder): raise HydeException("The given file [%s] does not reside" " in this hierarchy [%s]" % (afile, self.source_folder)) node = self.node_from_path(afile.parent) if not node: node = self.add_node(afile.parent) resource = node.add_child_resource(afile) self.resource_map[str(afile)] = resource logger.debug("Added resource [%s] to [%s]" % (resource.relative_path, self.source_folder)) return resource
def load(self): """ Walks the `source_folder` and loads the sitemap. Creates nodes and resources, reads metadata and injects attributes. This is the model for hyde. """ if not self.source_folder.exists: raise HydeException("The given source folder [%s]" " does not exist" % self.source_folder) with self.source_folder.walker as walker: def dont_ignore(name): for pattern in self.site.config.ignore: if fnmatch.fnmatch(name, pattern): return False return True @walker.folder_visitor def visit_folder(folder): if dont_ignore(folder.name): self.add_node(folder) else: logger.debug("Ignoring node: %s" % folder.name) return False @walker.file_visitor def visit_file(afile): if dont_ignore(afile.name): self.add_resource(afile)
def __init__(self, site): super(CleverCSSPlugin, self).__init__(site) try: import clevercss except ImportError as e: raise HydeException('Unable to import CleverCSS: ' + e.message) else: self.clevercss = clevercss
def get_tagger_sort_method(site): config = site.config content = site.content walker = 'walk_resources' sorter = None try: sorter = attrgetter('tagger.sorter')(config) walker = walker + '_sorted_by_%s' % sorter except AttributeError: pass try: walker = getattr(content, walker) except AttributeError: HydeException.reraise("Cannot find the sorter: %s" % sorter, sys.exc_info()) return walker
def __init__(self, site): super(SassyCSSPlugin, self).__init__(site) try: import scss except ImportError as e: raise HydeException('Unable to import pyScss: ' + e.message) else: self.scss = scss
def text_resource_complete(self, resource, text): if not resource.source_file.name == 'rjs.conf': return rjs = self.app target = File.make_temp('') args = [unicode(rjs)] args.extend(['-o', unicode(resource), ("out=" + target.fully_expanded_path)]) try: self.call_app(args) except subprocess.CalledProcessError: HydeException.reraise( "Cannot process %s. Error occurred when " "processing [%s]" % (self.app.name, resource.source_file), sys.exc_info()) return target.read_all()
def get_tagger_sort_method(site): config = site.config content = site.content walker = 'walk_resources' sorter = None try: sorter = attrgetter('tagger.sorter')(config) walker = walker + '_sorted_by_%s' % sorter except AttributeError: pass try: walker = getattr(content, walker) except AttributeError: HydeException.reraise( "Cannot find the sorter: %s" % sorter, sys.exc_info()) return walker
def __init__(self, site): super(SassPlugin, self).__init__(site) try: import sass except ImportError as e: raise HydeException('Unable to import libsass: ' + e.message) else: self.sass = sass self.resources = []
def text_resource_complete(self, resource, text): if not resource.source_file.name == 'rjs.conf': return rjs = self.app target = File.make_temp('') args = [str(rjs)] args.extend( ['-o', str(resource), ("out=" + target.fully_expanded_path)]) try: self.call_app(args) except subprocess.CalledProcessError: HydeException.reraise( "Cannot process %s. Error occurred when " "processing [%s]" % (self.app.name, resource.source_file), sys.exc_info()) return target.read_all()
def __init__(self, site): super(PILPlugin, self).__init__(site) try: from PIL import Image except ImportError: # No pillow try: import Image except ImportError, e: raise HydeException('Unable to load PIL: ' + e.message)
def __generate_resource__(self, resource, incremental=False): self.refresh_config() if not resource.is_processable: logger.debug("Skipping [%s]", resource) return if incremental and not self.has_resource_changed(resource): logger.debug("No changes found. Skipping resource [%s]", resource) return logger.debug("Processing [%s]", resource) with self.context_for_resource(resource) as context: target = File(self.site.config.deploy_root_path.child( resource.relative_deploy_path)) target.parent.make() if resource.simple_copy: logger.debug("Simply Copying [%s]", resource) resource.source_file.copy_to(target) elif resource.source_file.is_text: self.update_deps(resource) if resource.uses_template: logger.debug("Rendering [%s]", resource) try: text = self.template.render_resource(resource, context) except Exception as e: HydeException.reraise("Error occurred when processing" "template: [%s]: %s" % (resource, repr(e)), sys.exc_info()) else: text = resource.source_file.read_all() text = self.events.begin_text_resource( resource, text) or text text = self.events.text_resource_complete( resource, text) or text target.write(text) copymode(resource.source_file.path, target.path) else: logger.debug("Copying binary file [%s]", resource) self.events.begin_binary_resource(resource) resource.source_file.copy_to(target) self.events.binary_resource_complete(resource)
def __generate_resource__(self, resource, incremental=False): self.refresh_config() if not resource.is_processable: logger.debug("Skipping [%s]", resource) return if incremental and not self.has_resource_changed(resource): logger.debug("No changes found. Skipping resource [%s]", resource) return logger.debug("Processing [%s]", resource) with self.context_for_resource(resource) as context: target = File( self.site.config.deploy_root_path.child( resource.relative_deploy_path)) target.parent.make() if resource.simple_copy: logger.debug("Simply Copying [%s]", resource) resource.source_file.copy_to(target) elif resource.source_file.is_text: self.update_deps(resource) if resource.uses_template: logger.debug("Rendering [%s]", resource) try: text = self.template.render_resource(resource, context) except Exception as e: HydeException.reraise( "Error occurred when processing" "template: [%s]: %s" % (resource, repr(e)), sys.exc_info()) else: text = resource.source_file.read_all() text = self.events.begin_text_resource(resource, text) or text text = self.events.text_resource_complete(resource, text) or text target.write(text) copymode(resource.source_file.path, target.path) else: logger.debug("Copying binary file [%s]", resource) self.events.begin_binary_resource(resource) resource.source_file.copy_to(target) self.events.binary_resource_complete(resource)
def get_dependencies(self, path): """ Finds dependencies hierarchically based on the included files. """ text = self.env.loader.get_source(self.env, path)[0] from jinja2.meta import find_referenced_templates try: ast = self.env.parse(text) except Exception as e: HydeException.reraise("Error processing %s: \n%s" % (path, str(e)), sys.exc_info()) tpls = find_referenced_templates(ast) deps = list(self.env.globals["deps"].get("path", [])) for dep in tpls: deps.append(dep) if dep: deps.extend(self.get_dependencies(dep)) return list(set(deps))
def get_dependencies(self, path): """ Finds dependencies hierarchically based on the included files. """ text = self.env.loader.get_source(self.env, path)[0] from jinja2.meta import find_referenced_templates try: ast = self.env.parse(text) except Exception as e: HydeException.reraise("Error processing %s: \n%s" % (path, str(e)), sys.exc_info()) tpls = find_referenced_templates(ast) deps = list(self.env.globals['deps'].get('path', [])) for dep in tpls: deps.append(dep) if dep: deps.extend(self.get_dependencies(dep)) return list(set(deps))
def add_child_node(self, folder): """ Creates a new child node and adds it to the list of child nodes. """ if folder.parent != self.source_folder: raise HydeException("The given folder [%s] is not a" " direct descendant of [%s]" % (folder, self.source_folder)) node = Node(folder, self) self.child_nodes.append(node) return node
def create(self, args): """ The create command. Creates a new site from the template at the given sitepath. """ self.main(args) sitepath = Folder(Folder(args.sitepath).fully_expanded_path) if sitepath.exists and not args.overwrite: raise HydeException("The given site path [%s] already exists." " Use -f to overwrite." % sitepath) layout = Layout.find_layout(args.layout) logger.info("Creating site at [%s] with layout [%s]" % (sitepath, layout)) if not layout or not layout.exists: raise HydeException( "The given layout is invalid. Please check if you have the" " `layout` in the right place and the environment variable(%s)" " has been setup properly if you are using custom path for" " layouts" % HYDE_DATA) layout.copy_contents_to(args.sitepath) logger.info("Site creation complete")
def add_child_resource(self, afile): """ Creates a new resource and adds it to the list of child resources. """ if afile.parent != self.source_folder: raise HydeException("The given file [%s] is not" " a direct descendant of [%s]" % (afile, self.source_folder)) resource = Resource(afile, self) self.resources.append(resource) return resource
def __call_plugins__(*args): res = None if self.site.plugins: for plugin in self.site.plugins: if hasattr(plugin, method_name): checker = getattr(plugin, 'should_call__' + method_name) if checker(*args): function = getattr(plugin, method_name) try: res = function(*args) except: HydeException.reraise( 'Error occured when calling %s' % plugin.plugin_name, sys.exc_info()) targs = list(args) if len(targs): last = targs.pop() res = res if res else last targs.append(res) args = tuple(targs) return res
def import_to_include(match): if not match.lastindex: return '' path = match.groups(1)[0] afile = File(resource.source_file.parent.child(path)) if len(afile.kind.strip()) == 0: afile = File(afile.path + '.less') ref = self.site.content.resource_from_path(afile.path) if not ref: raise HydeException("Cannot import from path [%s]" % afile.path) ref.is_processable = False return self.template.get_include_statement(ref.relative_path)
def text_resource_complete(self, resource, text): """ Save the file to a temporary place and run stylus compiler. Read the generated file and return the text as output. Set the target path to have a css extension. """ if not resource.source_file.kind == 'styl': return stylus = self.app source = File.make_temp(text.strip()) target = source supported = [("compress", "c"), ("include", "I")] args = [unicode(stylus)] args.extend(self.process_args(supported)) args.append(unicode(source)) try: self.call_app(args) except subprocess.CalledProcessError: HydeException.reraise( "Cannot process %s. Error occurred when " "processing [%s]" % (stylus.name, resource.source_file), sys.exc_info()) return target.read_all()
def text_resource_complete(self, resource, text): """ Save the file to a temporary place and run stylus compiler. Read the generated file and return the text as output. Set the target path to have a css extension. """ if not resource.source_file.kind == 'styl': return stylus = self.app source = File.make_temp(text.strip()) supported = [("compress", "c"), ("include", "I")] args = [unicode(stylus)] args.extend(self.process_args(supported)) args.append(unicode(source)) try: self.call_app(args) except subprocess.CalledProcessError: HydeException.reraise( "Cannot process %s. Error occurred when " "processing [%s]" % (stylus.name, resource.source_file), sys.exc_info()) target = File(source.path + '.css') return target.read_all()
def create(self, args): """ The create command. Creates a new site from the template at the given sitepath. """ sitepath = self.main(args) markers = ['content', 'layout', 'site.yaml'] exists = any((FS(sitepath.child(item)).exists for item in markers)) if exists and not args.overwrite: raise HydeException( "The given site path [%s] already contains a hyde site." " Use -f to overwrite." % sitepath) layout = Layout.find_layout(args.layout) self.logger.info("Creating site at [%s] with layout [%s]" % (sitepath, layout)) if not layout or not layout.exists: raise HydeException( "The given layout is invalid. Please check if you have the" " `layout` in the right place and the environment variable(%s)" " has been setup properly if you are using custom path for" " layouts" % HYDE_DATA) layout.copy_contents_to(args.sitepath) self.logger.info("Site creation complete")
def __init__(self, source_folder, parent=None): super(Node, self).__init__(source_folder) if not source_folder: raise HydeException("Source folder is required" " to instantiate a node.") self.root = self self.module = None self.site = None self.source_folder = Folder(str(source_folder)) self.parent = parent if parent: self.root = self.parent.root self.module = self.parent.module if self.parent.module else self self.site = parent.site self.child_nodes = [] self.resources = []
def _create_tag_archive(self, config): """ Generates archives for each tag based on the given configuration. """ if not 'template' in config: raise HydeException( "No Template specified in tagger configuration.") content = self.site.content.source_folder source = Folder(config.get('source', '')) target = content.child_folder(config.get('target', 'tags')) if not target.exists: target.make() # Write meta data for the configuration meta = config.get('meta', {}) meta_text = u'' if meta: import yaml meta_text = yaml.dump(meta, default_flow_style=False) extension = config.get('extension', 'html') template = config['template'] archive_text = u""" --- extends: false %(meta)s --- {%% set tag = site.tagger.tags['%(tag)s'] %%} {%% set source = site.content.node_from_relative_path('%(node)s') %%} {%% set walker = source['walk_resources_tagged_with_%(tag)s'] %%} {%% extends "%(template)s" %%} """ for tagname, tag in self.site.tagger.tags.to_dict().iteritems(): tag_data = { "tag": tagname, "node": source.name, "template": template, "meta": meta_text } text = archive_text % tag_data archive_file = File(target.child("%s.%s" % (tagname, extension))) archive_file.delete() archive_file.write(text.strip()) self.site.content.add_resource(archive_file)