def add_node(self, a_folder):
"""
Adds a new node to this folder's hierarchy.
Also adds it to the hashtable of path to node associations
for quick lookup.
"""
folder = Folder(a_folder)
node = self.node_from_path(folder)
if node:
logger.debug("Node exists at [%s]" % node.relative_path)
return node
if not folder.is_descendant_of(self.source_folder):
raise HydeException("The given folder [%s] does not"
" belong to this hierarchy [%s]" %
(folder, self.source_folder))
p_folder = folder
parent = None
hierarchy = []
while not parent:
hierarchy.append(p_folder)
p_folder = p_folder.parent
parent = self.node_from_path(p_folder)
hierarchy.reverse()
node = parent if parent else self
for h_folder in hierarchy:
node = node.add_child_node(h_folder)
self.node_map[str(h_folder)] = node
logger.debug("Added node [%s] to [%s]" %
(node.relative_path, self.source_folder))
return node
def add_node(self, a_folder):
"""
Adds a new node to this folder's hierarchy.
Also adds it to the hashtable of path to node associations
for quick lookup.
"""
folder = Folder(a_folder)
node = self.node_from_path(folder)
if node:
logger.debug("Node exists at [%s]" % node.relative_path)
return node
if not folder.is_descendant_of(self.source_folder):
raise HydeException("The given folder [%s] does not"
" belong to this hierarchy [%s]" %
(folder, self.source_folder))
p_folder = folder
parent = None
hierarchy = []
while not parent:
hierarchy.append(p_folder)
p_folder = p_folder.parent
parent = self.node_from_path(p_folder)
hierarchy.reverse()
node = parent if parent else self
for h_folder in hierarchy:
node = node.add_child_node(h_folder)
self.node_map[str(h_folder)] = node
logger.debug("Added node [%s] to [%s]" % (
node.relative_path, self.source_folder))
return node
def begin_site(self):
"""
Finds all the folders that need flattening and changes the
relative deploy path of all resources in those folders.
"""
items = []
try:
items = self.site.config.flattener.items
except AttributeError:
pass
for item in items:
node = None
target = ''
try:
node = self.site.content.node_from_relative_path(item.source)
target = Folder(item.target)
except AttributeError:
continue
if node:
for resource in node.walk_resources():
target_path = target.child(resource.name)
self.logger.debug(
'Flattening resource path [%s] to [%s]' %
(resource, target_path))
resource.relative_deploy_path = target_path
for child in node.walk():
child.relative_deploy_path = target.path
def begin_site(self):
"""
Finds all the folders that need flattening and changes the
relative deploy path of all resources in those folders.
"""
items = []
try:
items = self.site.config.flattener.items
except AttributeError:
pass
for item in items:
node = None
target = ''
try:
node = self.site.content.node_from_relative_path(item.source)
target = Folder(item.target)
except AttributeError:
continue
if node:
for resource in node.walk_resources():
target_path = target.child(resource.name)
self.logger.debug('Flattening resource path [%s] to [%s]' %
(resource, target_path))
resource.relative_deploy_path = target_path
for child in node.walk():
child.relative_deploy_path = target.path
def node_from_path(self, path):
"""
Gets the node that maps to the given path.
If no match is found it returns None.
"""
if Folder(path) == self.source_folder:
return self
return self.node_map.get(str(Folder(path)), None)
def main(self, args):
"""
Will not be executed. A sub command is required. This function exists
to provide common parameters for the subcommands and some generic stuff
like version and metadata
"""
sitepath = Folder(args.sitepath).fully_expanded_path
return Folder(sitepath)
def _get_layout_folder(root, layout_name='basic'):
"""
Finds the layout folder from the given root folder.
If it does not exist, return None
"""
layouts_folder = Folder(unicode(root)).child_folder(LAYOUTS)
layout_folder = layouts_folder.child_folder(layout_name)
return layout_folder if layout_folder.exists else None
def __init__(self, sitepath, depends_file_name='.hyde_deps'):
self.sitepath = Folder(sitepath)
self.deps_file = File(self.sitepath.child(depends_file_name))
self.data = {}
if self.deps_file.exists:
self.data = yaml.load(self.deps_file.read_all())
import atexit
atexit.register(self.save)
def check_revision_already_published(proj, bucket_name, tree):
b = Bucket(bucket_name)
if not b.connect():
return None
sha = tree.get_revision_remote()
key_folder = Folder(proj).child_folder(tree.branch_name)
key_folder = key_folder.child_folder(sha)
key_path = key_folder.child(proj + '.zip')
return b.bucket.get_key(key_path)
def __init__(self, sitepath=None, config=None):
super(Site, self).__init__()
self.sitepath = Folder(Folder(sitepath).fully_expanded_path)
# Add sitepath to the list of module search paths so that
# local plugins can be included.
sys.path.insert(0, self.sitepath.fully_expanded_path)
self.config = config if config else Config(self.sitepath)
self.content = RootNode(self.config.content_root_path, self)
self.plugins = []
self.context = {}
def test_get_resource_from_relative_deploy_path():
s = Site(TEST_SITE_ROOT)
s.load()
path = 'blog/2010/december'
node = s.content.node_from_relative_path(path)
resource = node.get_resource('merry-christmas.html')
assert resource == s.content.resource_from_relative_deploy_path(
Folder(path).child('merry-christmas.html'))
resource.relative_deploy_path = Folder(path).child('merry-christmas.php')
assert resource == s.content.resource_from_relative_deploy_path(
Folder(path).child('merry-christmas.php'))
def test_load():
s = Site(TEST_SITE_ROOT)
s.load()
path = 'blog/2010/december'
node = s.content.node_from_relative_path(path)
assert node
assert Folder(node.relative_path) == Folder(path)
path += '/merry-christmas.html'
resource = s.content.resource_from_relative_path(path)
assert resource
assert resource.relative_path == path
assert not s.content.resource_from_relative_path('/happy-festivus.html')
def test_load_with_config(self):
s = Site(self.SITE_PATH, config=self.config)
s.load()
path = 'blog/2010/december'
node = s.content.node_from_relative_path(path)
assert node
assert Folder(node.relative_path) == Folder(path)
path += '/merry-christmas.html'
resource = s.content.resource_from_relative_path(path)
assert resource
assert resource.relative_path == path
assert not s.content.resource_from_relative_path(
'/happy-festivus.html')
def test_ignores_pattern_in_content(self):
text = """
{% markdown %}
Heading 1
===
Heading 2
===
{% endmarkdown %}
"""
about2 = File(TEST_SITE.child('content/about2.html'))
about2.write(text)
s = Site(TEST_SITE)
s.load()
res = s.content.resource_from_path(about2.path)
assert res.is_processable
s.config.plugins = ['hyde.ext.plugins.meta.MetaPlugin']
gen = Generator(s)
gen.generate_all()
target = File(Folder(s.config.deploy_root_path).child('about2.html'))
text = target.read_all()
q = PyQuery(text)
assert q("h1").length == 2
assert q("h1:nth-child(1)").text().strip() == "Heading 1"
assert q("h1:nth-child(2)").text().strip() == "Heading 2"
def __init__(self, source_folder, parent=None):
super(Node, self).__init__(source_folder)
if not source_folder:
raise HydeException("Source folder is required"
" to instantiate a node.")
self.root = self
self.module = None
self.site = None
self.source_folder = Folder(str(source_folder))
self.parent = parent
if parent:
self.root = self.parent.root
self.module = self.parent.module if self.parent.module else self
self.site = parent.site
self.child_nodes = []
self.resources = []
def translate_path(self, path):
"""
Finds the absolute path of the requested file by
referring to the `site` variable in the server.
"""
site = self.server.site
result = urlparse.urlparse(urllib.unquote(self.path).decode("utf-8"))
logger.debug("Trying to load file based on request: [%s]" % result.path)
path = result.path.lstrip("/")
res = None
if path.strip() == "" or File(path).kind.strip() == "":
deployed = site.config.deploy_root_path.child(path)
deployed = Folder.file_or_folder(deployed)
if isinstance(deployed, Folder):
node = site.content.node_from_relative_path(path)
res = node.get_resource("index.html")
elif hasattr(site.config, "urlcleaner") and hasattr(site.config.urlcleaner, "strip_extensions"):
for ext in site.config.urlcleaner.strip_extensions:
res = site.content.resource_from_relative_deploy_path(path + "." + ext)
if res:
break
for ext in site.config.urlcleaner.strip_extensions:
new_path = site.config.deploy_root_path.child(path + "." + ext)
if File(new_path).exists:
return new_path
else:
res = site.content.resource_from_relative_deploy_path(path)
if not res:
logger.error("Cannot load file: [%s]" % path)
return site.config.deploy_root_path.child(path)
else:
self.server.generate_resource(res)
new_path = site.config.deploy_root_path.child(res.relative_deploy_path)
return new_path
def _run_sphinx(self):
"""Run sphinx to generate the necessary output files.
This method creates a temporary directory for sphinx's output, then
run sphinx against the Hyde input directory.
"""
logger.info("running sphinx")
self.sphinx_build_dir = Folder(tempfile.mkdtemp())
conf_path = self.site.sitepath.child_folder(self.settings.conf_path)
sphinx_args = ["sphinx-build"]
sphinx_args.extend([
"-b", "hyde_json", "-c", conf_path.path, self.site.content.path,
self.sphinx_build_dir.path
])
if sphinx.main(sphinx_args) != 0:
raise RuntimeError("sphinx build failed")
def test_url_cleaner(self):
s = Site(TEST_SITE)
cfg = """
plugins:
- hyde.ext.plugins.urls.UrlCleanerPlugin
urlcleaner:
index_file_names:
- about.html
strip_extensions:
- html
append_slash: true
"""
s.config = Config(TEST_SITE, config_dict=yaml.load(cfg, Loader=yaml.FullLoader))
text = """
{% extends "base.html" %}
{% block main %}
<a id="index" href="{{ content_url('about.html') }}"></a>
<a id="blog" href=
"{{ content_url('blog/2010/december/merry-christmas.html') }}"></a>
{% endblock %}
"""
about2 = File(TEST_SITE.child('content/test.html'))
about2.write(text)
gen = Generator(s)
gen.generate_all()
from pyquery import PyQuery
target = File(Folder(s.config.deploy_root_path).child('test.html'))
text = target.read_all()
q = PyQuery(text)
assert q('a#index').attr("href") == '/'
assert q('a#blog').attr(
"href") == '/blog/2010/december/merry-christmas'
def test_read_from_specified_file(self):
File(TEST_SITE.child('another.yaml')).write(self.conf2)
c = Config(sitepath=TEST_SITE, config_file='another.yaml')
assert c.content_root_path == TEST_SITE.child_folder('site/stuff')
assert c.media_root_path == c.content_root_path.child_folder('mmm')
assert c.media_url == TEST_SITE.child_folder('/media')
assert c.deploy_root_path == Folder('~/deploy_site')
def thumb(self, defaults={}, width=None, height=None):
"""
Generate a thumbnail for the given image
"""
if width is None and height is None:
width, height = defaults['width'], defaults['height']
im = Image.open(self.path)
if im.mode != 'RGBA':
im = im.convert('RGBA')
# Convert to a thumbnail
if width is None:
# height is not None
width = im.size[0]*height/im.size[1] + 1
elif height is None:
# width is not None
height = im.size[1]*width/im.size[0] + 1
im.thumbnail((width, height), Image.ANTIALIAS)
# Prepare path
path = os.path.join(os.path.dirname(self.get_relative_deploy_path()),
"%s%s" % (defaults['prefix'],
self.name))
target = File(Folder(self.site.config.deploy_root_path).child(path))
target.parent.make()
if self.name.endswith(".jpg"):
im.save(target.path, "JPEG", optimize=True, quality=75)
else:
im.save(target.path, "PNG", optimize=True)
return Thumb(path, width=im.size[0], height=im.size[1])
def test_read_from_file_by_default(self):
File(TEST_SITE.child('site.yaml')).write(self.conf2)
c = Config(sitepath=TEST_SITE)
assert c.content_root_path == TEST_SITE.child_folder('site/stuff')
assert c.media_root_path == c.content_root_path.child_folder('mmm')
assert c.media_url == TEST_SITE.child_folder('/media')
assert c.deploy_root_path == Folder('~/deploy_site')
def load(sitepath, ctx):
"""
Load context from config data and providers.
"""
context = {}
try:
context.update(ctx.data.__dict__)
except AttributeError:
# No context data found
pass
providers = {}
try:
providers.update(ctx.providers.__dict__)
except AttributeError:
# No providers found
pass
for provider_name, resource_name in providers.items():
res = File(Folder(sitepath).child(resource_name))
if res.exists:
data = make_expando(
yaml.load(res.read_all(), Loader=yaml.FullLoader))
context[provider_name] = data
return context
def test_media_url_from_resource(self):
s = Site(self.SITE_PATH, config=self.config)
s.load()
path = 'css/site.css'
resource = s.content.resource_from_relative_path(
Folder("media").child(path))
assert resource
assert resource.full_url == "/media/" + path
def __init__(self, sitepath, depends_file_name='.hyde_deps'):
self.sitepath = Folder(sitepath)
self.deps_file = File(self.sitepath.child(depends_file_name))
self.data = {}
if self.deps_file.exists:
self.data = yaml.load(self.deps_file.read_all())
import atexit
atexit.register(self.save)
def test_multiple_levels(self):
page_d = {'title': 'An even nicer title'}
blog_d = {'author': 'Laks'}
content_d = {'title': 'A nice title', 'author': 'Lakshmi Vyas'}
site_d = {
'author': 'Lakshmi',
'twitter': 'lakshmivyas',
'nodemeta': 'meta.yaml'
}
text = """
---
title: %(title)s
---
{%% extends "base.html" %%}
{%% block main %%}
Hi!
I am a test template to make sure jinja2 generation works well with hyde.
<span class="title">{{resource.meta.title}}</span>
<span class="author">{{resource.meta.author}}</span>
<span class="twitter">{{resource.meta.twitter}}</span>
{%% endblock %%}
"""
about2 = File(TEST_SITE.child('content/blog/about2.html'))
about2.write(text % page_d)
content_meta = File(TEST_SITE.child('content/nodemeta.yaml'))
content_meta.write(yaml.dump(content_d))
content_meta = File(TEST_SITE.child('content/blog/meta.yaml'))
content_meta.write(yaml.dump(blog_d))
s = Site(TEST_SITE)
s.config.plugins = ['hyde.ext.plugins.meta.MetaPlugin']
s.config.meta = site_d
gen = Generator(s)
gen.generate_all()
expected = {}
expected.update(site_d)
expected.update(content_d)
expected.update(blog_d)
expected.update(page_d)
res = s.content.resource_from_path(about2.path)
assert hasattr(res, 'meta')
for k, v in list(expected.items()):
assert hasattr(res.meta, k)
assert getattr(res.meta, k) == v
target = File(
Folder(s.config.deploy_root_path).child('blog/about2.html'))
text = target.read_all()
q = PyQuery(text)
for k, v in list(expected.items()):
if k != 'nodemeta':
assert v in q("span." + k).text()
def gen(source, data, target=None):
source_command = ShellCommand(cwd=source.path)
if source.child_file("requirements.txt").exists:
source_command.call("pip", "install", "-r", "requirements.txt")
if source.child_file("package.json").exists:
source_command.call("npm", "install")
# Generate
target = target or data.target or source.parent.child("dist/www")
dist = Folder(target)
dist.make()
template = source.child_file(data.config_template or "env.yaml")
conf = source.child_file(data.config_file_name or "settings.gitbot")
transform(template, conf, data)
source_command.call("hyde", "gen", "-r", "-c", conf.name, "-d", dist.path)
return dist
def __init__(self, source, repo=None, remote='origin', branch='master'):
if not source:
raise Exception('You must provide the source')
self.source = Folder(source)
self.repo = repo
self.remote = remote
self.branch_name = branch
self.tagger = Tagger(self)
self.git = ShellCommand(cwd=self.source.path, cmd='git')
def test_plugin_chaining(self):
self.site.config.plugins = [
'test_plugin.ConstantReturnPlugin', 'test_plugin.NoReturnPlugin'
]
path = self.site.content.source_folder.child('about.html')
gen = Generator(self.site)
gen.generate_resource_at_path(path)
about = File(
Folder(self.site.config.deploy_root_path).child('about.html'))
assert about.read_all() == "Jam"
def test_generate_resource_from_path_with_is_processable_false(self):
site = Site(TEST_SITE)
site.load()
resource = site.content.resource_from_path(
TEST_SITE.child('content/about.html'))
resource.is_processable = False
gen = Generator(site)
gen.generate_resource_at_path(TEST_SITE.child('content/about.html'))
about = File(Folder(site.config.deploy_root_path).child('about.html'))
assert not about.exists
def test_generate_resource_from_path(self):
site = Site(TEST_SITE)
site.load()
gen = Generator(site)
gen.generate_resource_at_path(TEST_SITE.child('content/about.html'))
about = File(Folder(site.config.deploy_root_path).child('about.html'))
assert about.exists
text = about.read_all()
q = PyQuery(text)
assert about.name in q("div#main").text()
def has_resource_changed(self, resource):
"""
Checks if the given resource has changed since the
last generation.
"""
logger.debug("Checking for changes in %s" % resource)
self.load_template_if_needed()
self.load_site_if_needed()
target = File(
self.site.config.deploy_root_path.child(
resource.relative_deploy_path))
if not target.exists or target.older_than(resource.source_file):
logger.debug("Found changes in %s" % resource)
return True
if resource.source_file.is_binary:
logger.debug("No Changes found in %s" % resource)
return False
if self.site.config.needs_refresh() or \
not target.has_changed_since(self.site.config.last_modified):
logger.debug("Site configuration changed")
return True
deps = self.get_dependencies(resource)
if not deps or None in deps:
logger.debug("No changes found in %s" % resource)
return False
content = self.site.content.source_folder
layout = Folder(self.site.sitepath).child_folder('layout')
logger.debug("Checking for changes in dependents:%s" % deps)
for dep in deps:
if not dep:
return True
source = File(content.child(dep))
if not source.exists:
source = File(layout.child(dep))
if not source.exists:
return True
if target.older_than(source):
return True
logger.debug("No changes found in %s" % resource)
return False
def test_relative_deploy_path_override():
s = Site(TEST_SITE_ROOT)
s.load()
res = s.content.resource_from_relative_path(
'blog/2010/december/merry-christmas.html')
res.relative_deploy_path = 'blog/2010/december/happy-holidays.html'
for page in s.content.walk_resources():
if res.source_file == page.source_file:
assert page.relative_deploy_path == 'blog/2010/december/happy-holidays.html'
else:
assert page.relative_deploy_path == Folder(page.relative_path)
def has_resource_changed(self, resource):
"""
Checks if the given resource has changed since the
last generation.
"""
logger.debug("Checking for changes in %s" % resource)
self.load_template_if_needed()
self.load_site_if_needed()
target = File(self.site.config.deploy_root_path.child(
resource.relative_deploy_path))
if not target.exists or target.older_than(resource.source_file):
logger.debug("Found changes in %s" % resource)
return True
if resource.source_file.is_binary:
logger.debug("No Changes found in %s" % resource)
return False
if self.site.config.needs_refresh() or \
not target.has_changed_since(self.site.config.last_modified):
logger.debug("Site configuration changed")
return True
deps = self.get_dependencies(resource)
if not deps or None in deps:
logger.debug("No changes found in %s" % resource)
return False
content = self.site.content.source_folder
layout = Folder(self.site.sitepath).child_folder('layout')
logger.debug("Checking for changes in dependents:%s" % deps)
for dep in deps:
if not dep:
return True
source = File(content.child(dep))
if not source.exists:
source = File(layout.child(dep))
if not source.exists:
return True
if target.older_than(source):
return True
logger.debug("No changes found in %s" % resource)
return False
def __init__(self, sitepath, config_file=None, config_dict=None):
self.default_config = dict(
mode='production',
simple_copy=[],
content_root='content',
deploy_root='deploy',
media_root='media',
layout_root='layout',
media_url='/media',
base_url="/",
encode_safe=None,
not_found='404.html',
plugins=[],
ignore=["*~", "*.bak", ".hg", ".git", ".svn"],
meta={"nodemeta": 'meta.yaml'})
self.config_file = config_file
self.config_dict = config_dict
self.load_time = datetime.min
self.config_files = []
self.sitepath = Folder(sitepath)
super(Config, self).__init__(self.load())
def setup_env(user_name, data):
os.setuid(pwd.getpwnam(user_name)[2])
home = Folder('/mnt').child_folder(user_name)
os.environ['HOME'] = home.path
os.environ['BASH_ENV'] = home.child('.profile')
os.chdir(home.path)
venv = user_name.replace('gu', 'ge')
check_call(['/usr/bin/virtualenv', '--system-site-packages', venv])
def activate():
f = home.child_folder(venv).child('bin/activate_this.py')
execfile(f, dict(__file__=f))
activate()
if 'github_oauth' in data:
check_call(['git', 'config', '--global', 'credential.helper', 'store'])
credential = 'https://{oauth}:[email protected]'
cred_file = home.child_file('.git-credentials')
cred_file.write(credential.format(oauth=data['github_oauth']))
return home, activate
def __upload(proj, repo, branch, data, maker, force=True):
root, source, dist = (None, None, None)
try:
root = Folder(data.root or '~')
source = root.child_folder('src')
source.make()
source = source.child_folder(proj)
dist = root.child_folder('dist')
tree = Tree(source, repo=repo, branch=branch)
key = None
if not force:
key = check_revision_already_published(proj, data.bucket, tree)
if not key:
b = Bucket(data.bucket)
b.make()
key_folder = Folder(proj).child_folder(branch)
zippath = dist.child_file(proj + '.zip')
tree.clone(tip_only=True)
sha = tree.get_revision(short=False)
key_folder = key_folder.child_folder(sha)
target = dist.child_folder(proj)
target.make()
maker(source, target)
target.zip(zippath.path)
b.add_file(zippath, target_folder=key_folder.path)
key = b.bucket.get_key(key_folder.child(zippath.name))
finally:
if source:
source.delete()
if dist:
dist.delete()
return key.generate_url(30000)
def test_can_execute_optipng(self):
s = Site(TEST_SITE)
s.config.mode = "production"
s.config.plugins = ['hyde.ext.plugins.images.OptiPNGPlugin']
s.config.optipng = Expando(dict(args=dict(quiet="")))
source = File(TEST_SITE.child('content/media/images/hyde-lt-b.png'))
target = File(
Folder(
s.config.deploy_root_path).child('media/images/hyde-lt-b.png'))
gen = Generator(s)
gen.generate_resource_at_path(source)
assert target.exists
assert target.size < source.size
def test_generate_resource_from_path_with_uses_template_false(self):
site = Site(TEST_SITE)
site.load()
resource = site.content.resource_from_path(
TEST_SITE.child('content/about.html'))
resource.uses_template = False
gen = Generator(site)
gen.generate_resource_at_path(TEST_SITE.child('content/about.html'))
about = File(Folder(site.config.deploy_root_path).child('about.html'))
assert about.exists
text = about.read_all()
expected = resource.source_file.read_all()
assert text == expected
def test_can_uglify(self):
s = Site(TEST_SITE)
s.config.plugins = ['hyde.ext.plugins.js.UglifyPlugin']
s.config.mode = "production"
source = TEST_SITE.child('content/media/js/jquery.js')
target = File(Folder(s.config.deploy_root_path).child('media/js/jquery.js'))
gen = Generator(s)
gen.generate_resource_at_path(source)
assert target.exists
expected = File(UGLIFY_SOURCE.child('expected-jquery.js'))
# TODO: Very fragile. Better comparison needed.
assert target.read_all() == expected.read_all()
def test_combine_remove(self):
_, s = self._test_combine("""
---
combine:
files: script.*.js
remove: yes
---
First line""")
for i in range(1, 4):
assert not File(
Folder(s.config.deploy_root_path).child(
'media/js/script.%d.js' % i)).exists
def get_worker_outputs(data):
result = None
try:
region = data.region or 'us-east-1'
root = Folder(data.root or '~')
source = root.child_folder('src')
source.make()
source = source.child_folder('worker')
repo = data.worker_repo or 'git://github.com/gitbot/worker.git'
branch = data.worker_branch or 'master'
# 1. Pull worker repo
tree = Tree(source, repo, branch)
tree.clone(tip_only=True)
# 2. Call gitbot.stack.publish with 'gitbot.yaml'
worker_stack_name = stack.publish_stack(
source.child_file('gitbot.yaml'),
wait=True)
result = stack.get_outputs(worker_stack_name, region)
except Exception, e:
print repr(e)
raise
def __init__(self, source_folder, parent=None):
super(Node, self).__init__(source_folder)
if not source_folder:
raise HydeException("Source folder is required"
" to instantiate a node.")
self.root = self
self.module = None
self.site = None
self.source_folder = Folder(str(source_folder))
self.parent = parent
if parent:
self.root = self.parent.root
self.module = self.parent.module if self.parent.module else self
self.site = parent.site
self.child_nodes = []
self.resources = []
def _run_sphinx(self):
"""Run sphinx to generate the necessary output files.
This method creates a temporary directory for sphinx's output, then
run sphinx against the Hyde input directory.
"""
logger.info("running sphinx")
self.sphinx_build_dir = Folder(tempfile.mkdtemp())
conf_path = self.site.sitepath.child_folder(self.settings.conf_path)
sphinx_args = ["sphinx-build"]
sphinx_args.extend([
"-b", "hyde_json",
"-c", conf_path.path,
self.site.content.path,
self.sphinx_build_dir.path
])
if sphinx.main(sphinx_args) != 0:
raise RuntimeError("sphinx build failed")
class Dependents(IterableUserDict):
"""
Represents the dependency graph for hyde.
"""
def __init__(self, sitepath, depends_file_name='.hyde_deps'):
self.sitepath = Folder(sitepath)
self.deps_file = File(self.sitepath.child(depends_file_name))
self.data = {}
if self.deps_file.exists:
self.data = yaml.load(self.deps_file.read_all())
import atexit
atexit.register(self.save)
def save(self):
"""
Saves the dependency graph (just a dict for now).
"""
if self.deps_file.parent.exists:
self.deps_file.write(yaml.dump(self.data))
def __init__(self, sitepath, config_file=None, config_dict=None):
self.default_config = dict(
mode='production',
simple_copy=[],
content_root='content',
deploy_root='deploy',
media_root='media',
layout_root='layout',
media_url='/media',
base_url="/",
encode_safe=None,
not_found='404.html',
plugins=[],
ignore=["*~", "*.bak", ".hg", ".git", ".svn"],
meta={
"nodemeta": 'meta.yaml'
}
)
self.config_file = config_file
self.config_dict = config_dict
self.load_time = datetime.min
self.config_files = []
self.sitepath = Folder(sitepath)
super(Config, self).__init__(self.load())
class Tree(object):
def __init__(self, source, repo=None, remote='origin', branch='master'):
if not source:
raise Exception('You must provide the source')
self.source = Folder(source)
self.repo = repo
self.remote = remote
self.branch_name = branch
self.tagger = Tagger(self)
self.git = ShellCommand(cwd=self.source.path, cmd='git')
def get_revision_remote(self):
out = self.git.get('ls-remote', self.repo, self.branch_name, cwd='.')
return out.split()[0]
def get_branch(self):
return self.git.get('rev-parse', '--abbrev-ref', 'HEAD')
def get_revision(self, ref=None, short=True):
self.ensure_source_exists()
return self.git.get('rev-parse',
'--short' if short else '',
(ref or 'HEAD')).strip()
def get_last_committed(self, ref=None):
self.ensure_source_exists()
modified = self.git.get('log',
(ref or 'HEAD'),
'-1', '--pretty=format:%ad', '--date=local')
return datetime.strptime(modified, '%a %b %d %H:%M:%S %Y')
def ensure_repo_exists(self):
if not self.repo:
raise GitException('This tree [%s] does not have a repo.'
% self.source.path)
def ensure_source_exists(self):
if not self.source.exists:
raise GitException('The source directory [%s] is missing.'
% self.source.path)
def ensure_source_does_not_exist(self):
if self.source.exists:
raise GitException('The source directory [%s] exists already.'
% self.source.path)
def make(self, bare=False):
self.ensure_source_exists()
logger.info('Initializing git...')
self.git.call('init', '--bare' if bare else None)
def make_ref(self, ref, sha='HEAD'):
self.ensure_source_exists()
logger.info(
'Creating a ref (%s) to commit (%s)...' % (
ref,
sha
))
self.git.call('update-ref', ref, sha)
def add_remote(self):
self.ensure_source_exists()
logger.info(
'Adding remote repo (%s) with alias (%s)...' % (
self.repo,
self.remote
))
self.git.call('remote', 'add', self.remote, self.repo)
def clone(self, tip_only=False):
self.ensure_source_does_not_exist()
self.ensure_repo_exists()
logger.info('Cloning tree...')
self.source.parent.make()
self.git.call('clone',
self.repo,
self.source.name,
'--depth' if tip_only else None,
'1' if tip_only else None,
'--branch' if tip_only else None,
self.branch_name if tip_only else None,
cwd=self.source.parent.path)
self.checkout()
def checkout(self, ref=None):
self.ensure_source_exists()
logger.info('Checking out %s...' % (ref or self.branch_name))
self.git.call('checkout', (ref or self.branch_name))
def new_branch(self, branch_name):
logger.info('Setting up new branch %s...' % self.branch_name)
self.git.call('branch', branch_name)
@contextmanager
def branch(self, branch_name):
old_branch = self.branch_name
self.branch_name = branch_name
self.checkout()
yield
self.branch_name = old_branch
self.checkout()
def pull(self, tip_only=False):
if self.source.exists:
if self.has_changes(check_remote=False):
raise GitException(
'Source [%s] has changes. Please sync it with the remote.'
% self.source)
logger.info('Pulling remote changes ...')
self.git.call('pull', self.remote, self.branch_name)
else:
self.clone(tip_only=tip_only)
def fetch(self):
self.ensure_source_exists()
logger.info('Fetching source...')
self.git.call('fetch', self.remote)
def fetch_ref(self, ref, local=None, checkout=True, notags=False):
if not self.source.exists:
self.source.make()
self.make()
self.add_remote()
logger.info('Fetching ref...')
self.git.call('fetch',
'-n' if notags else '',
self.remote,
ref + (':' + local) if local else '')
if local:
self.checkout(local)
def push(self, force=False, set_upstream=False, ref=None):
self.ensure_source_exists()
logger.info('Pushing changes...')
push_ref = (ref or self.branch_name)
self.git.call('push',
'-f' if force else None,
'-u' if set_upstream else None,
self.remote,
push_ref)
def merge(self, ref, ff_only=True):
self.ensure_source_exists()
self.checkout()
logger.info('Merging %s into %s...' % (ref, self.branch_name))
try:
self.git.get('merge',
'--ff-only' if ff_only else None,
ref)
except CalledProcessError, called:
message = None
if ff_only:
message = 'Cannot fast forward the merge.'
else:
message = 'Conflicts detected. Unable to merge.'
logger.error(called.output)
raise GitException(message)
def maker(source, target):
stuff = ['apps', 'lib', 'app.js', 'package.json']
for f in stuff:
fs = Folder.file_or_folder(source.child(f))
fs.copy_to(target)
class SphinxPlugin(Plugin):
"""The plugin class for rendering sphinx-generated documentation."""
def __init__(self, site):
self.sphinx_build_dir = None
self._sphinx_config = None
super(SphinxPlugin, self).__init__(site)
@property
def plugin_name(self):
"""The name of the plugin, obivously."""
return "sphinx"
@property
def settings(self):
"""Settings for this plugin.
This property combines default settings with those specified in the
site config to produce the final settings for this plugin.
"""
settings = Expando({})
settings.sanity_check = True
settings.conf_path = "."
settings.block_map = {}
try:
user_settings = getattr(self.site.config, self.plugin_name)
except AttributeError:
pass
else:
for name in dir(user_settings):
if not name.startswith("_"):
setattr(settings,name,getattr(user_settings,name))
return settings
@property
def sphinx_config(self):
"""Configuration options for sphinx.
This is a lazily-generated property giving the options from the
sphinx configuration file. It's generated by actualy executing
the config file, so don't do anything silly in there.
"""
if self._sphinx_config is None:
conf_path = self.settings.conf_path
conf_path = self.site.sitepath.child_folder(conf_path)
# Sphinx always execs the config file in its parent dir.
conf_file = conf_path.child("conf.py")
self._sphinx_config = {"__file__":conf_file}
curdir = os.getcwd()
os.chdir(conf_path.path)
try:
execfile(conf_file,self._sphinx_config)
finally:
os.chdir(curdir)
return self._sphinx_config
def begin_site(self):
"""Event hook for when site processing begins.
This hook checks that the site is correctly configured for building
with sphinx, and adjusts any sphinx-controlled resources so that
hyde will process them correctly.
"""
settings = self.settings
if settings.sanity_check:
self._sanity_check()
# Find and adjust all the resource that will be handled by sphinx.
# We need to:
# * change the deploy name from .rst to .html
# * if a block_map is given, switch off default_block
suffix = self.sphinx_config.get("source_suffix",".rst")
for resource in self.site.content.walk_resources():
if resource.source_file.path.endswith(suffix):
new_name = resource.source_file.name_without_extension + ".html"
target_folder = File(resource.relative_deploy_path).parent
resource.relative_deploy_path = target_folder.child(new_name)
if settings.block_map:
resource.meta.default_block = None
def begin_text_resource(self,resource,text):
"""Event hook for processing an individual resource.
If the input resource is a sphinx input file, this method will replace
replace the text of the file with the sphinx-generated documentation.
Sphinx itself is run lazily the first time this method is called.
This means that if no sphinx-related resources need updating, then
we entirely avoid running sphinx.
"""
suffix = self.sphinx_config.get("source_suffix",".rst")
if not resource.source_file.path.endswith(suffix):
return text
if self.sphinx_build_dir is None:
self._run_sphinx()
output = []
settings = self.settings
sphinx_output = self._get_sphinx_output(resource)
# If they're set up a block_map, use the specific blocks.
# Otherwise, output just the body for use by default_block.
if not settings.block_map:
output.append(sphinx_output["body"])
else:
for (nm,content) in sphinx_output.iteritems():
try:
block = getattr(settings.block_map,nm)
except AttributeError:
pass
else:
output.append("{%% block %s %%}" % (block,))
output.append(content)
output.append("{% endblock %}")
return "\n".join(output)
def site_complete(self):
"""Event hook for when site processing ends.
This simply cleans up any temorary build file.
"""
if self.sphinx_build_dir is not None:
self.sphinx_build_dir.delete()
def _sanity_check(self):
"""Check the current site for sanity.
This method checks that the site is propertly set up for building
things with sphinx, e.g. it has a config file, a master document,
the hyde sphinx extension is enabled, and so-on.
"""
# Check that the sphinx config file actually exists.
try:
sphinx_config = self.sphinx_config
except EnvironmentError:
logger.error("Could not read the sphinx config file.")
conf_path = self.settings.conf_path
conf_path = self.site.sitepath.child_folder(conf_path)
conf_file = conf_path.child("conf.py")
logger.error("Please ensure %s is a valid sphinx config",conf_file)
logger.error("or set sphinx.conf_path to the directory")
logger.error("containing your sphinx conf.py")
raise
# Check that the hyde_json extension is loaded
extensions = sphinx_config.get("extensions",[])
if "hyde.ext.plugins.sphinx" not in extensions:
logger.error("The hyde_json sphinx extension is not configured.")
logger.error("Please add 'hyde.ext.plugins.sphinx' to the list")
logger.error("of extensions in your sphinx conf.py file.")
logger.info("(set sphinx.sanity_check=false to disable this check)")
raise RuntimeError("sphinx is not configured correctly")
# Check that the master doc exists in the source tree.
master_doc = sphinx_config.get("master_doc","index")
master_doc += sphinx_config.get("source_suffix",".rst")
master_doc = os.path.join(self.site.content.path,master_doc)
if not os.path.exists(master_doc):
logger.error("The sphinx master document doesn't exist.")
logger.error("Please create the file %s",master_doc)
logger.error("or change the 'master_doc' setting in your")
logger.error("sphinx conf.py file.")
logger.info("(set sphinx.sanity_check=false to disable this check)")
raise RuntimeError("sphinx is not configured correctly")
# Check that I am *before* the other plugins,
# with the possible exception of MetaPlugin
for plugin in self.site.plugins:
if plugin is self:
break
if not isinstance(plugin,_MetaPlugin):
logger.error("The sphinx plugin is installed after the")
logger.error("plugin %r.",plugin.__class__.__name__)
logger.error("It's quite likely that this will break things.")
logger.error("Please move the sphinx plugin to the top")
logger.error("of the plugins list.")
logger.info("(sphinx.sanity_check=false to disable this check)")
raise RuntimeError("sphinx is not configured correctly")
def _run_sphinx(self):
"""Run sphinx to generate the necessary output files.
This method creates a temporary directory for sphinx's output, then
run sphinx against the Hyde input directory.
"""
logger.info("running sphinx")
self.sphinx_build_dir = Folder(tempfile.mkdtemp())
conf_path = self.site.sitepath.child_folder(self.settings.conf_path)
sphinx_args = ["sphinx-build"]
sphinx_args.extend([
"-b", "hyde_json",
"-c", conf_path.path,
self.site.content.path,
self.sphinx_build_dir.path
])
if sphinx.main(sphinx_args) != 0:
raise RuntimeError("sphinx build failed")
def _get_sphinx_output(self,resource):
"""Get the sphinx output for a given resource.
This returns a dict mapping block names to HTML text fragments.
The most important fragment is "body" which contains the main text
of the document. The other fragments are for things like navigation,
related pages and so-on.
"""
relpath = File(resource.relative_path)
relpath = relpath.parent.child(relpath.name_without_extension+".fjson")
with open(self.sphinx_build_dir.child(relpath),"rb") as f:
return json.load(f)
class Node(Processable):
"""
Represents any folder that is processed by hyde
"""
def __init__(self, source_folder, parent=None):
super(Node, self).__init__(source_folder)
if not source_folder:
raise HydeException("Source folder is required"
" to instantiate a node.")
self.root = self
self.module = None
self.site = None
self.source_folder = Folder(str(source_folder))
self.parent = parent
if parent:
self.root = self.parent.root
self.module = self.parent.module if self.parent.module else self
self.site = parent.site
self.child_nodes = []
self.resources = []
def contains_resource(self, resource_name):
"""
Returns True if the given resource name exists as a file
in this node's source folder.
"""
return File(self.source_folder.child(resource_name)).exists
def get_resource(self, resource_name):
"""
Gets the resource if the given resource name exists as a file
in this node's source folder.
"""
if self.contains_resource(resource_name):
return self.root.resource_from_path(
self.source_folder.child(resource_name))
return None
def add_child_node(self, folder):
"""
Creates a new child node and adds it to the list of child nodes.
"""
if folder.parent != self.source_folder:
raise HydeException("The given folder [%s] is not a"
" direct descendant of [%s]" %
(folder, self.source_folder))
node = Node(folder, self)
self.child_nodes.append(node)
return node
def add_child_resource(self, afile):
"""
Creates a new resource and adds it to the list of child resources.
"""
if afile.parent != self.source_folder:
raise HydeException("The given file [%s] is not"
" a direct descendant of [%s]" %
(afile, self.source_folder))
resource = Resource(afile, self)
self.resources.append(resource)
return resource
def walk(self):
"""
Walks the node, first yielding itself then
yielding the child nodes depth-first.
"""
yield self
for child in sorted([node for node in self.child_nodes]):
for node in child.walk():
yield node
def rwalk(self):
"""
Walk the node upward, first yielding itself then
yielding its parents.
"""
x = self
while x:
yield x
x = x.parent
def walk_resources(self):
"""
Walks the resources in this hierarchy.
"""
for node in self.walk():
for resource in sorted([resource for resource in node.resources]):
yield resource
@property
def relative_path(self):
"""
Gets the path relative to the root folder (Content, Media, Layout)
"""
return self.source_folder.get_relative_path(self.root.source_folder)
# -*- coding: utf-8 -*-
"""
Use nose
`$ pip install nose`
`$ nosetests`
"""
from fswrap import File, Folder
from gitbot.lib.cat import Concat, FILES, BEGIN_INDENT, END_INDENT, END
from nose.tools import with_setup
import tempfile
TEMP = Folder(tempfile.gettempdir())
START = '''
<!doctype html>
<html>'''
HEAD = '''
<head>
<style>
body {
background-color: @black;
}
</style>
<title>@title</title>
</head>'''
BODY = '''
<body>
<h1>This is a simple html file</h1>
<h2>That split up and combined.</h2>
</body>'''
STOP = '''
class Config(Expando):
"""
Represents the hyde configuration file
"""
def __init__(self, sitepath, config_file=None, config_dict=None):
self.default_config = dict(
mode='production',
simple_copy=[],
content_root='content',
deploy_root='deploy',
media_root='media',
layout_root='layout',
media_url='/media',
base_url="/",
encode_safe=None,
not_found='404.html',
plugins=[],
ignore=["*~", "*.bak", ".hg", ".git", ".svn"],
meta={
"nodemeta": 'meta.yaml'
}
)
self.config_file = config_file
self.config_dict = config_dict
self.load_time = datetime.min
self.config_files = []
self.sitepath = Folder(sitepath)
super(Config, self).__init__(self.load())
@property
def last_modified(self):
return max((conf.last_modified for conf in self.config_files))
def needs_refresh(self):
if not self.config_files:
return True
return any((conf.has_changed_since(self.load_time)
for conf in self.config_files))
def load(self):
conf = dict(**self.default_config)
conf.update(self.read_config(self.config_file))
if self.config_dict:
conf.update(self.config_dict)
return conf
def reload(self):
if not self.config_file:
return
self.update(self.load())
def read_config(self, config_file):
"""
Reads the configuration file and updates this
object while allowing for inherited configurations.
"""
conf_file = self.sitepath.child(
config_file if
config_file else 'site.yaml')
conf = {}
if File(conf_file).exists:
self.config_files.append(File(conf_file))
logger.info("Reading site configuration from [%s]", conf_file)
with codecs.open(conf_file, 'r', 'utf-8') as stream:
conf = yaml.load(stream)
if 'extends' in conf:
parent = self.read_config(conf['extends'])
parent.update(conf)
conf = parent
self.load_time = datetime.now()
return conf
@property
def deploy_root_path(self):
"""
Derives the deploy root path from the site path
"""
return _expand_path(self.sitepath, self.deploy_root)
@property
def content_root_path(self):
"""
Derives the content root path from the site path
"""
return _expand_path(self.sitepath, self.content_root)
@property
def media_root_path(self):
"""
Derives the media root path from the content path
"""
path = Folder(self.content_root).child(self.media_root)
return _expand_path(self.sitepath, path)
@property
def layout_root_path(self):
"""
Derives the layout root path from the site path
"""
return _expand_path(self.sitepath, self.layout_root)
