def test_read(self):
"""
Check that the config router reads correctly from the filesystem
"""
router = ConfigRouter([self.path1, self.path2])
self.assertEqual(router.get("a"), 1)
self.assertEqual(router.get("b"), 2)
self.assertEqual(router.get("c"), None)
def test_duplicate(self):
"""
Check that the config router handles duplicate files properly.
"""
router = ConfigRouter([self.path1, self.path1])
router.set("a", 3)
router.write()
self.conf1.load()
self.assertEqual(self.conf1.get("a"), 3)
def test_read_write(self):
"""
Check that our config is readable after writing it
"""
router = ConfigRouter([self.path1, self.path2])
router.set("a", 3)
router.set("b", 4)
self.assertEqual(3, router.get("a"))
self.assertEqual(4, router.get("b"))
def test_collision(self):
"""
Check that we get the right key when there is a collision
"""
self.conf1.set("b", 3)
self.conf2.set("a", 4)
self.conf1.write()
self.conf2.write()
router = ConfigRouter([self.path1, self.path2])
self.assertEqual(router.get("a"), 1)
self.assertEqual(router.get("b"), 3)
def test_missing_file(self):
"""
Test that we don't throw on a missing file, and that the configuration
remains in a consistent state.
"""
wrong_path = os.path.join(self.path, "does_not_exist.json")
self.conf1.set("context", {"k1":"v1"})
self.conf1.write()
router = ConfigRouter([wrong_path, self.path1])
self.assertEqual(router.get("context").get("k1"), "v1")
def test_nested(self):
"""
Test that we support nested config for context
"""
self.conf1.set("context", {"k1": "v1"})
self.conf2.set("context", {"k2": "v2"})
self.conf1.write()
self.conf2.write()
router = ConfigRouter([self.path1, self.path2])
context = router.get("context", default={}, nested=True)
self.assertEqual(context.get("k1"), "v1")
self.assertEqual(context.get("k2"), "v2")
def test_broken_file(self):
"""
Test that we don't throw on a broken file, and that the configuration
remains in a consistent state.
"""
with open(self.path1, "w") as f:
f.write("{broken}")
self.conf2.set("context", {"k1":"v1"})
self.conf2.write()
router = ConfigRouter([self.path1, self.path2])
self.assertEqual(router.get("context").get("k1"), "v1")
def test_write(self):
"""
Check that the config router writes correctly to the filesystem
"""
router = ConfigRouter([self.path1, self.path2])
router.set("a", 3)
router.set("b", 4)
router.write()
self.conf1.load()
self.conf2.load()
self.assertEqual(self.conf1.get("a"), 3)
self.assertEqual(self.conf1.get("b"), None)
self.assertEqual(self.conf2.get("b"), 4)
self.assertEqual(self.conf2.get("a"), None)
def setUp(self):
super(SiteTestCase, self).setUp()
self.config_path = os.path.join(self.path, 'config.json')
self.conf = ConfigRouter([self.config_path])
self.conf.set('site-url', 'http://example.com/')
for k, v in self.get_config_for_test().items():
self.conf.set(k, v)
self.conf.write()
self.site = Site(self.path, [self.config_path])
self.site._parallel = PARALLEL_DISABLED
def __init__(self, path, config_paths=None, ui=None,
PluginManagerClass=None, ExternalManagerClass=None, DeploymentEngineClass=None,
verb=VERB_UNKNOWN):
# Load the config engine
if config_paths is None:
config_paths = []
self.config = ConfigRouter(config_paths)
self.verb = verb
# Load site-specific config values
self.prettify_urls = self.config.get('prettify', False)
self.compress_extensions = self.config.get('compress', ['html', 'css', 'js', 'txt', 'xml'])
self.fingerprint_extensions = self.config.get('fingerprint', [])
self.locale = self.config.get("locale", None)
# Verify our location looks correct
self.path = path
self.verify_path()
# Load Managers
if ui is None:
ui = ui_module
self.ui = ui
if PluginManagerClass is None:
PluginManagerClass = PluginManager
self.plugin_manager = PluginManagerClass(self,
[
CustomPluginsLoader(self.plugin_path), # User plugins
ObjectsPluginLoader([ # Builtin plugins
ContextPlugin(), CacheDurationPlugin(),
IgnorePatternsPlugin(), PageContextCompatibilityPlugin(),
])
]
)
if ExternalManagerClass is None:
ExternalManagerClass = ExternalManager
self.external_manager = ExternalManagerClass(self)
if DeploymentEngineClass is None:
hosting_provider = self.config.get("provider", DEFAULT_PROVIDER)
DeploymentEngineClass = get_deployment_engine_class(hosting_provider)
assert DeploymentEngineClass is not None, \
"Could not load Deployment for Provider: {0}".format(hosting_provider)
self.deployment_engine = DeploymentEngineClass(self)
# Load Django settings
self.setup()
class SiteTestCase(BaseTestCase):
def setUp(self):
super(SiteTestCase, self).setUp()
self.config_path = os.path.join(self.path, 'config.json')
self.conf = ConfigRouter([self.config_path])
self.conf.set('site-url', 'http://example.com/')
for k, v in self.get_config_for_test().items():
self.conf.set(k, v)
self.conf.write()
self.site = Site(self.path, [self.config_path])
self.site._parallel = PARALLEL_DISABLED
def get_config_for_test(self):
"""
Hook to set config keys in other tests.
"""
return {}
class Site(SiteCompatibilityLayer):
_path = None
_parallel = PARALLEL_CONSERVATIVE #TODO: Test me
_static = None
def __init__(self, path, config_paths=None, ui=None,
PluginManagerClass=None, ExternalManagerClass=None, DeploymentEngineClass=None):
# Load the config engine
if config_paths is None:
config_paths = []
self.config = ConfigRouter(config_paths)
# Load site-specific config values
self.prettify_urls = self.config.get('prettify', False)
self.compress_extensions = self.config.get('compress', ['html', 'css', 'js', 'txt', 'xml'])
self.fingerprint_extensions = self.config.get('fingerprint', [])
self.locale = self.config.get("locale", None)
# Verify our location looks correct
self.path = path
self.verify_path()
# Load Managers
if ui is None:
ui = ui_module
self.ui = ui
if PluginManagerClass is None:
PluginManagerClass = PluginManager
self.plugin_manager = PluginManagerClass(self,
[
CustomPluginsLoader(self.plugin_path), # User plugins
ObjectsPluginLoader([ # Builtin plugins
ContextPlugin(), CacheDurationPlugin(),
IgnorePatternsPlugin(), PageContextCompatibilityPlugin(),
])
]
)
if ExternalManagerClass is None:
ExternalManagerClass = ExternalManager
self.external_manager = ExternalManagerClass(self)
if DeploymentEngineClass is None:
hosting_provider = self.config.get("provider", DEFAULT_PROVIDER)
DeploymentEngineClass = get_deployment_engine_class(hosting_provider)
assert DeploymentEngineClass is not None, \
"Could not load Deployment for Provider: {0}".format(hosting_provider)
self.deployment_engine = DeploymentEngineClass(self)
# Load Django settings
self.setup()
@property
def url(self):
return self.config.get('site-url')
@url.setter
def url(self, value):
self.config.set('site-url', value)
self.config.write()
def verify_url(self):
"""
We need the site url to generate the sitemap.
"""
#TODO: Make a "required" option in the config.
#TODO: Use URL tags in the sitemap
# if self.url is None:
# self.url = self.ui.prompt_url("Enter your site URL (e.g. http://example.com/)")
@property
def path(self):
return self._path
@path.setter
def path(self, path):
self._path = path
self.build_path = os.path.join(path, '.build')
self.deploy_path = os.path.join(path, '.deploy')
self.template_path = os.path.join(path, 'templates')
self.page_path = os.path.join(path, 'pages')
self.plugin_path = os.path.join(path, 'plugins')
self.static_path = os.path.join(path, 'static')
self.script_path = os.path.join(os.getcwd(), __file__)
self.locale_path = os.path.join(path, "locale")
def setup(self):
"""
Configure django to use both our template and pages folder as locations
to look for included templates.
"""
settings = {
"TEMPLATE_DIRS": [self.template_path, self.page_path],
"INSTALLED_APPS": ['django_markwhat'],
}
if self.locale is not None:
settings.update({
"USE_I18N": True,
"USE_L10N": False,
"LANGUAGE_CODE": self.locale,
"LOCALE_PATHS": [self.locale_path],
})
django.conf.settings.configure(**settings)
# - Importing here instead of the top-level makes it work on Python 3.x (!)
# - loading add_to_builtins from loader implictly loads the loader_tags built-in
# - Injecting our tags using add_to_builtins ensures that Cactus tags don't require an import
from django.template.loader import add_to_builtins
add_to_builtins('cactus.template_tags')
def verify_path(self):
"""
Check if this path looks like a Cactus website
"""
required_subfolders = ['pages', 'static', 'templates', 'plugins']
if self.locale is not None:
required_subfolders.append('locale')
for p in required_subfolders:
if not os.path.isdir(os.path.join(self.path, p)):
logger.error('This does not look like a (complete) cactus project (missing "%s" subfolder)', p)
sys.exit(1)
@memoize
def context(self):
"""
Base context for the site: all the html pages.
"""
ctx = {
'CACTUS': {
'pages': [p for p in self.pages() if p.is_html()],
'static': [p for p in self.static()]
},
'__CACTUS_SITE__': self,
}
# Also make lowercase work
ctx['cactus'] = ctx['CACTUS']
return ctx
def make_messages(self):
"""
Generate the .po files for the site.
"""
if self.locale is None:
logger.error("You should set a locale in your configuration file before running this command.")
return
message_maker = MessageMaker(self)
message_maker.execute()
def compile_messages(self):
"""
Remove pre-existing compiled language files, and re-compile.
"""
#TODO: Make this cleaner
mo_path = os.path.join(self.locale_path, self.locale, "LC_MESSAGES", "django.mo")
try:
os.remove(mo_path)
except OSError:
# No .mo file yet
pass
message_compiler = MessageCompiler(self)
message_compiler.execute()
def clean(self):
"""
Remove all build files.
"""
logger.debug("*** CLEAN %s", self.path)
if os.path.isdir(self.build_path):
shutil.rmtree(self.build_path)
def build(self):
"""
Generate fresh site from templates.
"""
logger.debug("*** BUILD %s", self.path)
self.verify_url()
# Reset the static content
self._static = None
#TODO: Facility to reset the site, and reload config.
#TODO: Currently, we can't build a site instance multiple times
self.plugin_manager.reload() # Reload in case we're running on the server # We're still loading twice!
self.plugin_manager.preBuild(self)
logger.debug('Plugins: %s', ', '.join([p.plugin_name for p in self.plugin_manager.plugins]))
logger.debug('Processors: %s', ', '.join([p.__name__ for p in self.external_manager.processors]))
logger.debug('Optimizers: %s', ', '.join([p.__name__ for p in self.external_manager.optimizers]))
# Make sure the build path exists
if not os.path.exists(self.build_path):
os.mkdir(self.build_path)
# Prepare translations
if self.locale is not None:
self.compile_messages()
#TODO: Check the command actually completes (msgfmt might not be on the PATH!)
# Copy the static files
self.buildStatic()
# Always clean out the pages
build_static_path = os.path.join(self.build_path, "static")
for path in os.listdir(self.build_path):
path = os.path.join(self.build_path, path)
if path != build_static_path:
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
# Render the pages to their output files
mapper = multiMap if self._parallel >= PARALLEL_AGGRESSIVE else map_apply
mapper(lambda p: p.build(), self.pages())
self.plugin_manager.postBuild(self)
for static in self.static():
if os.path.isdir(static.pre_dir):
shutil.rmtree(static.pre_dir)
def static(self):
"""
Retrieve a list of static files for the site
"""
if self._static is None:
self._static = []
for path in fileList(self.static_path, relative=True):
full_path = os.path.join(self.static_path, path)
if os.path.islink(full_path):
if not os.path.exists(os.path.realpath(full_path)):
logger.warning("Skipping symlink that points to unexisting file:\n%s", full_path)
continue
self._static.append(Static(self, path))
return self._static
def _get_resource(self, src_url, resources):
if is_external(src_url):
return src_url
for split_char in ["#", "?"]:
if split_char in src_url:
src_url = src_url.split(split_char)[0]
resources_dict = dict((resource.link_url, resource) for resource in resources)
if src_url in resources_dict:
return resources_dict[src_url].final_url
return None
def _get_url(self, src_url, resources):
return self._get_resource(src_url, resources)
def get_url_for_static(self, src_path):
return self._get_url(src_path, self.static())
def get_url_for_page(self, src_path):
return self._get_url(src_path, self.pages())
def buildStatic(self):
"""
Build static files (pre-process, copy to static folder)
"""
mapper = multiMap if self._parallel > PARALLEL_DISABLED else map_apply
mapper(lambda s: s.build(), self.static())
def pages(self):
"""
List of pages.
"""
if not hasattr(self, "_page_cache"):
self._page_cache = {}
pages = []
for path in fileList(self.page_path, relative=True):
if path.endswith("~"):
continue
if path not in self._page_cache:
logger.debug("Found page: %s", path)
self._page_cache[path] = Page(self, path)
pages.append(self._page_cache[path])
return pages
def _rebuild_should_ignore(self, file_path):
file_relative_path = os.path.relpath(file_path, self.path)
# Ignore anything in a hidden folder like .git
for path_part in file_relative_path.split(os.path.sep):
if path_part.startswith("."):
return True
if file_path.startswith(self.page_path):
return False
if file_path.startswith(self.template_path):
return False
if file_path.startswith(self.static_path):
return False
if file_path.startswith(self.plugin_path):
return False
return True
def _rebuild(self, changes):
logger.debug("*** REBUILD %s", self.path)
logger.info('*** Rebuilding (%s changed)' % self.path)
# We will pause the listener while building so scripts that alter the output
# like coffeescript and less don't trigger the listener again immediately.
self.listener.pause()
try:
#TODO: Fix this.
#TODO: The static files should handle collection of their static folder on their own
#TODO: The static files should not run everything on __init__
#TODO: Only rebuild static files that changed
# We need to "clear out" the list of static first. Otherwise, processors will not run again
# They run on __init__ to run before fingerprinting, and the "built" static files themselves,
# which are in a temporary folder, have been deleted already!
# self._static = None
self.build()
except Exception as e:
logger.info('*** Error while building\n%s', e)
traceback.print_exc(file=sys.stdout)
changed_file_extension = set(map(lambda x: os.path.splitext(x)[1], changes["changed"]))
reload_css_file_extenstions = set([".css", ".sass", ".scss", ".styl"])
# When we have changes, we want to refresh the browser tabs with the updates.
# Mostly we just refresh the browser except when there are just css changes,
# then we reload the css in place.
local_hosts = [
"http://127.0.0.1:%s" % self._port,
"http://localhost:%s" % self._port,
"http://0.0.0.0:%s" % self._port
]
if len(changes["added"]) == 0 and len(changes["deleted"]) == 0 and changed_file_extension.issubset(reload_css_file_extenstions):
# browserReloadCSS(local_hosts)
self.server.reloadCSS()
else:
# browserReload(local_hosts)
self.server.reloadPage()
self.listener.resume()
def serve(self, browser=True, port=8000):
"""
Start a http server and rebuild on changes.
"""
self._parallel = PARALLEL_DISABLED
self._port = port
self.clean()
self.build()
logger.info('Running webserver at http://127.0.0.1:%s for %s' % (port, self.build_path))
ipc.signal("server.didstart")
logger.info('Type control-c to exit')
os.chdir(self.build_path)
self.listener = Listener(self.path, self._rebuild, ignore=self._rebuild_should_ignore)
self.listener.run()
self.server = WebServer(self.build_path, port=port)
try:
self.server.start()
# if browser is True:
# webbrowser.open('http://127.0.0.1:%s' % port)
except (KeyboardInterrupt, SystemExit):
self.server.stop()
logger.info("Bye")
def upload(self):
# Make sure we have internet
if not internetWorking():
logger.info('There does not seem to be internet here, check your connection')
return
logger.debug('Start upload')
self.build_path = self.deploy_path
self.clean()
self.build()
self.plugin_manager.preDeploy(self)
totalFiles = self.deployment_engine.deploy()
changedFiles = [r for r in totalFiles if r['changed']]
self.plugin_manager.postDeploy(self)
# Display done message and some statistics
logger.info('\nDone\n')
logger.info('%s total files with a size of %s' %
(len(totalFiles), fileSize(sum([r['size'] for r in totalFiles]))))
logger.info('%s changed files with a size of %s' %
(len(changedFiles), fileSize(sum([r['size'] for r in changedFiles]))))
logger.info('\nhttp://%s\n' % self.config.get('aws-bucket-website')) #TODO: Fix
def domain_setup(self):
# Make sure we have internet
if not internetWorking():
logger.info('There does not seem to be internet here, check your connection')
return
self.deployment_engine.domain_setup()
self.domain_list()
def domain_list(self):
self.deployment_engine.domain_list()
class Site(SiteCompatibilityLayer):
_path = None
_parallel = PARALLEL_CONSERVATIVE #TODO: Test me
_static = None
def __init__(self, path, config_paths=None, ui=None,
PluginManagerClass=None, ExternalManagerClass=None, DeploymentEngineClass=None):
# Load the config engine
if config_paths is None:
config_paths = []
self.config = ConfigRouter(config_paths)
# Load site-specific config values
self.prettify_urls = self.config.get('prettify', False)
self.compress_extensions = self.config.get('compress', ['html', 'css', 'js', 'txt', 'xml'])
self.fingerprint_extensions = self.config.get('fingerprint', [])
self.locale = self.config.get("locale", None)
# Verify our location looks correct
self.path = path
self.verify_path()
# Load Managers
if ui is None:
ui = ui_module
self.ui = ui
if PluginManagerClass is None:
PluginManagerClass = PluginManager
self.plugin_manager = PluginManagerClass(self,
[
CustomPluginsLoader(self.plugin_path), # User plugins
ObjectsPluginLoader([ # Builtin plugins
ContextPlugin(), CacheDurationPlugin(),
IgnorePatternsPlugin(), PageContextCompatibilityPlugin(),
])
]
)
if ExternalManagerClass is None:
ExternalManagerClass = ExternalManager
self.external_manager = ExternalManagerClass(self)
if DeploymentEngineClass is None:
hosting_provider = self.config.get("provider", DEFAULT_PROVIDER)
DeploymentEngineClass = get_deployment_engine_class(hosting_provider)
assert DeploymentEngineClass is not None, \
"Could not load Deployment for Provider: {0}".format(hosting_provider)
self.deployment_engine = DeploymentEngineClass(self)
# Load Django settings
self.setup()
@property
def url(self):
return self.config.get('site-url')
@url.setter
def url(self, value):
self.config.set('site-url', value)
self.config.write()
def verify_url(self):
"""
We need the site url to generate the sitemap.
"""
#TODO: Make a "required" option in the config.
#TODO: Use URL tags in the sitemap
if self.url is None:
self.url = self.ui.prompt_url("Enter your site URL (e.g. http://example.com/), you can change it later.")
@property
def path(self):
return self._path
@path.setter
def path(self, path):
self._path = path
self.build_path = os.path.join(path, '.build')
self.template_path = os.path.join(path, 'templates')
self.page_path = os.path.join(path, 'pages')
self.plugin_path = os.path.join(path, 'plugins')
self.static_path = os.path.join(path, 'static')
self.script_path = os.path.join(os.getcwd(), __file__)
self.locale_path = os.path.join(path, "locale")
def setup(self):
"""
Configure django to use both our template and pages folder as locations
to look for included templates.
"""
settings = {
"TEMPLATE_DIRS": [self.template_path, self.page_path],
"INSTALLED_APPS": ['django.contrib.markup'],
}
if self.locale is not None:
settings.update({
"USE_I18N": True,
"USE_L10N": False,
"LANGUAGE_CODE": self.locale,
"LOCALE_PATHS": [self.locale_path],
})
django.conf.settings.configure(**settings)
add_to_builtins('cactus.template_tags')
def verify_path(self):
"""
Check if this path looks like a Cactus website
"""
required_subfolders = ['pages', 'static', 'templates', 'plugins']
if self.locale is not None:
required_subfolders.append('locale')
for p in required_subfolders:
if not os.path.isdir(os.path.join(self.path, p)):
logger.error('This does not look like a (complete) cactus project (missing "%s" subfolder)', p)
sys.exit(1)
@memoize
def context(self):
"""
Base context for the site: all the html pages.
"""
ctx = {
'CACTUS': {'pages': [p for p in self.pages() if p.is_html()]},
'__CACTUS_SITE__': self,
}
return ctx
def make_messages(self):
"""
Generate the .po files for the site.
"""
if self.locale is None:
logger.error("You should set a locale in your configuration file before running this command.")
return
message_maker = MessageMaker(self)
message_maker.execute()
def compile_messages(self):
"""
Remove pre-existing compiled language files, and re-compile.
"""
#TODO: Make this cleaner
mo_path = os.path.join(self.locale_path, self.locale, "LC_MESSAGES", "django.mo")
try:
os.remove(mo_path)
except OSError:
# No .mo file yet
pass
message_compiler = MessageCompiler(self)
message_compiler.execute()
def clean(self):
"""
Remove all build files.
"""
if os.path.isdir(self.build_path):
shutil.rmtree(self.build_path)
def build(self):
"""
Generate fresh site from templates.
"""
self.verify_url()
#TODO: Facility to reset the site, and reload config.
#TODO: Currently, we can't build a site instance multiple times
self.plugin_manager.reload() # Reload in case we're running on the server # We're still loading twice!
self.plugin_manager.preBuild(self)
logger.info('Plugins: %s', ', '.join([p.plugin_name for p in self.plugin_manager.plugins]))
logger.info('Processors: %s', ', '.join([p.__name__ for p in self.external_manager.processors]))
logger.info('Optimizers: %s', ', '.join([p.__name__ for p in self.external_manager.optimizers]))
# Make sure the build path exists
if not os.path.exists(self.build_path):
os.mkdir(self.build_path)
# Prepare translations
if self.locale is not None:
self.compile_messages()
#TODO: Check the command actually completes (msgfmt might not be on the PATH!)
# Copy the static files
self.buildStatic()
# Render the pages to their output files
mapper = map
if self._parallel >= PARALLEL_AGGRESSIVE:
mapper = multiMap
mapper(lambda p: p.build(), self.pages())
self.plugin_manager.postBuild(self)
for static in self.static():
shutil.rmtree(static.pre_dir)
def static(self):
"""
Retrieve a list of static files for the site
"""
if self._static is None:
paths = fileList(self.static_path, relative=True)
self._static = [Static(self, path) for path in paths]
return self._static
def _get_resource(self, src_url, resources):
if is_external(src_url):
return src_url
resources_dict = dict((resource.link_url, resource) for resource in resources)
try:
return resources_dict[src_url]
except KeyError:
raise Exception('Resource does not exist: {0}'.format(src_url))
def _get_url(self, src_url, resources):
return self._get_resource(src_url, resources).final_url
def get_url_for_static(self, src_path):
return self._get_url(src_path, self.static())
def get_url_for_page(self, src_path):
return self._get_url(src_path, self.pages())
def buildStatic(self):
"""
Build static files (pre-process, copy to static folder)
"""
mapper = multiMap
if self._parallel <= PARALLEL_DISABLED:
mapper = map
mapper(lambda s: s.build(), self.static())
@memoize
def pages(self):
"""
List of pages.
"""
paths = fileList(self.page_path, relative=True)
paths = filter(lambda x: not x.endswith("~"), paths)
return [Page(self, p) for p in paths]
def serve(self, browser=True, port=8000):
"""
Start a http server and rebuild on changes.
"""
self._parallel = PARALLEL_DISABLED
self.clean()
self.build()
logger.info('Running webserver at 0.0.0.0:%s for %s' % (port, self.build_path))
logger.info('Type control-c to exit')
os.chdir(self.build_path)
def rebuild(changes):
logger.info('*** Rebuilding (%s changed)' % self.path)
# We will pause the listener while building so scripts that alter the output
# like coffeescript and less don't trigger the listener again immediately.
self.listener.pause()
try:
#TODO: Fix this.
#TODO: The static files should handle collection of their static folder on their own
#TODO: The static files should not run everything on __init__
#TODO: Only rebuild static files that changed
# We need to "clear out" the list of static first. Otherwise, processors will not run again
# They run on __init__ to run before fingerprinting, and the "built" static files themselves,
# which are in a temporary folder, have been deleted already!
self._static = None
self.build()
except Exception, e:
logger.info('*** Error while building\n%s', e)
traceback.print_exc(file=sys.stdout)
# When we have changes, we want to refresh the browser tabs with the updates.
# Mostly we just refresh the browser except when there are just css changes,
# then we reload the css in place.
if len(changes["added"]) == 0 and \
len(changes["deleted"]) == 0 and \
set(map(lambda x: os.path.splitext(x)[1], changes["changed"])) == set([".css"]):
browserReloadCSS('http://127.0.0.1:%s' % port)
else:
browserReload('http://127.0.0.1:%s' % port)
self.listener.resume()
self.listener = Listener(self.path, rebuild, ignore=lambda x: '/.build/' in x)
self.listener.run()
try:
httpd = Server(("", port), RequestHandler)
except socket.error:
logger.info('Could not start webserver, port is in use. To use another port:')
logger.info(' cactus serve %s' % (int(port) + 1))
return
if browser is True:
webbrowser.open('http://127.0.0.1:%s' % port)
try:
httpd.serve_forever()
except (KeyboardInterrupt, SystemExit):
httpd.server_close()
logger.info('See you!')
class Site(SiteCompatibilityLayer):
_path = None
_parallel = PARALLEL_CONSERVATIVE #TODO: Test me
_static = None
def __init__(self,
path,
config_paths=None,
ui=None,
PluginManagerClass=None,
ExternalManagerClass=None,
DeploymentEngineClass=None):
# Load the config engine
if config_paths is None:
config_paths = []
self.config = ConfigRouter(config_paths)
# Load site-specific config values
self.prettify_urls = self.config.get('prettify', False)
self.compress_extensions = self.config.get(
'compress', ['html', 'css', 'js', 'txt', 'xml'])
self.fingerprint_extensions = self.config.get('fingerprint', [])
self.locale = self.config.get("locale", None)
# Verify our location looks correct
self.path = path
self.verify_path()
# Load Managers
if ui is None:
ui = ui_module
self.ui = ui
if PluginManagerClass is None:
PluginManagerClass = PluginManager
self.plugin_manager = PluginManagerClass(
self,
[
CustomPluginsLoader(self.plugin_path), # User plugins
ObjectsPluginLoader([ # Builtin plugins
ContextPlugin(),
CacheDurationPlugin(),
IgnorePatternsPlugin(),
PageContextCompatibilityPlugin(),
])
])
if ExternalManagerClass is None:
ExternalManagerClass = ExternalManager
self.external_manager = ExternalManagerClass(self)
if DeploymentEngineClass is None:
hosting_provider = self.config.get("provider", DEFAULT_PROVIDER)
DeploymentEngineClass = get_deployment_engine_class(
hosting_provider)
assert DeploymentEngineClass is not None, \
"Could not load Deployment for Provider: {0}".format(hosting_provider)
self.deployment_engine = DeploymentEngineClass(self)
# Load Django settings
self.setup()
@property
def url(self):
return self.config.get('site-url')
@url.setter
def url(self, value):
self.config.set('site-url', value)
self.config.write()
def verify_url(self):
"""
We need the site url to generate the sitemap.
"""
#TODO: Make a "required" option in the config.
#TODO: Use URL tags in the sitemap
# if self.url is None:
# self.url = self.ui.prompt_url("Enter your site URL (e.g. http://example.com/)")
@property
def path(self):
return self._path
@path.setter
def path(self, path):
self._path = path
self.build_path = os.path.join(path, '.build')
self.deploy_path = os.path.join(path, '.deploy')
self.template_path = os.path.join(path, 'templates')
self.page_path = os.path.join(path, 'pages')
self.plugin_path = os.path.join(path, 'plugins')
self.static_path = os.path.join(path, 'static')
self.script_path = os.path.join(os.getcwd(), __file__)
self.locale_path = os.path.join(path, "locale")
def setup(self):
"""
Configure django to use both our template and pages folder as locations
to look for included templates.
"""
settings = {
"TEMPLATE_DIRS": [self.template_path, self.page_path],
"INSTALLED_APPS": ['django_markwhat'],
}
if self.locale is not None:
settings.update({
"USE_I18N": True,
"USE_L10N": False,
"LANGUAGE_CODE": self.locale,
"LOCALE_PATHS": [self.locale_path],
})
django.conf.settings.configure(**settings)
# - Importing here instead of the top-level makes it work on Python 3.x (!)
# - loading add_to_builtins from loader implictly loads the loader_tags built-in
# - Injecting our tags using add_to_builtins ensures that Cactus tags don't require an import
from django.template.loader import add_to_builtins
add_to_builtins('cactus.template_tags')
def verify_path(self):
"""
Check if this path looks like a Cactus website
"""
required_subfolders = ['pages', 'static', 'templates', 'plugins']
if self.locale is not None:
required_subfolders.append('locale')
for p in required_subfolders:
if not os.path.isdir(os.path.join(self.path, p)):
logger.error(
'This does not look like a (complete) cactus project (missing "%s" subfolder)',
p)
sys.exit(1)
@memoize
def context(self):
"""
Base context for the site: all the html pages.
"""
ctx = {
'CACTUS': {
'pages': [p for p in self.pages() if p.is_html()],
'static': [p for p in self.static()]
},
'__CACTUS_SITE__': self,
}
# Also make lowercase work
ctx['cactus'] = ctx['CACTUS']
return ctx
def make_messages(self):
"""
Generate the .po files for the site.
"""
if self.locale is None:
logger.error(
"You should set a locale in your configuration file before running this command."
)
return
message_maker = MessageMaker(self)
message_maker.execute()
def compile_messages(self):
"""
Remove pre-existing compiled language files, and re-compile.
"""
#TODO: Make this cleaner
mo_path = os.path.join(self.locale_path, self.locale, "LC_MESSAGES",
"django.mo")
try:
os.remove(mo_path)
except OSError:
# No .mo file yet
pass
message_compiler = MessageCompiler(self)
message_compiler.execute()
def clean(self):
"""
Remove all build files.
"""
logger.debug("*** CLEAN %s", self.path)
if os.path.isdir(self.build_path):
shutil.rmtree(self.build_path)
def build(self):
"""
Generate fresh site from templates.
"""
logger.debug("*** BUILD %s", self.path)
self.verify_url()
# Reset the static content
self._static = None
self._static_resources_dict = None
#TODO: Facility to reset the site, and reload config.
#TODO: Currently, we can't build a site instance multiple times
self.plugin_manager.reload(
) # Reload in case we're running on the server # We're still loading twice!
self.plugin_manager.preBuild(self)
logger.debug(
'Plugins: %s',
', '.join([p.plugin_name for p in self.plugin_manager.plugins]))
logger.debug(
'Processors: %s',
', '.join([p.__name__ for p in self.external_manager.processors]))
logger.debug(
'Optimizers: %s',
', '.join([p.__name__ for p in self.external_manager.optimizers]))
# Make sure the build path exists
if not os.path.exists(self.build_path):
os.mkdir(self.build_path)
# Prepare translations
if self.locale is not None:
self.compile_messages()
#TODO: Check the command actually completes (msgfmt might not be on the PATH!)
# Copy the static files
self.buildStatic()
# Always clean out the pages
build_static_path = os.path.join(self.build_path, "static")
for path in os.listdir(self.build_path):
path = os.path.join(self.build_path, path)
if path != build_static_path:
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
# Render the pages to their output files
mapper = multiMap if self._parallel >= PARALLEL_AGGRESSIVE else map_apply
mapper(lambda p: p.build(), self.pages())
self.plugin_manager.postBuild(self)
for static in self.static():
if os.path.isdir(static.pre_dir):
shutil.rmtree(static.pre_dir)
def static(self):
"""
Retrieve a list of static files for the site
"""
if self._static is None:
self._static = []
for path in fileList(self.static_path, relative=True):
full_path = os.path.join(self.static_path, path)
if os.path.islink(full_path):
if not os.path.exists(os.path.realpath(full_path)):
logger.warning(
"Skipping symlink that points to unexisting file:\n%s",
full_path)
continue
self._static.append(Static(self, path))
return self._static
def static_resources_dict(self):
"""
Retrieve a dictionary mapping URL's to static files
"""
if self._static_resources_dict is None:
self._static_resources_dict = dict(
(resource.link_url, resource) for resource in self.static())
return self._static_resources_dict
def _get_resource(self, src_url, resources):
if is_external(src_url):
return src_url
for split_char in ["#", "?"]:
if split_char in src_url:
src_url = src_url.split(split_char)[0]
if src_url in resources:
return resources[src_url].final_url
return None
def _get_url(self, src_url, resources):
return self._get_resource(src_url, resources)
def get_url_for_static(self, src_path):
return self._get_url(src_path, self.static_resources_dict())
def get_url_for_page(self, src_path):
return self._get_url(
src_path,
dict((resource.link_url, resource) for resource in self.pages()))
def buildStatic(self):
"""
Build static files (pre-process, copy to static folder)
"""
mapper = multiMap if self._parallel > PARALLEL_DISABLED else map_apply
mapper(lambda s: s.build(), self.static())
def pages(self):
"""
List of pages.
"""
if not hasattr(self, "_page_cache"):
self._page_cache = {}
pages = []
for path in fileList(self.page_path, relative=True):
if path.endswith("~"):
continue
if path not in self._page_cache:
logger.debug("Found page: %s", path)
self._page_cache[path] = Page(self, path)
pages.append(self._page_cache[path])
return pages
def _rebuild_should_ignore(self, file_path):
file_relative_path = os.path.relpath(file_path, self.path)
# Ignore anything in a hidden folder like .git
for path_part in file_relative_path.split(os.path.sep):
if path_part.startswith("."):
return True
if file_path.startswith(self.page_path):
return False
if file_path.startswith(self.template_path):
return False
if file_path.startswith(self.static_path):
return False
if file_path.startswith(self.plugin_path):
return False
return True
def _rebuild(self, changes):
logger.debug("*** REBUILD %s", self.path)
logger.info('*** Rebuilding (%s changed)' % self.path)
# We will pause the listener while building so scripts that alter the output
# like coffeescript and less don't trigger the listener again immediately.
self.listener.pause()
try:
#TODO: Fix this.
#TODO: The static files should handle collection of their static folder on their own
#TODO: The static files should not run everything on __init__
#TODO: Only rebuild static files that changed
# We need to "clear out" the list of static first. Otherwise, processors will not run again
# They run on __init__ to run before fingerprinting, and the "built" static files themselves,
# which are in a temporary folder, have been deleted already!
# self._static = None
self.build()
except Exception as e:
logger.info('*** Error while building\n%s', e)
traceback.print_exc(file=sys.stdout)
changed_file_extension = set(
map(lambda x: os.path.splitext(x)[1], changes["changed"]))
reload_css_file_extenstions = set([".css", ".sass", ".scss", ".styl"])
# When we have changes, we want to refresh the browser tabs with the updates.
# Mostly we just refresh the browser except when there are just css changes,
# then we reload the css in place.
local_hosts = [
"http://127.0.0.1:%s" % self._port,
"http://localhost:%s" % self._port,
"http://0.0.0.0:%s" % self._port
]
if len(changes["added"]) == 0 and len(
changes["deleted"]) == 0 and changed_file_extension.issubset(
reload_css_file_extenstions):
# browserReloadCSS(local_hosts)
self.server.reloadCSS()
else:
# browserReload(local_hosts)
self.server.reloadPage()
self.listener.resume()
def serve(self, browser=True, port=8000):
"""
Start a http server and rebuild on changes.
"""
self._parallel = PARALLEL_DISABLED
self._port = port
self.clean()
self.build()
logger.info('Running webserver at http://127.0.0.1:%s for %s' %
(port, self.build_path))
ipc.signal("server.didstart")
logger.info('Type control-c to exit')
with chdir(self.build_path):
self.listener = Listener(self.path,
self._rebuild,
ignore=self._rebuild_should_ignore)
self.listener.run()
self.server = WebServer(self.build_path, port=port)
try:
self.server.start()
# if browser is True:
# webbrowser.open('http://127.0.0.1:%s' % port)
except (KeyboardInterrupt, SystemExit):
self.server.stop()
logger.info("Bye")
def upload(self):
# Make sure we have internet
if not internetWorking():
logger.info(
'There does not seem to be internet here, check your connection'
)
return
logger.debug('Start upload')
self.build_path = self.deploy_path
self.clean()
self.build()
self.plugin_manager.preDeploy(self)
totalFiles = self.deployment_engine.deploy()
changedFiles = [r for r in totalFiles if r['changed']]
self.plugin_manager.postDeploy(self)
# Display done message and some statistics
logger.info('\nDone\n')
logger.info(
'%s total files with a size of %s' %
(len(totalFiles), fileSize(sum([r['size'] for r in totalFiles]))))
logger.info(
'%s changed files with a size of %s' %
(len(changedFiles), fileSize(sum([r['size']
for r in changedFiles]))))
logger.info('\nhttp://%s\n' %
self.config.get('aws-bucket-website')) #TODO: Fix
def domain_setup(self):
# Make sure we have internet
if not internetWorking():
logger.info(
'There does not seem to be internet here, check your connection'
)
return
self.deployment_engine.domain_setup()
self.domain_list()
def domain_list(self):
self.deployment_engine.domain_list()
