def parse_posts(directory):
"""Retrieve all the posts from the directory specified.
Returns a list of the posts sorted in reverse by date."""
posts = []
post_filename_re = re.compile(
".*((\.textile$)|(\.markdown$)|(\.org$)|(\.html$))")
if not os.path.isdir("_posts"):
logger.error("There is no _posts directory")
return []
post_file_names = [f for f in os.listdir(directory) \
if post_filename_re.match(f)]
for post_fn in post_file_names:
post_path = util.path_join(directory,post_fn)
logger.debug("Parsing post: %s" % post_path)
#IMO codecs.open is broken on Win32.
#It refuses to open files without replacing newlines with CR+LF
#reverting to regular open and decode:
src = open(post_path,"r").read().decode(config.blog_post_encoding)
p = Post(src, filename=post_fn)
#Exclude some posts
if not (p.permalink == None):
posts.append(p)
posts.sort(key=operator.attrgetter('date'), reverse=True)
return posts
def do_build(args, load_config=True):
if load_config:
config_init(args)
writer = Writer(output_dir=util.path_join("_site",util.fs_site_path_helper()))
logger.debug("Running user's pre_build() function..")
config.pre_build()
writer.write_site()
logger.debug("Running user's post_build() function..")
config.post_build()
def __write_files(self):
"""Write all files for the blog to _site
Convert all templates to straight HTML
Copy other non-template files directly"""
#find mako templates in template_dir
for root, dirs, files in os.walk("."):
if root.startswith("./"):
root = root[2:]
for d in list(dirs):
#Exclude some dirs
d_path = util.path_join(root,d)
if util.should_ignore_path(d_path):
logger.debug("Ignoring directory: " + d_path)
dirs.remove(d)
try:
util.mkdir(util.path_join(self.output_dir, root))
except OSError: #pragma: no cover
pass
for t_fn in files:
t_fn_path = util.path_join(root, t_fn)
if util.should_ignore_path(t_fn_path):
#Ignore this file.
logger.debug("Ignoring file: " + t_fn_path)
continue
elif t_fn.endswith(".mako"):
logger.info("Processing mako file: " + t_fn_path)
#Process this template file
t_name = t_fn[:-5]
t_file = open(t_fn_path)
template = Template(t_file.read().decode("utf-8"),
output_encoding="utf-8",
lookup=self.template_lookup)
t_file.close()
path = util.path_join(self.output_dir, root, t_name)
html_file = open(path, "w")
# Prepare the "path" variable for the template context.
page_path = util.path_join(root, t_name)
if page_path.startswith('./'):
page_path = page_path[2:]
page_path = '/' + page_path
context = dict(path=page_path, logger=template_logger)
#render the page
html = self.template_render(template, context)
#Write to disk
html_file.write(html)
else:
#Copy this non-template file
f_path = util.path_join(root, t_fn)
logger.debug("Copying file: " + f_path)
out_path = util.path_join(self.output_dir, f_path)
if self.bf.config.site.use_hard_links:
# Try hardlinking first, and if that fails copy
try:
os.link(f_path, out_path)
except StandardError:
shutil.copyfile(f_path, out_path)
else:
shutil.copyfile(f_path, out_path)
def __init__(self, output_dir):
self.config = config
#Base templates are templates (usually in ./_templates) that are only
#referenced by other templates.
self.base_template_dir = util.path_join(".", "_templates")
self.output_dir = output_dir
self.template_lookup = TemplateLookup(
directories=[".", self.base_template_dir],
input_encoding='utf-8', output_encoding='utf-8',
encoding_errors='replace')
def __init__(self, output_dir):
self.config = config
#Base templates are templates (usually in ./_templates) that are only
#referenced by other templates.
self.base_template_dir = util.path_join(".", "_templates")
self.output_dir = output_dir
self.template_lookup = TemplateLookup(
directories=[".", self.base_template_dir],
input_encoding='utf-8', output_encoding='utf-8',
encoding_errors='replace')
def __load_bf_cache(self):
#Template cache object, used to transfer state to/from each template:
self.bf = cache.bf
self.bf.config = self.config
self.bf.writer = self
self.bf.util = util
self.bf.logger = logger
self.bf.filter = filter
if self.config.blog_enabled == True:
self.bf.posts = post.parse_posts("_posts")
self.bf.blog_dir = util.path_join(self.output_dir,self.config.blog_path)
def materialize_template(self, template_name, location, attrs={}):
"""Render a named template with attrs to a location in the _site dir"""
template = self.template_lookup.get_template(template_name)
template.output_encoding = "utf-8"
rendered = self.template_render(template, attrs)
path = util.path_join(self.output_dir, location)
#Create the path if it doesn't exist:
util.mkdir(os.path.split(path)[0])
f = open(path, "w")
f.write(rendered)
f.close()
def load_filter(name):
"""Load a filter directory from the site's _filters directory"""
logger.debug("Loading filter: "+name)
try:
return __loaded_filters[name]
except KeyError:
try:
__loaded_filters[name] = imp.load_source(
"filter_"+name,util.path_join("_filters",name+".py"))
return __loaded_filters[name]
except:
logging.error("Cannot load filter: "+name)
raise
def __write_files(self):
"""Write all files for the blog to _site
Convert all templates to straight HTML
Copy other non-template files directly"""
#find mako templates in template_dir
for root, dirs, files in os.walk("."):
if root.startswith("./"):
root = root[2:]
for d in list(dirs):
#Exclude some dirs
d_path = util.path_join(root, d)
if util.should_ignore_path(d_path):
logger.debug("Ignoring directory: " + d_path)
dirs.remove(d)
try:
util.mkdir(util.path_join(self.output_dir, root))
except OSError: #pragma: no cover
pass
for t_fn in files:
t_fn_path = util.path_join(root, t_fn)
if util.should_ignore_path(t_fn_path):
#Ignore this file.
logger.debug("Ignoring file: " + t_fn_path)
continue
elif t_fn.endswith(".mako"):
logger.info("Processing mako file: " + t_fn_path)
#Process this template file
t_name = t_fn[:-5]
t_file = open(t_fn_path)
template = Template(t_file.read().decode("utf-8"),
output_encoding="utf-8",
lookup=self.template_lookup)
#Remember the original path for later when setting context
template.bf_meta = {"path": t_fn_path}
t_file.close()
path = util.path_join(self.output_dir, root, t_name)
html_file = open(path, "w")
html = self.template_render(template)
#Write to disk
html_file.write(html)
else:
#Copy this non-template file
f_path = util.path_join(root, t_fn)
logger.debug("Copying file: " + f_path)
out_path = util.path_join(self.output_dir, f_path)
if self.config.site.overwrite_warning and os.path.exists(
out_path):
logger.warn(
"Location is used more than once: {0}".format(
f_path))
if self.bf.config.site.use_hard_links:
# Try hardlinking first, and if that fails copy
try:
os.link(f_path, out_path)
except StandardError:
shutil.copyfile(f_path, out_path)
else:
shutil.copyfile(f_path, out_path)
def do_build(args, load_config=True):
if load_config:
config_init(args)
output_dir = util.path_join("_site", util.fs_site_path_helper())
writer = Writer(output_dir=output_dir)
logger.debug("Running user's pre_build() function...")
config.pre_build()
try:
writer.write_site()
logger.debug("Running user's post_build() function...")
config.post_build()
finally:
logger.debug("Running user's build_finally() function...")
config.build_finally()
def materialize_template(self, template_name, location, attrs={}, lookup=None):
"""Render a named template with attrs to a location in the _site dir"""
if lookup==None:
lookup = self.template_lookup
template = lookup.get_template(template_name)
template.output_encoding = "utf-8"
rendered = self.template_render(template, attrs)
path = util.path_join(self.output_dir, location)
#Create the path if it doesn't exist:
util.mkdir(os.path.split(path)[0])
if self.config.site.overwrite_warning and os.path.exists(path):
logger.warn("Location is used more than once: {0}".format(location))
f = open(path, "w")
f.write(rendered)
f.close()
def __write_files(self):
"""Write all files for the blog to _site
Convert all templates to straight HTML
Copy other non-template files directly"""
#find mako templates in template_dir
for root, dirs, files in os.walk(".", followlinks=True):
if root.startswith("./"):
root = root[2:]
for d in list(dirs):
#Exclude some dirs
d_path = util.path_join(root,d)
if util.should_ignore_path(d_path):
logger.debug("Ignoring directory: " + d_path)
dirs.remove(d)
try:
util.mkdir(util.path_join(self.output_dir, root))
except OSError: #pragma: no cover
pass
for t_fn in files:
t_fn_path = util.path_join(root, t_fn)
if util.should_ignore_path(t_fn_path):
#Ignore this file.
logger.debug("Ignoring file: " + t_fn_path)
continue
elif t_fn.endswith(".mako"):
logger.info("Processing mako file: " + t_fn_path)
#Process this template file
t_name = t_fn[:-5]
t_file = open(t_fn_path)
template = Template(t_file.read().decode("utf-8"),
output_encoding="utf-8",
lookup=self.template_lookup)
#Remember the original path for later when setting context
template.bf_meta = {"path":t_fn_path}
t_file.close()
path = util.path_join(self.output_dir, root, t_name)
html_file = open(path, "w")
html = self.template_render(template)
#Write to disk
html_file.write(html)
else:
#Copy this non-template file
f_path = util.path_join(root, t_fn)
logger.debug("Copying file: " + f_path)
out_path = util.path_join(self.output_dir, f_path)
if self.config.site.overwrite_warning and os.path.exists(out_path):
logger.warn("Location is used more than once: {0}".format(f_path))
if self.bf.config.site.use_hard_links:
# Try hardlinking first, and if that fails copy
try:
os.link(f_path, out_path)
except StandardError:
shutil.copyfile(f_path, out_path)
else:
shutil.copyfile(f_path, out_path)
def materialize_template(self, template_name, location, attrs={}, lookup=None):
"""Render a named template with attrs to a location in the _site dir"""
if lookup==None:
lookup = self.template_lookup
template = lookup.get_template(template_name)
template.output_encoding = "utf-8"
rendered = self.template_render(template, attrs)
path = util.path_join(self.output_dir, location)
#Create the path if it doesn't exist:
util.mkdir(os.path.split(path)[0])
if self.config.site.overwrite_warning and os.path.exists(path):
logger.warn("Location is used more than once: {0}".format(location))
f = open(path, "w")
f.write(rendered)
f.close()
def __setup_output_dir(self):
"""Setup the staging directory"""
if os.path.isdir(self.output_dir): #pragma: no cover
# I *would* just shutil.rmtree the whole thing and recreate it,
# but I want the output_dir to retain its same inode on the
# filesystem to be compatible with some HTTP servers.
# So this just deletes the *contents* of output_dir
for f in os.listdir(self.output_dir):
f = util.path_join(self.output_dir, f)
try:
os.remove(f)
except OSError:
pass
try:
shutil.rmtree(f)
except OSError:
pass
util.mkdir(self.output_dir)
def __setup_output_dir(self):
"""Setup the staging directory"""
if os.path.isdir(self.output_dir): #pragma: no cover
# I *would* just shutil.rmtree the whole thing and recreate it,
# but I want the output_dir to retain its same inode on the
# filesystem to be compatible with some HTTP servers.
# So this just deletes the *contents* of output_dir
for f in os.listdir(self.output_dir):
f = util.path_join(self.output_dir, f)
try:
os.remove(f)
except OSError:
pass
try:
shutil.rmtree(f)
except OSError:
pass
util.mkdir(self.output_dir)
def __run_controllers(self):
"""Run all the controllers in the _controllers directory"""
#Store imported controllers on the bf cache
self.bf.controllers = cache.Cache()
if(not os.path.isdir("_controllers")):
return
for py_file in [p for p in sorted(os.listdir("_controllers")) if
p.endswith(".py")]:
controller_name = (py_file.split(".")[0].replace("-","_"))
import_name = "controller_mod_"+controller_name
mod = imp.load_source(import_name,util.path_join("_controllers",py_file))
setattr(self.bf.controllers,controller_name,mod)
for py_file in [p for p in sorted(os.listdir("_controllers")) if
p.endswith(".py")]:
logger.info("Running controller: "+py_file)
controller_name = (py_file.split(".")[0].replace("-","_"))
mod = getattr(self.bf.controllers,controller_name)
if "run" in dir(mod):
mod.run()
else:
logger.debug("Controller %s has no run() function, skipping it." % py_file)
def __write_files(self):
"""Write all files for the blog to _site
Convert all templates to straight HTML
Copy other non-template files directly"""
#find mako templates in template_dir
for root, dirs, files in os.walk("."):
excluded_roots = []
if root.startswith("./"):
root = root[2:]
for d in list(dirs):
#Exclude some dirs
d_path = util.path_join(root,d)
if util.should_ignore_path(d_path):
logger.debug("Ignoring directory: " + d_path)
dirs.remove(d)
try:
util.mkdir(util.path_join(self.output_dir, root))
except OSError: #pragma: no cover
pass
for t_fn in files:
t_fn_path = util.path_join(root, t_fn)
if util.should_ignore_path(t_fn_path):
#Ignore this file.
logger.debug("Ignoring file: " + t_fn_path)
continue
elif t_fn.endswith(".mako"):
logger.info("Processing mako file: " + t_fn_path)
#Process this template file
t_name = t_fn[:-5]
t_file = open(t_fn_path)
template = Template(t_file.read().decode("utf-8"),
output_encoding="utf-8",
lookup=self.template_lookup)
t_file.close()
path = util.path_join(self.output_dir, root, t_name)
html_file = open(path, "w")
html = self.template_render(template)
#Write to disk
html_file.write(html)
else:
#Copy this non-template file
f_path = util.path_join(root, t_fn)
logger.debug("Copying file: " + f_path)
shutil.copyfile(f_path,
util.path_join(self.output_dir, f_path))
def shell_script_path():
return util.path_join(util.root_path(), "bookmark.sh")
def __init__(self):
Base.__init__(self, None, None)
self.path = util.path_join("cmd/help/bookmark_help_doc.txt")
def __init__(self):
Base.__init__(self, None, None)
self.path = util.path_join("cmd/version/bm.version")
def get_details(self):
path = util.path_join(util.storage_path(), self.name + ".json")
with open(path, "r") as file:
data = json.load(file)
self.details = data
def _get_simple_message(self, page_name):
path = util.path_join(util.storage_path(), page_name)
with open(path, "r") as page_file:
data = json.load(page_file)
url = data["url"]
self.pages[page_name.replace(".json", "")] = url
def open(self):
path = util.path_join(util.storage_path(), self.name + ".json")
with open(path, "r") as file:
data = json.load(file)
webbrowser.open(data["url"])
self.logger.info("open page %s finished." % self.name)
def send_folder(self, src, dest, max_retry=None, delay=None, as_user=None):
"""
Send a local folder to the server
:param src: The file to send
:type src: str
:param dest: Where to place the folder
:type dest: str
:param max_retry: Number of retry in case of failure, default: SshConnection.MAX_ATTEMPT
:type max_retry: int|None
:param delay: The sleep time before a new retry, in ms, default: SshConnection.RETRY_DELAY
:type delay: int|None
:param as_user: run command as user, default None
:type as_user: str|None
:return: True in case of success, raise an exception otherwise
:rtype: bool
"""
max_retry = max_retry if max_retry is not None else SshConnection.MAX_ATTEMPT
delay = delay if delay is not None else SshConnection.RETRY_DELAY
if not os.path.exists(src):
raise RuntimeError(
"source folder " + src +
" doesn't exists, unable to send it to server " + self._ip)
if not os.path.isdir(src):
raise RuntimeError(src +
" is a file, please use send_file instead")
code, out, err = self.run("test -d '" + dest +
"' && echo D || (test -f '" + dest +
"' && echo F || echo N)",
max_retry=max_retry,
delay=delay,
shell=True,
as_user=as_user)
out = out.strip()
if out == "F": # it's a file on destination
raise RuntimeError(dest + " is a file on target server " +
self._ip + ", it can't contains anything")
if out == "N": # it doesn't exists on destination
self.run(["mkdir", "-p", os.path.dirname(dest)],
max_retry=max_retry,
delay=delay,
as_user=as_user)
else: # out == "D": it's a folder on destination
self.run(["rm", "-rf", dest],
max_retry=max_retry,
delay=delay,
as_user=as_user)
dest_folder = dest if not as_user else self.run(
["mktemp", "-d"], max_retry=max_retry, delay=delay)[1]
dest_folder = dest_folder.strip()
i = 0
while True:
try:
cmd = ['scp', '-r']
cmd.extend(SshConnection.SSH_ARGS)
cmd.extend(['-i', self._priv_key_path])
cmd.extend(
[src, self._user + "@" + self._ip + ":" + dest_folder])
new_env = os.environ.copy()
new_env["LC_ALL"] = "en_US.UTF-8"
subprocess.check_call(cmd,
stderr=subprocess.STDOUT,
env=new_env)
if as_user:
tmp_path = util.path_join(dest_folder,
os.path.basename(dest))
self.run(["mv", tmp_path, dest],
max_retry=max_retry,
delay=delay,
as_user=as_user)
self.run(["test", "-d", dest], max_retry=0, as_user=as_user)
return True
except (subprocess.CalledProcessError, RuntimeError):
with error_util.saved_stack() as err:
i += 1
if i >= max_retry:
err.reraise()
time.sleep(float(delay) / 1000)
def drop(self):
path = util.path_join("pages", self.name + ".json")
util.delete_file(path)
self.logger.info("delete page %s successfully" % self.name)
def __init__(self):
c = ConfigParser()
c.read(util.path_join(util.root_path(), "config/bookmark.ini"))
self.bin_py_path = c.get("install", "bin_python_path")
self.python_cmd = c.get("install", "python_command")
self.pip_cmd = c.get("install", "pip_command")
