def _encode_content(name, path):
'''
This is a private subroutine for ``dir_to_fields``
'''
if name.endswith('.json'):
try:
return util.read_json(path, raise_on_error=True)
except ValueError:
logger.error("Json invalid in %s", path)
return ''
try:
content = util.read(path)
except UnicodeDecodeError:
logger.warning("%s isn't encoded in utf8", path)
content = util.read(path, utf8=False)
try:
content.encode('utf-8')
except UnicodeError:
logger.warning("plan B didn't work, %s is a binary", path)
logger.warning("use plan C: encode to base64")
content = ("base64-encoded;%s" % base64.b64encode(content))
return content
def _encode_content(name, path):
'''
This is a private subroutine for ``dir_to_fields``
'''
if name.endswith('.json'):
try:
return util.read_json(path, raise_on_error=True)
except ValueError:
logger.error("Json invalid in %s", path)
return ''
try:
content = util.read(path)
except UnicodeDecodeError:
logger.warning("%s isn't encoded in utf8", path)
content = util.read(path, utf8=False)
try:
content.encode('utf-8')
except UnicodeError:
logger.warning("plan B didn't work, %s is a binary", path)
logger.warning("use plan C: encode to base64")
content = ("base64-encoded;%s" % base64.b64encode(content))
return content
def rjson(mo):
if mo.group(2).startswith('_attachments'):
# someone want to include from attachments
path = os.path.join(app_dir, mo.group(2).strip())
filenum = 0
for filename in glob.iglob(path):
logger.debug("process json macro: %s" % filename)
library = ''
try:
if filename.endswith('.json'):
library = util.read_json(filename)
else:
library = util.read(filename)
except IOError, e:
raise MacroError(str(e))
filenum += 1
current_file = filename.split(app_dir)[1]
fields = current_file.split('/')
count = len(fields)
include_to = included
for i, field in enumerate(fields):
if i + 1 < count:
include_to[field] = {}
include_to = include_to[field]
else:
include_to[field] = library
if not filenum:
raise MacroError("Processing code: No file matching '%s'" %
mo.group(2))
def rjson(mo):
if mo.group(2).startswith('_attachments'):
# someone want to include from attachments
path = os.path.join(app_dir, mo.group(2).strip())
filenum = 0
for filename in glob.iglob(path):
logger.debug("process json macro: %s" % filename)
library = ''
try:
if filename.endswith('.json'):
library = util.read_json(filename)
else:
library = util.read(filename)
except IOError, e:
raise MacroError(str(e))
filenum += 1
current_file = filename.split(app_dir)[1]
fields = current_file.split('/')
count = len(fields)
include_to = included
for i, field in enumerate(fields):
if i+1 < count:
include_to[field] = {}
include_to = include_to[field]
else:
include_to[field] = library
if not filenum:
raise MacroError("Processing code: No file matching '%s'" %
mo.group(2))
def compress_css(self, css):
re_url = re.compile('url\s*\(([^\s"].*)\)')
src_fpath = ''
fname_dir = ''
def replace_url(mo):
""" make sure urls are relative to css path """
css_url = mo.group(0)[4:].strip(")").replace("'",
"").replace('"', '')
css_path = os.path.join(os.path.dirname(src_fpath), css_url)
rel_path = util.relpath(css_path, fname_dir)
return "url(%s)" % rel_path
for fname, src_files in css.iteritems():
output_css = ''
dest_path = os.path.join(self.attach_dir, fname)
fname_dir = os.path.dirname(dest_path)
for src_fname in src_files:
src_fpath = os.path.join(self.appdir, src_fname)
if os.path.exists(src_fpath):
content_css = str(
compress_css.CSSParser(util.read(src_fpath)))
content_css = re_url.sub(replace_url, content_css)
output_css += content_css
logger.debug("Merging %s in %s" % (src_fname, fname))
if not os.path.isdir(fname_dir):
os.makedirs(fname_dir)
util.write(dest_path, output_css)
def compress_css(self, css):
re_url = re.compile('url\s*\(([^\s"].*)\)')
src_fpath = ''
fname_dir = ''
def replace_url(mo):
""" make sure urls are relative to css path """
css_url = mo.group(0)[4:].strip(")").replace("'", "").replace('"',
'')
css_path = os.path.join(os.path.dirname(src_fpath), css_url)
rel_path = util.relpath(css_path, fname_dir)
return "url(%s)" % rel_path
for fname, src_files in css.iteritems():
output_css = ''
dest_path = os.path.join(self.attach_dir, fname)
fname_dir = os.path.dirname(dest_path)
for src_fname in src_files:
src_fpath = os.path.join(self.appdir, src_fname)
if os.path.exists(src_fpath):
content_css = \
str(compress_css.CSSParser(util.read(src_fpath)))
content_css = re_url.sub(replace_url, content_css)
output_css += content_css
logger.debug("Merging %s in %s" % (src_fname, fname))
if not os.path.isdir(fname_dir):
os.makedirs(fname_dir)
util.write(dest_path, output_css)
def get_id(self):
"""
if there is an _id file, docid is extracted from it,
else we take the current folder name.
"""
idfile = os.path.join(self.docdir, '_id')
if os.path.exists(idfile):
docid = util.read(idfile).split("\n")[0].strip()
if docid: return docid
return "_design/%s" % os.path.split(self.docdir)[1]
def get_id(self):
"""
if there is an ``_id`` file, docid is extracted from it,
else we take the current folder name.
"""
idfile = os.path.join(self.docdir, '_id')
if os.path.exists(idfile):
docid = util.read(idfile).split("\n")[0].strip()
if docid:
return docid
dirname = os.path.split(self.docdir)[1]
return '_design/%s' % dirname if self.is_ddoc else dirname
def rreq(mo):
# just read the file and return it
path = os.path.join(app_dir, mo.group(2).strip())
library = ''
filenum = 0
for filename in glob.iglob(path):
logger.debug("process code macro: %s" % filename)
try:
cnt = util.read(filename)
if cnt.find("!code") >= 0:
cnt = run_code_macros(cnt, app_dir)
library += cnt
except IOError, e:
raise MacroError(str(e))
filenum += 1
def rreq(mo):
# just read the file and return it
path = os.path.join(app_dir, mo.group(2).strip())
library = ''
filenum = 0
for filename in glob.iglob(path):
logger.debug("process code macro: %s" % filename)
try:
cnt = util.read(filename)
if cnt.find("!code") >= 0:
cnt = run_code_macros(cnt, app_dir)
library += cnt
except IOError, e:
raise MacroError(str(e))
filenum += 1
def compress_js(self, backend, js):
logger.info("compress js with %s " % backend.__about__)
for fname, src_files in js.iteritems():
output_js = ''
dest_path = os.path.join(self.attach_dir, fname)
fname_dir = os.path.dirname(dest_path)
for src_fname in src_files:
src_fpath = os.path.join(self.appdir, src_fname)
if os.path.isfile(src_fpath):
output_js += "/* %s */\n" % src_fpath
output_js += util.read(src_fpath)
logger.debug("merging %s in %s" % (src_fname, fname))
if not os.path.isdir(fname_dir):
os.makedirs(fname_dir)
output_js = backend.compress(output_js)
util.write(dest_path, output_js)
def compress_js(self, backend, js):
logger.info("compress js with %s " % backend.__about__)
for fname, src_files in js.iteritems():
output_js = ''
dest_path = os.path.join(self.attach_dir, fname)
fname_dir = os.path.dirname(dest_path)
for src_fname in src_files:
src_fpath = os.path.join(self.appdir, src_fname)
if os.path.isfile(src_fpath):
output_js += "/* %s */\n" % src_fpath
output_js += util.read(src_fpath)
logger.debug("merging %s in %s" % (src_fname, fname))
if not os.path.isdir(fname_dir):
os.makedirs(fname_dir)
output_js = backend.compress(output_js)
util.write(dest_path, output_js)
def dir_to_fields(self, current_dir="", depth=0, manifest=[]):
""" process a directory and get all members """
fields = {}
if not current_dir:
current_dir = self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
if name.startswith("."):
continue
elif self.check_ignore(name):
continue
elif depth == 0 and name.startswith("_"):
# files starting with "_" are always "special"
continue
elif name == "_attachments":
continue
elif depth == 0 and (name == "couchapp" or name == "couchapp.json"):
# we are in app_meta
if name == "couchapp":
manifest.append("%s/" % rel_path)
content = self.dir_to_fields(current_path, depth=depth + 1, manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
if not isinstance(content, dict):
content = {"meta": content}
if "signatures" in content:
del content["signatures"]
if "manifest" in content:
del content["manifest"]
if "objects" in content:
del content["objects"]
if "length" in content:
del content["length"]
if "couchapp" in fields:
fields["couchapp"].update(content)
else:
fields["couchapp"] = content
elif os.path.isdir(current_path):
manifest.append("%s/" % rel_path)
fields[name] = self.dir_to_fields(current_path, depth=depth + 1, manifest=manifest)
else:
logger.debug("push %s", rel_path)
content = ""
if name.endswith(".json"):
try:
content = util.read_json(current_path)
except ValueError:
logger.error("Json invalid in %s", current_path)
else:
try:
content = util.read(current_path).strip()
except UnicodeDecodeError:
logger.warning("%s isn't encoded in utf8", current_path)
content = util.read(current_path, utf8=False)
try:
content.encode("utf-8")
except UnicodeError:
logger.warning("plan B didn't work, " "%s is a binary", current_path)
logger.warning("use plan C: encode to base64")
content = "base64-encoded;%s" % base64.b64encode(content)
# remove extension
name, ext = os.path.splitext(name)
if name in fields:
logger.warning(
"%(name)s is already in properties. " "Can't add (%(fqn)s)", {"name": name, "fqn": rel_path}
)
else:
manifest.append(rel_path)
fields[name] = content
return fields
def dir_to_fields(self, current_dir='', depth=0,
manifest=[]):
""" process a directory and get all members """
fields={}
if not current_dir:
current_dir = self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
if name.startswith("."):
continue
elif self.check_ignore(name):
continue
elif depth == 0 and name.startswith('_'):
# files starting with "_" are always "special"
continue
elif name == '_attachments':
continue
elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'):
# we are in app_meta
if name == "couchapp":
manifest.append('%s/' % rel_path)
content = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
if not isinstance(content, dict):
content = { "meta": content }
if 'signatures' in content:
del content['signatures']
if 'manifest' in content:
del content['manifest']
if 'objects' in content:
del content['objects']
if 'length' in content:
del content['length']
if 'couchapp' in fields:
fields['couchapp'].update(content)
else:
fields['couchapp'] = content
elif os.path.isdir(current_path):
manifest.append('%s/' % rel_path)
fields[name] = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
logger.debug("push %s" % rel_path)
content = ''
if name.endswith('.json'):
try:
content = util.read_json(current_path)
except ValueError:
logger.error("Json invalid in %s" % current_path)
else:
try:
content = util.read(current_path).strip()
except UnicodeDecodeError, e:
logger.warning("%s isn't encoded in utf8" % current_path)
content = util.read(current_path, utf8=False)
try:
content.encode('utf-8')
except UnicodeError, e:
logger.warning(
"plan B didn't work, %s is a binary" % current_path)
logger.warning("use plan C: encode to base64")
content = "base64-encoded;%s" % base64.b64encode(
content)
# remove extension
name, ext = os.path.splitext(name)
if name in fields and ext in ('.txt'):
logger.warning(
"%(name)s is already in properties. Can't add (%(name)s%(ext)s)" % {
"name": name, "ext": ext })
else:
manifest.append(rel_path)
fields[name] = content
def dir_to_fields(self, current_dir='', depth=0,
manifest=[]):
""" process a directory and get all members """
fields={}
if not current_dir:
current_dir = self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
if name.startswith("."):
continue
elif self.check_ignore(name):
continue
elif depth == 0 and name.startswith('_'):
# files starting with "_" are always "special"
continue
elif name == '_attachments':
continue
elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'):
# we are in app_meta
if name == "couchapp":
manifest.append('%s/' % rel_path)
content = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
if not isinstance(content, dict):
content = { "meta": content }
if 'signatures' in content:
del content['signatures']
if 'manifest' in content:
del content['manifest']
if 'objects' in content:
del content['objects']
if 'length' in content:
del content['length']
if 'couchapp' in fields:
fields['couchapp'].update(content)
else:
fields['couchapp'] = content
elif os.path.isdir(current_path):
manifest.append('%s/' % rel_path)
fields[name] = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
logger.debug("push %s" % rel_path)
content = ''
if name.endswith('.json'):
try:
content = util.read_json(current_path)
except ValueError:
logger.error("Json invalid in %s" % current_path)
else:
try:
content = util.read(current_path).strip()
except UnicodeDecodeError, e:
logger.warning("%s isn't encoded in utf8" % current_path)
content = util.read(current_path, utf8=False)
try:
content.encode('utf-8')
except UnicodeError, e:
logger.warning(
"plan B didn't work, %s is a binary" % current_path)
logger.warning("use plan C: encode to base64")
content = "base64-encoded;%s" % base64.b64encode(
content)
# remove extension
name, ext = os.path.splitext(name)
if name in fields:
logger.warning(
"%(name)s is already in properties. Can't add (%(fqn)s)" % {
"name": name, "fqn": rel_path })
else:
manifest.append(rel_path)
fields[name] = content