def _process_attachments(self, path, vendor=None):
"""
Processing directory to yield attachments.
"""
if not os.path.isdir(path):
raise StopIteration()
for root, dirs, files in os.walk(path):
for dir_ in dirs:
_relpath = util.relpath(os.path.join(root, dir_),
self.docdir)
if self.check_ignore(_relpath):
dirs.remove(dir_)
if not files:
continue
for filename in files:
filepath = os.path.join(root, filename)
_relpath = util.relpath(filepath, self.docdir)
if self.check_ignore(_relpath):
continue
name = util.relpath(filepath, path)
if vendor is not None:
name = os.path.join('vendor', vendor, name)
name = _replace_backslash(name)
yield (name, filepath)
def _process_attachments(self, path, vendor=None):
"""
Processing directory to yield attachments.
"""
if not os.path.isdir(path):
raise StopIteration()
for root, dirs, files in os.walk(path):
for dir_ in dirs:
_relpath = util.relpath(os.path.join(root, dir_), self.docdir)
if self.check_ignore(_relpath):
dirs.remove(dir_)
if not files:
continue
for filename in files:
filepath = os.path.join(root, filename)
_relpath = util.relpath(filepath, self.docdir)
if self.check_ignore(_relpath):
continue
name = util.relpath(filepath, path)
if vendor is not None:
name = os.path.join('vendor', vendor, name)
name = _replace_backslash(name)
yield (name, filepath)
def copy_helper(path, directory, tname="templates"):
""" copy helper used to generate an app"""
if tname == "vendor":
tname = os.path.join("templates", tname)
templatedir = find_template_dir(tname, directory)
if templatedir:
if directory == "vendor":
path = os.path.join(path, directory)
try:
os.makedirs(path)
except:
pass
for root, dirs, files in os.walk(templatedir):
rel = relpath(root, templatedir)
if rel == ".":
rel = ""
target_path = os.path.join(path, rel)
for d in dirs:
try:
os.makedirs(os.path.join(target_path, d))
except:
continue
for f in files:
shutil.copy2(os.path.join(root, f),
os.path.join(target_path, f))
else:
raise AppError(
"Can't create a CouchApp in %s: default template not found." %
(path))
def copy_helper(path, directory, tname="templates"):
""" copy helper used to generate an app"""
if tname == "vendor":
tname = os.path.join("templates", tname)
templatedir = find_template_dir(tname, directory)
if templatedir:
if directory == "vendor":
path = os.path.join(path, directory)
try:
os.makedirs(path)
except:
pass
for root, dirs, files in os.walk(templatedir):
rel = relpath(root, templatedir)
if rel == ".":
rel = ""
target_path = os.path.join(path, rel)
for d in dirs:
try:
os.makedirs(os.path.join(target_path, d))
except:
continue
for f in files:
shutil.copy2(os.path.join(root, f),
os.path.join(target_path, f))
else:
raise AppError(
"Can't create a CouchApp in %s: default template not found." %
(path))
def _process_attachments(self, path, vendor=None):
""" the function processing directory to yeld
attachments. """
if os.path.isdir(path):
for root, dirs, files in os.walk(path, topdown=True, followlinks=True):
for dirname in dirs:
if dirname.startswith('.'):
dirs.remove(dirname)
elif self.check_ignore(dirname):
dirs.remove(dirname)
if files:
for filename in files:
if filename.startswith('.'):
continue
if os.path.islink(os.path.join(root, filename)):
continue
elif self.check_ignore(filename):
continue
else:
filepath = os.path.join(root, filename)
name = util.relpath(filepath, path)
if vendor is not None:
name = os.path.join('vendor', vendor, name)
name = _replace_backslash(name)
yield (name, os.path.abspath(filepath))
def replace_url(mo):
""" make sure urls are relative to css path """
css_url = mo.group(0)[4:].strip(")").replace("'",
"").replace('"', '')
css_path = os.path.join(os.path.dirname(src_fpath), css_url)
rel_path = util.relpath(css_path, fname_dir)
return "url(%s)" % rel_path
def replace_url(mo):
""" make sure urls are relative to css path """
css_url = mo.group(0)[4:].strip(")").replace("'", "").replace('"',
'')
css_path = os.path.join(os.path.dirname(src_fpath), css_url)
rel_path = util.relpath(css_path, fname_dir)
return "url(%s)" % rel_path
def dir_to_fields(self, current_dir=None, depth=0, manifest=None):
"""
Process a directory and get all members
:param manifest: ``list``. We will have side effect on this param.
"""
fields = {} # return value
manifest = manifest if manifest is not None else []
current_dir = current_dir if current_dir else self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(
util.relpath(current_path, self.docdir))
if name.startswith('.'):
continue
elif self.check_ignore(rel_path):
continue
elif depth == 0 and name.startswith('_'):
# files starting with "_" are always "special"
continue
elif name == '_attachments':
continue
elif depth == 0 and (name in ('couchapp', 'couchapp.json')):
# we are in app_meta
if name == "couchapp":
manifest.append('%s/' % rel_path)
content = self.dir_to_fields(current_path,
depth=depth + 1,
manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
fields, content = self._meta_to_fields(fields, content)
elif os.path.isdir(current_path):
manifest.append('%s/' % rel_path)
fields[name] = self.dir_to_fields(current_path,
depth=depth + 1,
manifest=manifest)
else: # handler for normal file
logger.debug('push %s', rel_path)
# remove extension
_name, ext = os.path.splitext(name)
if _name in fields:
logger.warning(
"%(name)s is already in properties. "
"Can't add (%(fqn)s)", {
'name': _name,
'fqn': rel_path
})
else:
manifest.append(rel_path)
fields[_name] = self._encode_content(name, current_path)
return fields
def dir_to_fields(self, current_dir=None, depth=0, manifest=None):
"""
Process a directory and get all members
:param manifest: ``list``. We will have side effect on this param.
"""
fields = {} # return value
manifest = manifest if manifest is not None else []
current_dir = current_dir if current_dir else self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(util.relpath(current_path,
self.docdir))
if name.startswith('.'):
continue
elif self.check_ignore(rel_path):
continue
elif depth == 0 and name.startswith('_'):
# files starting with "_" are always "special"
continue
elif name == '_attachments':
continue
elif depth == 0 and (name in ('couchapp', 'couchapp.json')):
# we are in app_meta
if name == "couchapp":
manifest.append('%s/' % rel_path)
content = self.dir_to_fields(
current_path, depth=depth + 1, manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
fields, content = self._meta_to_fields(fields, content)
elif os.path.isdir(current_path):
manifest.append('%s/' % rel_path)
fields[name] = self.dir_to_fields(
current_path, depth=depth + 1, manifest=manifest)
else: # handler for normal file
logger.debug('push %s', rel_path)
# remove extension
_name, ext = os.path.splitext(name)
if _name in fields:
logger.warning("%(name)s is already in properties. "
"Can't add (%(fqn)s)",
{'name': _name, 'fqn': rel_path})
else:
manifest.append(rel_path)
fields[_name] = self._encode_content(name, current_path)
return fields
def _process_attachments(self, path, vendor=None):
""" the function processing directory to yeld
attachments. """
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for dirname in dirs:
if self.check_ignore(dirname):
dirs.remove(dirname)
if files:
for filename in files:
if self.check_ignore(filename):
continue
else:
filepath = os.path.join(root, filename)
name = util.relpath(filepath, path)
if vendor is not None:
name = os.path.join("vendor", vendor, name)
name = _replace_backslash(name)
yield (name, filepath)
def _process_attachments(self, path, vendor=None):
""" the function processing directory to yeld
attachments. """
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for dirname in dirs:
if self.check_ignore(dirname):
dirs.remove(dirname)
if files:
for filename in files:
if self.check_ignore(filename):
continue
else:
filepath = os.path.join(root, filename)
name = util.relpath(filepath, path)
if vendor is not None:
name = os.path.join('vendor', vendor, name)
name = _replace_backslash(name)
yield (name, filepath)
def dir_to_fields(self, current_dir="", depth=0, manifest=[]):
""" process a directory and get all members """
fields = {}
if not current_dir:
current_dir = self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
if name.startswith("."):
continue
elif self.check_ignore(name):
continue
elif depth == 0 and name.startswith("_"):
# files starting with "_" are always "special"
continue
elif name == "_attachments":
continue
elif depth == 0 and (name == "couchapp" or name == "couchapp.json"):
# we are in app_meta
if name == "couchapp":
manifest.append("%s/" % rel_path)
content = self.dir_to_fields(current_path, depth=depth + 1, manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
if not isinstance(content, dict):
content = {"meta": content}
if "signatures" in content:
del content["signatures"]
if "manifest" in content:
del content["manifest"]
if "objects" in content:
del content["objects"]
if "length" in content:
del content["length"]
if "couchapp" in fields:
fields["couchapp"].update(content)
else:
fields["couchapp"] = content
elif os.path.isdir(current_path):
manifest.append("%s/" % rel_path)
fields[name] = self.dir_to_fields(current_path, depth=depth + 1, manifest=manifest)
else:
logger.debug("push %s", rel_path)
content = ""
if name.endswith(".json"):
try:
content = util.read_json(current_path)
except ValueError:
logger.error("Json invalid in %s", current_path)
else:
try:
content = util.read(current_path).strip()
except UnicodeDecodeError:
logger.warning("%s isn't encoded in utf8", current_path)
content = util.read(current_path, utf8=False)
try:
content.encode("utf-8")
except UnicodeError:
logger.warning("plan B didn't work, " "%s is a binary", current_path)
logger.warning("use plan C: encode to base64")
content = "base64-encoded;%s" % base64.b64encode(content)
# remove extension
name, ext = os.path.splitext(name)
if name in fields:
logger.warning(
"%(name)s is already in properties. " "Can't add (%(fqn)s)", {"name": name, "fqn": rel_path}
)
else:
manifest.append(rel_path)
fields[name] = content
return fields
def dir_to_fields(self, current_dir='', depth=0,
manifest=[]):
""" process a directory and get all members """
fields={}
if not current_dir:
current_dir = self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
if name.startswith("."):
continue
elif self.check_ignore(name):
continue
elif depth == 0 and name.startswith('_'):
# files starting with "_" are always "special"
continue
elif name == '_attachments':
continue
elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'):
# we are in app_meta
if name == "couchapp":
manifest.append('%s/' % rel_path)
content = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
if not isinstance(content, dict):
content = { "meta": content }
if 'signatures' in content:
del content['signatures']
if 'manifest' in content:
del content['manifest']
if 'objects' in content:
del content['objects']
if 'length' in content:
del content['length']
if 'couchapp' in fields:
fields['couchapp'].update(content)
else:
fields['couchapp'] = content
elif os.path.isdir(current_path):
manifest.append('%s/' % rel_path)
fields[name] = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
logger.debug("push %s" % rel_path)
content = ''
if name.endswith('.json'):
try:
content = util.read_json(current_path)
except ValueError:
logger.error("Json invalid in %s" % current_path)
else:
try:
content = util.read(current_path).strip()
except UnicodeDecodeError, e:
logger.warning("%s isn't encoded in utf8" % current_path)
content = util.read(current_path, utf8=False)
try:
content.encode('utf-8')
except UnicodeError, e:
logger.warning(
"plan B didn't work, %s is a binary" % current_path)
logger.warning("use plan C: encode to base64")
content = "base64-encoded;%s" % base64.b64encode(
content)
# remove extension
name, ext = os.path.splitext(name)
if name in fields and ext in ('.txt'):
logger.warning(
"%(name)s is already in properties. Can't add (%(name)s%(ext)s)" % {
"name": name, "ext": ext })
else:
manifest.append(rel_path)
fields[name] = content
def dir_to_fields(self, current_dir='', depth=0,
manifest=[]):
""" process a directory and get all members """
fields={}
if not current_dir:
current_dir = self.docdir
for name in os.listdir(current_dir):
current_path = os.path.join(current_dir, name)
rel_path = _replace_backslash(util.relpath(current_path, self.docdir))
if name.startswith("."):
continue
elif self.check_ignore(name):
continue
elif depth == 0 and name.startswith('_'):
# files starting with "_" are always "special"
continue
elif name == '_attachments':
continue
elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'):
# we are in app_meta
if name == "couchapp":
manifest.append('%s/' % rel_path)
content = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
manifest.append(rel_path)
content = util.read_json(current_path)
if not isinstance(content, dict):
content = { "meta": content }
if 'signatures' in content:
del content['signatures']
if 'manifest' in content:
del content['manifest']
if 'objects' in content:
del content['objects']
if 'length' in content:
del content['length']
if 'couchapp' in fields:
fields['couchapp'].update(content)
else:
fields['couchapp'] = content
elif os.path.isdir(current_path):
manifest.append('%s/' % rel_path)
fields[name] = self.dir_to_fields(current_path,
depth=depth+1, manifest=manifest)
else:
logger.debug("push %s" % rel_path)
content = ''
if name.endswith('.json'):
try:
content = util.read_json(current_path)
except ValueError:
logger.error("Json invalid in %s" % current_path)
else:
try:
content = util.read(current_path).strip()
except UnicodeDecodeError, e:
logger.warning("%s isn't encoded in utf8" % current_path)
content = util.read(current_path, utf8=False)
try:
content.encode('utf-8')
except UnicodeError, e:
logger.warning(
"plan B didn't work, %s is a binary" % current_path)
logger.warning("use plan C: encode to base64")
content = "base64-encoded;%s" % base64.b64encode(
content)
# remove extension
name, ext = os.path.splitext(name)
if name in fields:
logger.warning(
"%(name)s is already in properties. Can't add (%(fqn)s)" % {
"name": name, "fqn": rel_path })
else:
manifest.append(rel_path)
fields[name] = content